This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 4687799928 [python] Cast WHERE literals for DECIMAL, DATE and TIME
columns (#10119)
4687799928 is described below
commit 4687799928fdad925ba98eac417e204e20642167
Author: jackylee <[email protected]>
AuthorDate: Thu Sep 24 17:39:32 2026 +0800
[python] Cast WHERE literals for DECIMAL, DATE and TIME columns (#10119)
---
paimon-python/pypaimon/common/where_parser.py | 83 ++++++++++++-
.../pypaimon/tests/py36/where_literal_cast_test.py | 63 ++++++++++
paimon-python/pypaimon/tests/where_parser_test.py | 130 ++++++++++++++++++++-
3 files changed, 273 insertions(+), 3 deletions(-)
diff --git a/paimon-python/pypaimon/common/where_parser.py
b/paimon-python/pypaimon/common/where_parser.py
index 3286c81a5c..1ba78986a5 100644
--- a/paimon-python/pypaimon/common/where_parser.py
+++ b/paimon-python/pypaimon/common/where_parser.py
@@ -40,6 +40,8 @@ Examples:
"age > 18 OR (name = 'Bob' AND status = 'active')"
"""
+import datetime
+import decimal
import re
from typing import Any, Dict, List, Optional
@@ -114,10 +116,79 @@ def _build_field_type_map(fields: List[DataField]) ->
Dict[str, Optional[str]]:
return result
+def _cast_decimal(value_str: str, type_name: str) -> decimal.Decimal:
+ """Cast a DECIMAL literal, rescaled to the column's scale.
+
+ A DECIMAL column reads back as decimal.Decimal, and the pushed-down Arrow
+ filter binds the literal's own scale rather than the column's, so a literal
+ written at a different scale (e.g. ``50`` against a ``DECIMAL(10, 2)``
+ ``50.00``) would silently match nothing. Rescale to the column scale when
the
+ value is exact there; otherwise keep it as written (it then correctly
matches
+ no row). A malformed literal is re-raised as ValueError, which
+ parse_where_clause documents and the CLI relies on.
+ """
+ try:
+ value = decimal.Decimal(value_str)
+ except decimal.InvalidOperation:
+ raise ValueError(f"Invalid DECIMAL literal: {value_str!r}") from None
+
+ match = re.search(r'\(\s*(\d+)\s*,\s*(\d+)\s*\)', type_name)
+ precision = int(match.group(1)) if match else 38
+ scale = int(match.group(2)) if match else 0
+ # Quantize in a context wide enough for the column precision. The default
+ # context caps precision at 28, so a DECIMAL(38, 2) literal in integer form
+ # would raise InvalidOperation here, fall through unscaled, and then match
+ # nothing once the Arrow filter binds it at scale 0.
+ context = decimal.Context(prec=max(precision, 1))
+ try:
+ rescaled = value.quantize(decimal.Decimal(1).scaleb(-scale),
context=context)
+ except decimal.InvalidOperation:
+ return value
+ return rescaled if rescaled == value else value
+
+
+def _cast_date(value_str: str) -> datetime.date:
+ """Parse a DATE literal as ``YYYY-MM-DD``.
+
+ ``datetime.date.fromisoformat`` only exists on Python 3.7+, while pypaimon
+ still declares ``python_requires >= 3.6`` and runs a 3.6 test lane, so
parse
+ through ``strptime`` instead.
+ """
+ try:
+ return datetime.datetime.strptime(value_str.strip(), '%Y-%m-%d').date()
+ except ValueError:
+ raise ValueError(f"Invalid DATE literal: {value_str!r}") from None
+
+
+_TIME_PATTERN = re.compile(r'(\d{1,2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?)?')
+
+
+def _cast_time(value_str: str) -> datetime.time:
+ """Parse a TIME literal as a wall-clock ``HH:MM[:SS[.ffffff]]``.
+
+ ``datetime.time.fromisoformat`` only exists on Python 3.7+, and it also
+ accepts a UTC offset (``12:30:00+01:00``) that a Paimon TIME -- which has
no
+ time zone -- cannot represent; Arrow would then compare only the wall-clock
+ part and match a value the user did not ask for. Parse a plain time here
and
+ reject any offset.
+ """
+ match = _TIME_PATTERN.fullmatch(value_str.strip())
+ if match is None:
+ raise ValueError(f"Invalid TIME literal: {value_str!r}")
+ hour, minute, second, fraction = match.groups()
+ microsecond = int(fraction.ljust(6, '0')) if fraction else 0
+ try:
+ return datetime.time(int(hour), int(minute),
+ int(second) if second else 0, microsecond)
+ except ValueError:
+ raise ValueError(f"Invalid TIME literal: {value_str!r}") from None
+
+
def _cast_literal(value_str: str, type_name: str) -> Any:
"""Cast a literal string to the appropriate Python type based on the field
type."""
integer_types = {'TINYINT', 'SMALLINT', 'INT', 'INTEGER', 'BIGINT'}
float_types = {'FLOAT', 'DOUBLE'}
+ decimal_types = {'DECIMAL', 'NUMERIC', 'DEC'}
base_type = type_name.split('(')[0].strip()
@@ -125,10 +196,18 @@ def _cast_literal(value_str: str, type_name: str) -> Any:
return int(value_str)
if base_type in float_types:
return float(value_str)
- if base_type.startswith('DECIMAL') or base_type in ('DECIMAL', 'NUMERIC',
'DEC'):
- return float(value_str)
+ if base_type in decimal_types:
+ return _cast_decimal(value_str, type_name)
if base_type == 'BOOLEAN':
return value_str.lower() in ('true', '1', 'yes')
+ if base_type == 'DATE':
+ # DATE/TIME columns read back as datetime.date / datetime.time;
leaving the
+ # literal a string makes the arrow comparison kernel raise instead of
+ # filtering. TIMESTAMP is intentionally left out: its LOCAL TIME ZONE
form
+ # reads back tz-aware and needs dedicated normalization.
+ return _cast_date(value_str)
+ if base_type == 'TIME':
+ return _cast_time(value_str)
return value_str
diff --git a/paimon-python/pypaimon/tests/py36/where_literal_cast_test.py
b/paimon-python/pypaimon/tests/py36/where_literal_cast_test.py
new file mode 100644
index 0000000000..98daef9bb3
--- /dev/null
+++ b/paimon-python/pypaimon/tests/py36/where_literal_cast_test.py
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""WHERE literal casting on the Python 3.6/3.7 lane.
+
+The full ``where_parser_test`` is excluded from the 3.6/3.7 subset, yet DATE
and
+TIME casting must stay off ``date.fromisoformat`` / ``time.fromisoformat``
(both
+3.7+). These pure-casting checks run on that lane to guard the floor.
+"""
+
+import datetime
+import decimal
+import unittest
+
+from pypaimon.common.where_parser import _cast_literal
+
+
+class WhereLiteralCastPy36Test(unittest.TestCase):
+
+ def test_cast_date(self):
+ self.assertEqual(_cast_literal('2024-01-01', 'DATE'),
+ datetime.date(2024, 1, 1))
+
+ def test_cast_time(self):
+ self.assertEqual(_cast_literal('12:30:00', 'TIME(0)'),
+ datetime.time(12, 30, 0))
+
+ def test_cast_time_with_fraction(self):
+ self.assertEqual(_cast_literal('12:30:00.5', 'TIME(3)'),
+ datetime.time(12, 30, 0, 500000))
+
+ def test_cast_time_rejects_offset(self):
+ with self.assertRaises(ValueError):
+ _cast_literal('12:30:00+01:00', 'TIME(0)')
+
+ def test_cast_date_rejects_malformed(self):
+ with self.assertRaises(ValueError):
+ _cast_literal('not-a-date', 'DATE')
+
+ def test_cast_high_precision_decimal_rescales(self):
+ value = _cast_literal('123456789012345678901234567890123456',
+ 'DECIMAL(38, 2)')
+ self.assertEqual(value.as_tuple().exponent, -2)
+ self.assertEqual(
+ value, decimal.Decimal('123456789012345678901234567890123456.00'))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/paimon-python/pypaimon/tests/where_parser_test.py
b/paimon-python/pypaimon/tests/where_parser_test.py
index c5ba552ac7..4f38d71a91 100644
--- a/paimon-python/pypaimon/tests/where_parser_test.py
+++ b/paimon-python/pypaimon/tests/where_parser_test.py
@@ -15,8 +15,16 @@
# specific language governing permissions and limitations
# under the License.
+import datetime
+import decimal
+import os
+import shutil
+import tempfile
import unittest
+import pyarrow as pa
+
+from pypaimon import CatalogFactory, Schema
from pypaimon.common.where_parser import parse_where_clause, _tokenize,
_cast_literal
from pypaimon.schema.data_types import ArrayType, AtomicType, DataField
@@ -83,7 +91,43 @@ class WhereParserCastLiteralTest(unittest.TestCase):
self.assertAlmostEqual(_cast_literal('3.14', 'DOUBLE'), 3.14)
def test_cast_decimal(self):
- self.assertAlmostEqual(_cast_literal('99.99', 'DECIMAL(10,2)'), 99.99)
+ # A DECIMAL literal must stay exact (decimal.Decimal); a float would
never
+ # compare equal to the Decimal a DECIMAL column reads back as.
+ value = _cast_literal('99.99', 'DECIMAL(10,2)')
+ self.assertIsInstance(value, decimal.Decimal)
+ self.assertEqual(value, decimal.Decimal('99.99'))
+
+ def test_cast_decimal_high_precision_rescales(self):
+ # A DECIMAL(38, 2) literal in integer form exceeds the default decimal
+ # context (precision 28); without a column-wide context the rescale
would
+ # be dropped and the literal would bind at scale 0.
+ value = _cast_literal('123456789012345678901234567890123456',
+ 'DECIMAL(38, 2)')
+ self.assertEqual(value.as_tuple().exponent, -2)
+ self.assertEqual(
+ value, decimal.Decimal('123456789012345678901234567890123456.00'))
+
+ def test_cast_date(self):
+ value = _cast_literal('2024-01-01', 'DATE')
+ self.assertEqual(value, datetime.date(2024, 1, 1))
+
+ def test_cast_time(self):
+ value = _cast_literal('12:30:00', 'TIME(0)')
+ self.assertEqual(value, datetime.time(12, 30, 0))
+
+ def test_cast_time_with_fraction(self):
+ value = _cast_literal('12:30:00.5', 'TIME(3)')
+ self.assertEqual(value, datetime.time(12, 30, 0, 500000))
+
+ def test_cast_time_rejects_offset(self):
+ # A Paimon TIME has no time zone; an offset-bearing literal must not
+ # silently drop the offset and match the wall-clock value.
+ with self.assertRaises(ValueError):
+ _cast_literal('12:30:00+01:00', 'TIME(0)')
+
+ def test_cast_time_rejects_malformed(self):
+ with self.assertRaises(ValueError):
+ _cast_literal('25:00:00', 'TIME(0)')
def test_cast_boolean(self):
self.assertTrue(_cast_literal('true', 'BOOLEAN'))
@@ -400,5 +444,89 @@ class WhereParserParseTest(unittest.TestCase):
self.assertIn("tags", str(context.exception))
+class WhereParserScanTest(unittest.TestCase):
+ """End-to-end: a WHERE clause on DECIMAL/DATE/TIME columns returns the
row."""
+
+ @classmethod
+ def setUpClass(cls):
+ cls.tempdir = tempfile.mkdtemp()
+ cls.catalog = CatalogFactory.create(
+ {'warehouse': os.path.join(cls.tempdir, 'warehouse')})
+ cls.catalog.create_database('default', False)
+ pa_schema = pa.schema([
+ ('id', pa.int32()),
+ ('price', pa.decimal128(10, 2)),
+ ('big', pa.decimal128(38, 2)),
+ ('d', pa.date32()),
+ ('t', pa.time32('ms')),
+ ])
+ cls.catalog.create_table(
+ 'default.where_literal_types',
Schema.from_pyarrow_schema(pa_schema), False)
+ cls.table = cls.catalog.get_table('default.where_literal_types')
+ data = pa.table({
+ 'id': pa.array([1, 2], pa.int32()),
+ 'price': pa.array(
+ [decimal.Decimal('99.99'), decimal.Decimal('50.00')],
pa.decimal128(10, 2)),
+ 'big': pa.array(
+ [decimal.Decimal('123456789012345678901234567890123456.00'),
+ decimal.Decimal('1.00')], pa.decimal128(38, 2)),
+ 'd': pa.array(
+ [datetime.date(2024, 1, 1), datetime.date(2020, 1, 1)],
pa.date32()),
+ 't': pa.array(
+ [datetime.time(12, 30, 0), datetime.time(1, 0, 0)],
pa.time32('ms')),
+ })
+ wb = cls.table.new_batch_write_builder()
+ writer = wb.new_write()
+ commit = wb.new_commit()
+ writer.write_arrow(data)
+ commit.commit(writer.prepare_commit())
+ writer.close()
+ commit.close()
+
+ @classmethod
+ def tearDownClass(cls):
+ shutil.rmtree(cls.tempdir, ignore_errors=True)
+
+ def _scan_ids(self, where):
+ predicate = parse_where_clause(where, self.table.table_schema.fields)
+ read_builder = self.table.new_read_builder().with_filter(predicate)
+ splits = read_builder.new_scan().plan().splits()
+ return
read_builder.new_read().to_arrow(splits).column('id').to_pylist()
+
+ def test_decimal_equality_returns_matching_row(self):
+ # DECIMAL(10,2) reads back as Decimal; the old float literal dropped
the row.
+ self.assertEqual(self._scan_ids("price = 99.99"), [1])
+
+ def test_decimal_literal_rescaled_to_column_scale(self):
+ # The pushed-down filter binds the literal's own scale, so an integer
or
+ # trailing-zero literal must be rescaled to the column scale to match.
+ self.assertEqual(self._scan_ids("price = 50"), [2])
+ self.assertEqual(self._scan_ids("price = 99.990"), [1])
+
+ def test_decimal_literal_finer_than_column_matches_nothing(self):
+ # A literal too precise for the column (99.999 on DECIMAL(10,2))
cannot equal
+ # any stored value; it must return no rows rather than round into a
match.
+ self.assertEqual(self._scan_ids("price = 99.999"), [])
+
+ def test_high_precision_decimal_integer_literal_matches(self):
+ # A DECIMAL(38, 2) integer-form literal exceeds the default decimal
+ # context; it must still rescale and match the stored scale-2 value.
+ self.assertEqual(
+ self._scan_ids("big = 123456789012345678901234567890123456"), [1])
+
+ def test_malformed_decimal_raises_value_error(self):
+ # parse_where_clause documents ValueError; a bad decimal must not leak
+ # decimal.InvalidOperation past the CLI's `except ValueError`.
+ with self.assertRaises(ValueError):
+ parse_where_clause("price = abc", self.table.table_schema.fields)
+
+ def test_date_equality_returns_matching_row(self):
+ # DATE literal left as str raised in the arrow comparison kernel.
+ self.assertEqual(self._scan_ids("d = '2024-01-01'"), [1])
+
+ def test_time_equality_returns_matching_row(self):
+ self.assertEqual(self._scan_ids("t = '12:30:00'"), [1])
+
+
if __name__ == '__main__':
unittest.main()
