This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 46669fad0 fix(conversions): reject a Decimal whose scale is negative
(#3997)
46669fad0 is described below
commit 46669fad031d5b4e7a71c1df13940d17bbc9e71d
Author: Rodrigo Stachlewski Palma
<[email protected]>
AuthorDate: Sat Oct 24 06:45:20 2026 +0000
fix(conversions): reject a Decimal whose scale is negative (#3997)
* fix(conversions): reject a Decimal whose scale is negative
to_bytes compared abs(exponent) against the type scale, so a Decimal
carrying
a negative scale passed as if it had the matching positive one. The
exponent is
then dropped by decimal_to_unscaled, and the value is written four orders of
magnitude off without an error: Decimal('1E+2') stored in a decimal(10, 2)
reads back as 0.01.
Decimal('100').normalize() produces exactly that form, and to_bytes writes
the
lower and upper bounds of a data file, so a wrong bound prunes files that do
hold matching rows.
Compare the signed scale instead.
* fix(expressions): reject decimal literals with a negative scale
DecimalLiteral.to and StringLiteral.to compared abs(exponent) against the
type's scale, so a filter value like Decimal("1E+2") bound to decimal(10, 2)
as if it were 100.00 while keeping unscaled value 1. Bucket and truncate
projections then pruned the files holding matching rows and the scan
silently
returned fewer rows. Compare the signed scale so binding raises instead.
Co-authored-by: Copilot App <[email protected]>
---------
Co-authored-by: Kevin Liu <[email protected]>
Co-authored-by: Kevin Liu <[email protected]>
Co-authored-by: Copilot App <[email protected]>
---
pyiceberg/conversions.py | 8 +++++---
pyiceberg/expressions/literals.py | 4 ++--
tests/expressions/test_literals.py | 6 ++++++
tests/test_conversions.py | 7 +++++++
4 files changed, 20 insertions(+), 5 deletions(-)
diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py
index 6dd964f43..fde188544 100644
--- a/pyiceberg/conversions.py
+++ b/pyiceberg/conversions.py
@@ -310,9 +310,11 @@ def _(primitive_type: DecimalType, value: Decimal) ->
bytes:
bytes: The byte representation of `value`.
"""
_, digits, exponent = value.as_tuple()
- exponent = abs(int(exponent))
- if exponent != primitive_type.scale:
- raise ValueError(f"Cannot serialize value, scale of value does not
match type {primitive_type}: {exponent}")
+ # A Decimal carries the negated scale as its exponent, so a value with a
negative
+ # scale (1E+2) must not be read as if it had the matching positive one
+ value_scale = -int(exponent)
+ if value_scale != primitive_type.scale:
+ raise ValueError(f"Cannot serialize value, scale of value does not
match type {primitive_type}: {value_scale}")
elif len(digits) > primitive_type.precision:
raise ValueError(
f"Cannot serialize value, precision of value is greater than
precision of type {primitive_type}: {len(digits)}"
diff --git a/pyiceberg/expressions/literals.py
b/pyiceberg/expressions/literals.py
index 61581d9b3..6f9b79e77 100644
--- a/pyiceberg/expressions/literals.py
+++ b/pyiceberg/expressions/literals.py
@@ -522,7 +522,7 @@ class DecimalLiteral(Literal[Decimal]):
@to.register(DecimalType)
def _(self, type_var: DecimalType) -> Literal[Decimal]:
- if type_var.scale == abs(int(self.value.as_tuple().exponent)):
+ if type_var.scale == -int(self.value.as_tuple().exponent):
return self
raise ValueError(f"Could not convert {self.value} into a {type_var}")
@@ -628,7 +628,7 @@ class StringLiteral(Literal[str]):
@to.register(DecimalType)
def _(self, type_var: DecimalType) -> Literal[Decimal]:
dec = Decimal(self.value)
- scale = abs(int(dec.as_tuple().exponent))
+ scale = -int(dec.as_tuple().exponent)
if type_var.scale == scale:
return DecimalLiteral(dec)
else:
diff --git a/tests/expressions/test_literals.py
b/tests/expressions/test_literals.py
index 9251e79a7..9b73a5022 100644
--- a/tests/expressions/test_literals.py
+++ b/tests/expressions/test_literals.py
@@ -291,6 +291,9 @@ def test_decimal_to_decimal_conversion() -> None:
with pytest.raises(ValueError) as e:
_ = lit.to(DecimalType(9, 3))
assert "Could not convert 34.11 into a decimal(9, 3)" in str(e.value)
+ with pytest.raises(ValueError) as e:
+ _ = literal(Decimal("1E+2")).to(DecimalType(9, 2))
+ assert "Could not convert 1E+2 into a decimal(9, 2)" in str(e.value)
def test_timestamp_to_date() -> None:
@@ -947,6 +950,9 @@ def test_string_to_decimal_type_invalid_value() -> None:
with pytest.raises(ValueError) as e:
_ = literal("18.15").to(DecimalType(10, 0))
assert "Could not convert 18.15 into a decimal(10, 0), scales differ 0 <>
2" in str(e.value)
+ with pytest.raises(ValueError) as e:
+ _ = literal("1E+2").to(DecimalType(10, 2))
+ assert "Could not convert 1E+2 into a decimal(10, 2), scales differ 2 <>
-2" in str(e.value)
def test_decimal_literal_increment() -> None:
diff --git a/tests/test_conversions.py b/tests/test_conversions.py
index e786ae068..322e58a9b 100644
--- a/tests/test_conversions.py
+++ b/tests/test_conversions.py
@@ -511,6 +511,13 @@ def test_raise_on_unregistered_type() -> None:
"primitive_type, value, expected_error_message",
[
(DecimalType(7, 3), Decimal("123.4567"), "Cannot serialize value,
scale of value does not match type decimal(7, 3): 4"),
+ # A Decimal with a negative scale must not pass as its positive
counterpart
+ (DecimalType(10, 2), Decimal("1E+2"), "Cannot serialize value, scale
of value does not match type decimal(10, 2): -2"),
+ (
+ DecimalType(10, 2),
+ Decimal("100").normalize(),
+ "Cannot serialize value, scale of value does not match type
decimal(10, 2): -2",
+ ),
(
DecimalType(18, 8),
Decimal("123456789.123456789"),
