rahulsmahadev commented on code in PR #4407:
URL: https://github.com/apache/iceberg-python/pull/4407#discussion_r4603371161
##########
pyiceberg/transforms.py:
##########
@@ -1030,6 +1033,57 @@ def pyarrow_transform(self, source: IcebergType) ->
"Callable[[pa.Array], pa.Arr
return lambda arr: pa.nulls(len(arr), type=arr.type)
+def _integer_bounds(source: IcebergType) -> tuple[int, int]:
+ """Return the minimum value and the size of the range of an integer
type."""
+ if isinstance(source, IntegerType):
+ return -(2**31), 2**32
+ return -(2**63), 2**64
+
+
+def _wrap_integer(value: int, source: IcebergType) -> int:
+ """Wrap an integer into the range of the source type, like Java's two's
complement arithmetic."""
+ minimum, modulus = _integer_bounds(source)
+ return (value - minimum) % modulus + minimum
+
+
+def _truncate_integer_projection(
+ name: str,
+ pred: BoundLiteralPredicate,
+ transform: Callable[[Any | None], Any | None],
+ source: IcebergType,
+ width: int,
+) -> UnboundPredicate | None:
+ """Project a predicate on a truncate partition of an integer column.
+
+ The values from the minimum of the source type truncate to a partition
value at the top of the
+ type's range, which does not sort with the other partition values. A range
that can match those
+ values therefore also matches that partition.
+ """
+ minimum, _ = _integer_bounds(source)
+ floor_mod = minimum % width
+ if floor_mod == 0 or not isinstance(pred.literal, LongLiteral):
+ return _truncate_number(name, pred, transform)
+
+ wrapped_minimum = _wrap_integer(minimum - floor_mod, source)
+ # The first partition value after the block that holds the minimum of the
source type
+ next_block_start = minimum - floor_mod + width
+ boundary = pred.literal.value
+
+ if isinstance(pred, BoundGreaterThan):
+ if boundary + 1 < next_block_start:
+ return Or(GreaterThanOrEqual(Reference(name), next_block_start),
EqualTo(Reference(name), wrapped_minimum))
+ return _truncate_number(name, pred, transform)
+ elif isinstance(pred, BoundGreaterThanOrEqual):
Review Comment:
No test covers this branch: with `if boundary < next_block_start:` replaced
by `if False:`, every test still passes, although `x >= MIN` then skips all
ordinary partitions. An assert next to the `GreaterThan` one in
`test_int_truncate_projection_near_minimum` would cover it:
```python
assert project(GreaterThanOrEqual("id", -(2**63) + 1)) == Or(
GreaterThanOrEqual("id_trunc", -9223372036854775800),
EqualTo("id_trunc", wrapped_minimum)
)
```
That test projects the `LongType` `id` column, so an `IntegerType` case
would help too; only the scan test covers int.
##########
pyiceberg/expressions/visitors.py:
##########
@@ -1985,7 +1985,7 @@ def visit_bound_predicate(self, predicate:
BoundPredicate) -> BooleanExpression:
inclusive_projection = part.transform.project(part.name, predicate)
inclusive_result = None
- if inclusive_projection is not None:
+ if isinstance(inclusive_projection, UnboundPredicate):
Review Comment:
`TruncateTransform.project()` can now return an `Or`, but
`Transform.project()`, `TruncateTransform.project()` and
`_truncate_integer_projection()` are still annotated `UnboundPredicate | None`,
and this guard shows that callers relied on that. Widening the annotations to
`BooleanExpression | None` would let mypy point at callers like this one.
#4278 rewrites this block into `_eval_projection(projection:
UnboundPredicate | None)`, which calls `.bind()` on the projection, so
whichever of the two PRs merges second needs this guard there too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]