Fokko opened a new pull request, #6487:
URL: https://github.com/apache/iceberg/pull/6487
This breaks partitioning:
```
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[38], line 3
1 from pyiceberg.expressions import And, GreaterThanOrEqual, LessThan
----> 3 df = tbl.scan(row_filter=And(
4 GreaterThanOrEqual("tpep_pickup_datetime",
"2021-04-01T00:00:00.000000+00:00"),
5 LessThan("tpep_pickup_datetime",
"2021-05-01T00:00:00.000000+00:00")
6 )).to_arrow().to_pandas()
8 df
File /usr/local/lib/python3.9/site-packages/pyiceberg/table/__init__.py:350,
in DataScan.to_arrow(self)
347 fs = self.table.io.get_fs(scheme)
349 locations = []
--> 350 for task in self.plan_files():
351 if isinstance(task, FileScanTask):
352 _, path = PyArrowFileIO.parse_location(task.file.file_path)
File /usr/local/lib/python3.9/site-packages/pyiceberg/table/__init__.py:335,
in DataScan.plan_files(self)
332 all_files = files(io.new_input(manifest.manifest_path))
333 matching_partition_files = filter(partition_filter, all_files)
--> 335 yield from (FileScanTask(file) for file in matching_partition_files)
File /usr/local/lib/python3.9/site-packages/pyiceberg/table/__init__.py:335,
in <genexpr>(.0)
332 all_files = files(io.new_input(manifest.manifest_path))
333 matching_partition_files = filter(partition_filter, all_files)
--> 335 yield from (FileScanTask(file) for file in matching_partition_files)
File /usr/local/lib/python3.9/site-packages/pyiceberg/table/__init__.py:305,
in DataScan._build_partition_evaluator.<locals>.<lambda>(data_file)
302 wrapper = _DictAsStruct(partition_type)
303 evaluator = visitors.expression_evaluator(partition_schema,
partition_expr, self.case_sensitive)
--> 305 return lambda data_file: evaluator(wrapper.wrap(data_file.partition))
File
/usr/local/lib/python3.9/site-packages/pyiceberg/expressions/visitors.py:437,
in _ExpressionEvaluator.eval(self, struct)
435 def eval(self, struct: StructProtocol) -> bool:
436 self.struct = struct
--> 437 return visit(self.bound, self)
File /usr/local/lib/python3.9/functools.py:888, in
singledispatch.<locals>.wrapper(*args, **kw)
884 if not args:
885 raise TypeError(f'{funcname} requires at least '
886 '1 positional argument')
--> 888 return dispatch(args[0].__class__)(*args, **kw)
File
/usr/local/lib/python3.9/site-packages/pyiceberg/expressions/visitors.py:164,
in _(obj, visitor)
161 @visit.register(And)
162 def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T:
163 """Visit an And boolean expression with a concrete
BooleanExpressionVisitor"""
--> 164 left_result: T = visit(obj.left, visitor=visitor)
165 right_result: T = visit(obj.right, visitor=visitor)
166 return visitor.visit_and(left_result=left_result,
right_result=right_result)
File /usr/local/lib/python3.9/functools.py:888, in
singledispatch.<locals>.wrapper(*args, **kw)
884 if not args:
885 raise TypeError(f'{funcname} requires at least '
886 '1 positional argument')
--> 888 return dispatch(args[0].__class__)(*args, **kw)
File
/usr/local/lib/python3.9/site-packages/pyiceberg/expressions/visitors.py:178,
in _(obj, visitor)
175 @visit.register(BoundPredicate)
176 def _(obj: BoundPredicate[L], visitor: BooleanExpressionVisitor[T])
-> T:
177 """Visit a bound boolean expression with a concrete
BooleanExpressionVisitor"""
--> 178 return visitor.visit_bound_predicate(predicate=obj)
File
/usr/local/lib/python3.9/site-packages/pyiceberg/expressions/visitors.py:326,
in BoundBooleanExpressionVisitor.visit_bound_predicate(self, predicate)
321 def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> T:
322 """Visit a bound predicate
323 Args:
324 predicate (BoundPredicate[L]): A bound predicate
325 """
--> 326 return visit_bound_predicate(predicate, self)
File /usr/local/lib/python3.9/functools.py:888, in
singledispatch.<locals>.wrapper(*args, **kw)
884 if not args:
885 raise TypeError(f'{funcname} requires at least '
886 '1 positional argument')
--> 888 return dispatch(args[0].__class__)(*args, **kw)
File
/usr/local/lib/python3.9/site-packages/pyiceberg/expressions/visitors.py:377,
in _(expr, visitor)
374 @visit_bound_predicate.register(BoundGreaterThanOrEqual)
375 def _(expr: BoundGreaterThanOrEqual[L], visitor:
BoundBooleanExpressionVisitor[T]) -> T:
376 """Visit a bound GreaterThanOrEqual predicate"""
--> 377 return visitor.visit_greater_than_or_equal(term=expr.term,
literal=expr.literal)
File
/usr/local/lib/python3.9/site-packages/pyiceberg/expressions/visitors.py:466,
in _ExpressionEvaluator.visit_greater_than_or_equal(self, term, literal)
465 def visit_greater_than_or_equal(self, term: BoundTerm[L], literal:
Literal[L]) -> bool:
--> 466 return term.eval(self.struct) >= literal.value
TypeError: '>=' not supported between instances of 'datetime.date' and 'int'
```
After the change this works:
```
➜ python git:(fd-fix-partitioning) python3
Python 3.10.9 (main, Dec 15 2022, 17:11:09) [Clang 14.0.0
(clang-1400.0.29.202)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> from pyiceberg.catalog import load_catalog
>>>
>>> catalog = load_catalog('local')
>>>
>>> tbl = catalog.load_table('pyiceberg.taxis')
>>>
>>> from pyiceberg.expressions import GreaterThanOrEqual, LessThan, And
>>>
>>>
>>> tbl.scan(row_filter=And(
... GreaterThanOrEqual("tpep_pickup_datetime",
"2001-04-01T00:00:00.000000+00:00"),
... LessThan("tpep_pickup_datetime", "2021-05-01T00:00:00.000000+00:00")
... )).to_arrow()
/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/table/__init__.py:340:
UserWarning: Projection is currently done by name instead of Field ID, this can
lead to incorrect results in some cases.
warnings.warn(
pyarrow.Table
VendorID: int64
tpep_pickup_datetime: timestamp[us, tz=+00:00]
tpep_dropoff_datetime: timestamp[us, tz=+00:00]
passenger_count: double
trip_distance: double
RatecodeID: double
store_and_fwd_flag: string
PULocationID: int64
DOLocationID: int64
payment_type: int64
fare_amount: double
extra: double
mta_tax: double
tip_amount: double
tolls_amount: double
improvement_surcharge: double
total_amount: double
congestion_surcharge: double
airport_fee: double
----
VendorID: [[2],[2,2,2,2,2,...,2,2,2,2,2],...,[2,2],[2,2,2,2,2]]
tpep_pickup_datetime: [[2003-01-23 02:25:38.000000],[2008-12-31
23:58:56.000000,2008-12-31 23:20:01.000000,2008-12-31
23:07:32.000000,2008-12-31 23:03:41.000000,2008-12-31
23:08:12.000000,...,2008-12-31
23:04:00.000000,2008-12-31 23:10:33.000000,2008-12-31
23:25:16.000000,2008-12-31 23:02:25.000000,2008-12-31
23:06:49.000000],...,[2008-12-31 23:02:01.000000,2008-12-31
23:02:04.000000],[2009-01-01
12:03:53.000000,2009-01-01 02:36:08.000000,2009-01-01
00:02:23.000000,2009-01-01 15:55:59.000000,2009-01-01 06:35:59.000000]]
tpep_dropoff_datetime: [[2003-01-23 02:28:43.000000],[2009-01-01
00:42:43.000000,2008-12-31 23:20:11.000000,2009-01-01
18:44:18.000000,2008-12-31 23:28:09.000000,2009-01-01
15:28:59.000000,...,2009-01-01
14:26:52.000000,2009-01-01 13:23:06.000000,2008-12-31
23:32:31.000000,2009-01-01 12:40:19.000000,2009-01-01
14:08:51.000000],...,[2009-01-01 00:13:52.000000,2009-01-01
17:04:21.000000],[2009-01-01
22:20:48.000000,2009-01-01 18:02:26.000000,2009-01-01
20:55:27.000000,2009-01-01 16:01:09.000000,2009-01-01 20:57:27.000000]]
passenger_count: [[1],[2,1,1,1,1,...,1,1,3,1,1],...,[1,1],[1,1,1,1,1]]
trip_distance:
[[0],[20.33,0.07,0,2.77,1.08,...,0.6,1.26,1.3,0.41,6.97],...,[23.41,1.13],[0.63,0.33,1.16,1.43,1.54]]
RatecodeID: [[1],[2,1,1,1,1,...,1,1,1,1,1],...,[1,1],[1,1,1,1,1]]
store_and_fwd_flag:
[["N"],["N","N","N","N","N",...,"N","N","N","N","N"],...,["N","N"],["N","N","N","N","N"]]
PULocationID:
[[264],[132,170,193,233,75,...,90,137,237,162,116],...,[132,90],[142,170,246,142,87]]
DOLocationID:
[[264],[116,170,193,4,151,...,90,79,170,237,170],...,[227,170],[142,170,50,238,79]]
payment_type: [[2],[2,2,1,1,2,...,1,1,2,1,2],...,[2,1],[1,1,1,2,1]]
... ```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
