smaheshwar-pltr commented on code in PR #3364:
URL: https://github.com/apache/iceberg-python/pull/3364#discussion_r3432208622
##########
pyiceberg/table/__init__.py:
##########
@@ -2104,115 +2180,357 @@ def plan_files(self) -> Iterable[FileScanTask]:
return self._plan_files_local()
def to_arrow(self) -> pa.Table:
- """Read an Arrow table eagerly from this DataScan.
+ return _to_arrow_via_file_scan_tasks(self, self.plan_files())
- All rows will be loaded into memory at once.
+ def to_arrow_batch_reader(self) -> pa.RecordBatchReader:
+ return _to_arrow_batch_reader_via_file_scan_tasks(self,
self.plan_files())
- Returns:
- pa.Table: Materialized Arrow Table from the Iceberg table's
DataScan
- """
+ def count(self) -> int:
from pyiceberg.io.pyarrow import ArrowScan
- return ArrowScan(
- self.table_metadata, self.io, self.projection(), self.row_filter,
self.case_sensitive, self.limit
- ).to_table(self.plan_files())
+ # Usage: Calculates the total number of records in a Scan that haven't
had positional deletes.
+ res = 0
+ # every task is a FileScanTask
+ tasks = self.plan_files()
- def to_arrow_batch_reader(self) -> pa.RecordBatchReader:
- """Return an Arrow RecordBatchReader from this DataScan.
Review Comment:
Thank you for catching this, apologies! Addressed in
https://github.com/apache/iceberg-python/pull/3511
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
