robertwb commented on a change in pull request #13443:
URL: https://github.com/apache/beam/pull/13443#discussion_r534382361
##########
File path: sdks/python/apache_beam/dataframe/io.py
##########
@@ -181,39 +193,153 @@ def expand(self, root):
self.args,
self.kwargs,
self.incremental,
+ self.splittable,
self.binary)))
from apache_beam.dataframe import convert
return convert.to_dataframe(
pcoll, proxy=_prefix_range_index_with(':', sample[:0]))
-# TODO(robertwb): Actually make an SDF.
-class _ReadFromPandasDoFn(beam.DoFn):
- def __init__(self, reader, args, kwargs, incremental, binary):
+class _TruncatingFileHandle(object):
+ """A wrapper of a file-like object representing the restriction of the
+ underling handle according to the given SDF restriction tracker, breaking
+ the file only after the given delimiter.
+
+ For example, if the underling restriction is [103, 607) and each line were
+ exactly 10 characters long (i.e. every 10th charcter was a newline), than
this
Review comment:
Done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
