lidavidm commented on a change in pull request #10118:
URL: https://github.com/apache/arrow/pull/10118#discussion_r634310151
##########
File path: python/pyarrow/tests/test_dataset.py
##########
@@ -206,6 +206,63 @@ def dataset(mockfs):
return factory.finish()
[email protected](params=[
+ (True, True),
+ (True, False),
+ (False, True),
+ (False, False)
+], ids=['threaded-async', 'threaded-sync', 'serial-async', 'serial-sync'])
+def dataset_reader(request):
+ '''
+ Fixture which allows dataset scanning operations to be
+ run with/without threads and with/without async
+ '''
+ use_threads, use_async = request.param
+
+ class reader:
+
+ def __init__(self):
+ self.use_threads = use_threads
+ self.use_async = use_async
+
+ def _patch_kwargs(self, kwargs):
+ if 'use_threads' in kwargs:
+ raise Exception(
+ ('Invalid use of dataset_reader, do not specify'
+ ' use_threads'))
+ if 'use_async' in kwargs:
+ raise Exception(
+ 'Invalid use of dataset_reader, do not specify use_async')
+ kwargs['use_threads'] = use_threads
+ kwargs['use_async'] = use_async
+
+ def to_table(self, dataset, **kwargs):
+ self._patch_kwargs(kwargs)
+ return dataset.to_table(**kwargs)
+
+ def to_batches(self, dataset, **kwargs):
+ self._patch_kwargs(kwargs)
+ return dataset.to_batches(**kwargs)
+
+ def scanner(self, dataset, **kwargs):
+ self._patch_kwargs(kwargs)
+ return dataset.scanner(**kwargs)
+
+ def head(self, dataset, num_rows, **kwargs):
Review comment:
I think the way it is is fine too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
