kohillyang commented on a change in pull request #13606: Complimentary gluon
DataLoader improvements
URL: https://github.com/apache/incubator-mxnet/pull/13606#discussion_r240520527
##########
File path: python/mxnet/gluon/data/dataloader.py
##########
@@ -551,14 +568,17 @@ def same_process_iter():
# multi-worker
return _MultiWorkerIter(self._worker_pool, self._batchify_fn,
self._batch_sampler,
- pin_memory=self._pin_memory,
worker_fn=_worker_fn,
- prefetch=self._prefetch)
+ pin_memory=self._pin_memory,
+ worker_fn=_thread_worker_fn if
self._thread_pool else _worker_fn,
+ prefetch=self._prefetch,
+ dataset=self._dataset if self._thread_pool
else None)
def __len__(self):
return len(self._batch_sampler)
def __del__(self):
if self._worker_pool:
- # manually terminate due to a bug that pool is not automatically
terminated on linux
+ # manually terminate due to a bug that pool is not automatically
terminated
+ # https://bugs.python.org/issue34172
assert isinstance(self._worker_pool, multiprocessing.pool.Pool)
Review comment:
What happens here if threadpool is True?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
