This is an automated email from the ASF dual-hosted git repository.
zhreshold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new f6b4665 Fix #13521 (#13537)
f6b4665 is described below
commit f6b4665995f8f8ff32862a029b2074475d8467eb
Author: Joshua Z. Zhang <[email protected]>
AuthorDate: Wed Dec 5 13:39:17 2018 -0800
Fix #13521 (#13537)
* fix pool release
* fix
---
python/mxnet/gluon/data/dataloader.py | 6 ++++++
tests/python/unittest/test_gluon_data.py | 11 +++++++++++
2 files changed, 17 insertions(+)
diff --git a/python/mxnet/gluon/data/dataloader.py
b/python/mxnet/gluon/data/dataloader.py
index ad0f534..586e620 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -556,3 +556,9 @@ class DataLoader(object):
def __len__(self):
return len(self._batch_sampler)
+
+ def __del__(self):
+ if self._worker_pool:
+ # manually terminate due to a bug that pool is not automatically
terminated on linux
+ assert isinstance(self._worker_pool, multiprocessing.pool.Pool)
+ self._worker_pool.terminate()
diff --git a/tests/python/unittest/test_gluon_data.py
b/tests/python/unittest/test_gluon_data.py
index e420609..a3ba222 100644
--- a/tests/python/unittest/test_gluon_data.py
+++ b/tests/python/unittest/test_gluon_data.py
@@ -244,6 +244,17 @@ def test_multi_worker_forked_data_loader():
for i, data in enumerate(loader):
pass
+@with_seed()
+def test_multi_worker_dataloader_release_pool():
+ # will trigger too many open file if pool is not released properly
+ for _ in range(100):
+ A = np.random.rand(999, 2000)
+ D = mx.gluon.data.DataLoader(A, batch_size=8, num_workers=8)
+ the_iter = iter(D)
+ next(the_iter)
+ del the_iter
+ del D
+
if __name__ == '__main__':
import nose
nose.runmodule()
