This is an automated email from the ASF dual-hosted git repository.
sxjscience pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new c9ec311 [MXNET-86] Revert to pre-profile-changes copy code (#10090)
c9ec311 is described below
commit c9ec3118688c233a66ad847003a9e8d2d09e5952
Author: Chris Olivier <cjolivie...@gmail.com>
AuthorDate: Tue Mar 13 21:36:16 2018 -0700
[MXNET-86] Revert to pre-profile-changes copy code (#10090)
* Revert to pre-profile-changes copy code
* Add test
* Trigger rebuild
---
src/ndarray/ndarray_function.cc | 19 ++------
tests/python/unittest/test_gluon_data.py | 81 ++++++++++++++++++++++++++++++++
2 files changed, 85 insertions(+), 15 deletions(-)
diff --git a/src/ndarray/ndarray_function.cc b/src/ndarray/ndarray_function.cc
index 927c906..552555a 100644
--- a/src/ndarray/ndarray_function.cc
+++ b/src/ndarray/ndarray_function.cc
@@ -26,7 +26,6 @@
#include "./ndarray_function.h"
#include "./ndarray_function-inl.h"
#include "../common/utils.h"
-#include "../operator/mxnet_op.h"
namespace mxnet {
namespace ndarray {
@@ -34,27 +33,17 @@ template<>
void Copy<cpu, cpu>(const TBlob &from, TBlob *to,
Context from_ctx, Context to_ctx,
RunContext ctx) {
- using namespace mxnet::op;
MSHADOW_TYPE_SWITCH(to->type_flag_, DType, {
if (to->type_flag_ == from.type_flag_) {
- TBlob dest = to->FlatTo1D<cpu, DType>();
- TBlob src = from.FlatTo1D<cpu, DType>();
- const size_t size = src.Size();
- if (dest.CheckContiguous() && src.CheckContiguous() && size >= 20000 /*
non-trivial size */) {
- CHECK_EQ(dest.shape_, src.shape_)
- << "Copy:shape mismatch:" << dest.shape_ << " vs " << src.shape_;
- mxnet_op::Kernel<mxnet_op::op_with_req<mshadow_op::identity,
kWriteTo>, cpu>::Launch(
- ctx.get_stream<cpu>(), src.Size(), dest.dptr<DType>(),
src.dptr<DType>());
- } else {
- mshadow::Copy(to->FlatTo1D<cpu, DType>(), from.FlatTo1D<cpu, DType>());
- }
+ mshadow::Copy(to->FlatTo1D<cpu, DType>(),
+ from.FlatTo1D<cpu, DType>());
} else {
MSHADOW_TYPE_SWITCH(from.type_flag_, SrcDType, {
to->FlatTo1D<cpu, DType>() =
- mshadow::expr::tcast<DType>(from.FlatTo1D<cpu, SrcDType>());
+ mshadow::expr::tcast<DType>(from.FlatTo1D<cpu, SrcDType>());
})
}
- });
+ })
}
template<typename DType, typename IType>
diff --git a/tests/python/unittest/test_gluon_data.py
b/tests/python/unittest/test_gluon_data.py
index 49b1b8e..93160aa 100644
--- a/tests/python/unittest/test_gluon_data.py
+++ b/tests/python/unittest/test_gluon_data.py
@@ -20,8 +20,14 @@ import tarfile
import unittest
import mxnet as mx
import numpy as np
+import random
from mxnet import gluon
+import platform
from common import setup_module, with_seed
+from mxnet.gluon.data import DataLoader
+import mxnet.ndarray as nd
+from mxnet import context
+from mxnet.gluon.data.dataset import Dataset
@with_seed()
def test_array_dataset():
@@ -112,6 +118,81 @@ def test_multi_worker():
for i, batch in enumerate(loader):
assert (batch.asnumpy() == i).all()
+@with_seed()
+def test_multi_worker_forked_data_loader():
+ """
+ Test should successfully run its course of multi-process/forked data
loader without errors
+ """
+ class Dummy(Dataset):
+ def __init__(self, random_shape):
+ self.random_shape = random_shape
+
+ def __getitem__(self, idx):
+ key = idx
+ if self.random_shape:
+ out = np.random.uniform(size=(random.randint(1000, 1100), 40))
+ labels = np.random.uniform(size=(random.randint(10, 15)))
+ else:
+ out = np.random.uniform(size=(1000, 40))
+ labels = np.random.uniform(size=(10))
+ return key, out, labels
+
+ def __len__(self):
+ return 50
+
+ def batchify(self, data):
+ """
+ Collate data into batch. Use shared memory for stacking.
+
+ :param data: a list of array, with layout of 'NTC'.
+ :return either x and x's unpadded lengths, or x, x's unpadded
lengths, y and y's unpadded lengths
+ if labels are not supplied.
+ """
+
+ # input layout is NTC
+ keys, inputs, labels = [item[0] for item in data], [item[1] for
item in data], \
+ [item[2] for item in data]
+
+ if len(data) > 1:
+ max_data_len = max([seq.shape[0] for seq in inputs])
+ max_labels_len = 0 if not labels else max([seq.shape[0] for
seq in labels])
+ else:
+ max_data_len = inputs[0].shape[0]
+ max_labels_len = 0 if not labels else labels[0].shape[0]
+
+ x_lens = [item.shape[0] for item in inputs]
+ y_lens = [item.shape[0] for item in labels]
+
+ for i, seq in enumerate(inputs):
+ pad_len = max_data_len - seq.shape[0]
+ inputs[i] = np.pad(seq, ((0, pad_len), (0, 0)), 'constant',
constant_values=0)
+ labels[i] = np.pad(labels[i], (0, max_labels_len -
labels[i].shape[0]),
+ 'constant', constant_values=-1)
+
+ inputs = np.asarray(inputs, dtype=np.float32)
+ if labels is not None:
+ labels = np.asarray(labels, dtype=np.float32)
+ inputs = inputs.transpose((1, 0, 2))
+ labels = labels.transpose((1, 0))
+
+ return (nd.array(inputs, dtype=inputs.dtype,
ctx=context.Context('cpu_shared', 0)),
+ nd.array(x_lens, ctx=context.Context('cpu_shared', 0))) \
+ if labels is None else (
+ nd.array(inputs, dtype=inputs.dtype,
ctx=context.Context('cpu_shared', 0)),
+ nd.array(x_lens, ctx=context.Context('cpu_shared', 0)),
+ nd.array(labels, dtype=labels.dtype,
ctx=context.Context('cpu_shared', 0)),
+ nd.array(y_lens, ctx=context.Context('cpu_shared', 0)))
+
+
+ # This test is pointless on Windows because Windows doesn't fork
+ if platform.system() != 'Windows':
+ data = Dummy(True)
+ loader = DataLoader(data, batch_size=40, batchify_fn=data.batchify,
num_workers=2)
+ for epoch in range(1):
+ for i, data in enumerate(loader):
+ if i % 100 == 0:
+ print(data)
+ print('{}:{}'.format(epoch, i))
if __name__ == '__main__':
import nose
--
To stop receiving notification emails like this one, please contact
sxjscie...@apache.org.
