This is an automated email from the ASF dual-hosted git repository. sxjscience pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new c9ec311 [MXNET-86] Revert to pre-profile-changes copy code (#10090) c9ec311 is described below commit c9ec3118688c233a66ad847003a9e8d2d09e5952 Author: Chris Olivier <cjolivie...@gmail.com> AuthorDate: Tue Mar 13 21:36:16 2018 -0700 [MXNET-86] Revert to pre-profile-changes copy code (#10090) * Revert to pre-profile-changes copy code * Add test * Trigger rebuild --- src/ndarray/ndarray_function.cc | 19 ++------ tests/python/unittest/test_gluon_data.py | 81 ++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/src/ndarray/ndarray_function.cc b/src/ndarray/ndarray_function.cc index 927c906..552555a 100644 --- a/src/ndarray/ndarray_function.cc +++ b/src/ndarray/ndarray_function.cc @@ -26,7 +26,6 @@ #include "./ndarray_function.h" #include "./ndarray_function-inl.h" #include "../common/utils.h" -#include "../operator/mxnet_op.h" namespace mxnet { namespace ndarray { @@ -34,27 +33,17 @@ template<> void Copy<cpu, cpu>(const TBlob &from, TBlob *to, Context from_ctx, Context to_ctx, RunContext ctx) { - using namespace mxnet::op; MSHADOW_TYPE_SWITCH(to->type_flag_, DType, { if (to->type_flag_ == from.type_flag_) { - TBlob dest = to->FlatTo1D<cpu, DType>(); - TBlob src = from.FlatTo1D<cpu, DType>(); - const size_t size = src.Size(); - if (dest.CheckContiguous() && src.CheckContiguous() && size >= 20000 /* non-trivial size */) { - CHECK_EQ(dest.shape_, src.shape_) - << "Copy:shape mismatch:" << dest.shape_ << " vs " << src.shape_; - mxnet_op::Kernel<mxnet_op::op_with_req<mshadow_op::identity, kWriteTo>, cpu>::Launch( - ctx.get_stream<cpu>(), src.Size(), dest.dptr<DType>(), src.dptr<DType>()); - } else { - mshadow::Copy(to->FlatTo1D<cpu, DType>(), from.FlatTo1D<cpu, DType>()); - } + mshadow::Copy(to->FlatTo1D<cpu, DType>(), + from.FlatTo1D<cpu, DType>()); } else { MSHADOW_TYPE_SWITCH(from.type_flag_, SrcDType, { to->FlatTo1D<cpu, DType>() = - mshadow::expr::tcast<DType>(from.FlatTo1D<cpu, SrcDType>()); + mshadow::expr::tcast<DType>(from.FlatTo1D<cpu, SrcDType>()); }) } - }); + }) } template<typename DType, typename IType> diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py index 49b1b8e..93160aa 100644 --- a/tests/python/unittest/test_gluon_data.py +++ b/tests/python/unittest/test_gluon_data.py @@ -20,8 +20,14 @@ import tarfile import unittest import mxnet as mx import numpy as np +import random from mxnet import gluon +import platform from common import setup_module, with_seed +from mxnet.gluon.data import DataLoader +import mxnet.ndarray as nd +from mxnet import context +from mxnet.gluon.data.dataset import Dataset @with_seed() def test_array_dataset(): @@ -112,6 +118,81 @@ def test_multi_worker(): for i, batch in enumerate(loader): assert (batch.asnumpy() == i).all() +@with_seed() +def test_multi_worker_forked_data_loader(): + """ + Test should successfully run its course of multi-process/forked data loader without errors + """ + class Dummy(Dataset): + def __init__(self, random_shape): + self.random_shape = random_shape + + def __getitem__(self, idx): + key = idx + if self.random_shape: + out = np.random.uniform(size=(random.randint(1000, 1100), 40)) + labels = np.random.uniform(size=(random.randint(10, 15))) + else: + out = np.random.uniform(size=(1000, 40)) + labels = np.random.uniform(size=(10)) + return key, out, labels + + def __len__(self): + return 50 + + def batchify(self, data): + """ + Collate data into batch. Use shared memory for stacking. + + :param data: a list of array, with layout of 'NTC'. + :return either x and x's unpadded lengths, or x, x's unpadded lengths, y and y's unpadded lengths + if labels are not supplied. + """ + + # input layout is NTC + keys, inputs, labels = [item[0] for item in data], [item[1] for item in data], \ + [item[2] for item in data] + + if len(data) > 1: + max_data_len = max([seq.shape[0] for seq in inputs]) + max_labels_len = 0 if not labels else max([seq.shape[0] for seq in labels]) + else: + max_data_len = inputs[0].shape[0] + max_labels_len = 0 if not labels else labels[0].shape[0] + + x_lens = [item.shape[0] for item in inputs] + y_lens = [item.shape[0] for item in labels] + + for i, seq in enumerate(inputs): + pad_len = max_data_len - seq.shape[0] + inputs[i] = np.pad(seq, ((0, pad_len), (0, 0)), 'constant', constant_values=0) + labels[i] = np.pad(labels[i], (0, max_labels_len - labels[i].shape[0]), + 'constant', constant_values=-1) + + inputs = np.asarray(inputs, dtype=np.float32) + if labels is not None: + labels = np.asarray(labels, dtype=np.float32) + inputs = inputs.transpose((1, 0, 2)) + labels = labels.transpose((1, 0)) + + return (nd.array(inputs, dtype=inputs.dtype, ctx=context.Context('cpu_shared', 0)), + nd.array(x_lens, ctx=context.Context('cpu_shared', 0))) \ + if labels is None else ( + nd.array(inputs, dtype=inputs.dtype, ctx=context.Context('cpu_shared', 0)), + nd.array(x_lens, ctx=context.Context('cpu_shared', 0)), + nd.array(labels, dtype=labels.dtype, ctx=context.Context('cpu_shared', 0)), + nd.array(y_lens, ctx=context.Context('cpu_shared', 0))) + + + # This test is pointless on Windows because Windows doesn't fork + if platform.system() != 'Windows': + data = Dummy(True) + loader = DataLoader(data, batch_size=40, batchify_fn=data.batchify, num_workers=2) + for epoch in range(1): + for i, data in enumerate(loader): + if i % 100 == 0: + print(data) + print('{}:{}'.format(epoch, i)) if __name__ == '__main__': import nose -- To stop receiving notification emails like this one, please contact sxjscie...@apache.org.