sxjscience closed pull request #10090: [MXNET-86] Revert to pre-profile-changes
copy code
URL: https://github.com/apache/incubator-mxnet/pull/10090
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/src/ndarray/ndarray_function.cc b/src/ndarray/ndarray_function.cc
index 927c90690a4..552555adf84 100644
--- a/src/ndarray/ndarray_function.cc
+++ b/src/ndarray/ndarray_function.cc
@@ -26,7 +26,6 @@
#include "./ndarray_function.h"
#include "./ndarray_function-inl.h"
#include "../common/utils.h"
-#include "../operator/mxnet_op.h"
namespace mxnet {
namespace ndarray {
@@ -34,27 +33,17 @@ template<>
void Copy<cpu, cpu>(const TBlob &from, TBlob *to,
Context from_ctx, Context to_ctx,
RunContext ctx) {
- using namespace mxnet::op;
MSHADOW_TYPE_SWITCH(to->type_flag_, DType, {
if (to->type_flag_ == from.type_flag_) {
- TBlob dest = to->FlatTo1D<cpu, DType>();
- TBlob src = from.FlatTo1D<cpu, DType>();
- const size_t size = src.Size();
- if (dest.CheckContiguous() && src.CheckContiguous() && size >= 20000 /*
non-trivial size */) {
- CHECK_EQ(dest.shape_, src.shape_)
- << "Copy:shape mismatch:" << dest.shape_ << " vs " << src.shape_;
- mxnet_op::Kernel<mxnet_op::op_with_req<mshadow_op::identity,
kWriteTo>, cpu>::Launch(
- ctx.get_stream<cpu>(), src.Size(), dest.dptr<DType>(),
src.dptr<DType>());
- } else {
- mshadow::Copy(to->FlatTo1D<cpu, DType>(), from.FlatTo1D<cpu, DType>());
- }
+ mshadow::Copy(to->FlatTo1D<cpu, DType>(),
+ from.FlatTo1D<cpu, DType>());
} else {
MSHADOW_TYPE_SWITCH(from.type_flag_, SrcDType, {
to->FlatTo1D<cpu, DType>() =
- mshadow::expr::tcast<DType>(from.FlatTo1D<cpu, SrcDType>());
+ mshadow::expr::tcast<DType>(from.FlatTo1D<cpu, SrcDType>());
})
}
- });
+ })
}
template<typename DType, typename IType>
diff --git a/tests/python/unittest/test_gluon_data.py
b/tests/python/unittest/test_gluon_data.py
index 49b1b8e935f..93160aa0940 100644
--- a/tests/python/unittest/test_gluon_data.py
+++ b/tests/python/unittest/test_gluon_data.py
@@ -20,8 +20,14 @@
import unittest
import mxnet as mx
import numpy as np
+import random
from mxnet import gluon
+import platform
from common import setup_module, with_seed
+from mxnet.gluon.data import DataLoader
+import mxnet.ndarray as nd
+from mxnet import context
+from mxnet.gluon.data.dataset import Dataset
@with_seed()
def test_array_dataset():
@@ -112,6 +118,81 @@ def test_multi_worker():
for i, batch in enumerate(loader):
assert (batch.asnumpy() == i).all()
+@with_seed()
+def test_multi_worker_forked_data_loader():
+ """
+ Test should successfully run its course of multi-process/forked data
loader without errors
+ """
+ class Dummy(Dataset):
+ def __init__(self, random_shape):
+ self.random_shape = random_shape
+
+ def __getitem__(self, idx):
+ key = idx
+ if self.random_shape:
+ out = np.random.uniform(size=(random.randint(1000, 1100), 40))
+ labels = np.random.uniform(size=(random.randint(10, 15)))
+ else:
+ out = np.random.uniform(size=(1000, 40))
+ labels = np.random.uniform(size=(10))
+ return key, out, labels
+
+ def __len__(self):
+ return 50
+
+ def batchify(self, data):
+ """
+ Collate data into batch. Use shared memory for stacking.
+
+ :param data: a list of array, with layout of 'NTC'.
+ :return either x and x's unpadded lengths, or x, x's unpadded
lengths, y and y's unpadded lengths
+ if labels are not supplied.
+ """
+
+ # input layout is NTC
+ keys, inputs, labels = [item[0] for item in data], [item[1] for
item in data], \
+ [item[2] for item in data]
+
+ if len(data) > 1:
+ max_data_len = max([seq.shape[0] for seq in inputs])
+ max_labels_len = 0 if not labels else max([seq.shape[0] for
seq in labels])
+ else:
+ max_data_len = inputs[0].shape[0]
+ max_labels_len = 0 if not labels else labels[0].shape[0]
+
+ x_lens = [item.shape[0] for item in inputs]
+ y_lens = [item.shape[0] for item in labels]
+
+ for i, seq in enumerate(inputs):
+ pad_len = max_data_len - seq.shape[0]
+ inputs[i] = np.pad(seq, ((0, pad_len), (0, 0)), 'constant',
constant_values=0)
+ labels[i] = np.pad(labels[i], (0, max_labels_len -
labels[i].shape[0]),
+ 'constant', constant_values=-1)
+
+ inputs = np.asarray(inputs, dtype=np.float32)
+ if labels is not None:
+ labels = np.asarray(labels, dtype=np.float32)
+ inputs = inputs.transpose((1, 0, 2))
+ labels = labels.transpose((1, 0))
+
+ return (nd.array(inputs, dtype=inputs.dtype,
ctx=context.Context('cpu_shared', 0)),
+ nd.array(x_lens, ctx=context.Context('cpu_shared', 0))) \
+ if labels is None else (
+ nd.array(inputs, dtype=inputs.dtype,
ctx=context.Context('cpu_shared', 0)),
+ nd.array(x_lens, ctx=context.Context('cpu_shared', 0)),
+ nd.array(labels, dtype=labels.dtype,
ctx=context.Context('cpu_shared', 0)),
+ nd.array(y_lens, ctx=context.Context('cpu_shared', 0)))
+
+
+ # This test is pointless on Windows because Windows doesn't fork
+ if platform.system() != 'Windows':
+ data = Dummy(True)
+ loader = DataLoader(data, batch_size=40, batchify_fn=data.batchify,
num_workers=2)
+ for epoch in range(1):
+ for i, data in enumerate(loader):
+ if i % 100 == 0:
+ print(data)
+ print('{}:{}'.format(epoch, i))
if __name__ == '__main__':
import nose
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
