SINGA-362 Add funcitons to support einsum function 1. fix the bug and support some test case 2. need to do some fix and use transpose function developed by yisen
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/7d25ed93 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/7d25ed93 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/7d25ed93 Branch: refs/heads/master Commit: 7d25ed93b4a05ef24ec73b8277945bd24db716d2 Parents: 16c6111 Author: sheyujian <[email protected]> Authored: Tue May 22 10:04:20 2018 +0800 Committer: sheyujian <[email protected]> Committed: Thu May 24 10:43:35 2018 +0800 ---------------------------------------------------------------------- include/singa/core/device.h | 2 +- include/singa/core/tensor.h | 6 +-- python/singa/tensor.py | 107 ++++++++++++++++++++++++++++++++------- src/api/core_tensor.i | 7 +++ src/core/device/device.cc | 2 +- src/core/tensor/tensor.cc | 55 ++++++++++++++------ test/python/test_tensor.py | 61 ++++++++++++++++++++-- test/singa/test_tensor.cc | 31 ++++++++++++ 8 files changed, 228 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/include/singa/core/device.h ---------------------------------------------------------------------- diff --git a/include/singa/core/device.h b/include/singa/core/device.h index 24569f4..d6b8bf3 100644 --- a/include/singa/core/device.h +++ b/include/singa/core/device.h @@ -78,7 +78,7 @@ class Device { virtual void RepeatDataToFrom(Block* dst, Block* src, size_t nBytes, CopyDirection direct, bool broadcast_flag, int axis_shape, int shape_outer, int chunk, - vector<int> repeats, int dst_offset, int src_offset); + vector<size_t> repeats, int dst_offset, int src_offset); void CopyDataFromHostPtr(Block* dst, const void* src, size_t nBytes, size_t dst_offset = 0); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/include/singa/core/tensor.h ---------------------------------------------------------------------- diff --git a/include/singa/core/tensor.h b/include/singa/core/tensor.h index c7958ff..3cfafc5 100644 --- a/include/singa/core/tensor.h +++ b/include/singa/core/tensor.h @@ -165,7 +165,7 @@ class Tensor { /// Meta data would not be copied! void CopyData(const Tensor &other); - void RepeatData(vector<int> repeats, int axis, int total_repeats, const Tensor &other); + void RepeatData(vector<size_t> repeats, int axis, int total_repeats, const Tensor &other); /// Deserialize data, shape and transpose from protobuf object. void FromProto(const singa::TensorProto &proto); @@ -177,7 +177,7 @@ class Tensor { /// device. If 'device' is nullptr, then clone it one the current device. Tensor Clone(std::shared_ptr<Device> device = nullptr) const; - Tensor Repeat(vector<int> repeats, int axis, std::shared_ptr<Device> device = nullptr) ; + Tensor Repeat(vector<size_t> repeats, int axis, std::shared_ptr<Device> device = nullptr) ; // Tensor operations @@ -291,7 +291,7 @@ Tensor Reshape(const Tensor &in, Shape &&s); void CopyDataToFrom(Tensor *dst, const Tensor &src, const size_t num, const size_t dst_offset = 0, const size_t src_offset = 0); -void RepeatDataToFrom(bool broadcast_flag, vector<int> repeats, int axis, +void RepeatDataToFrom(bool broadcast_flag, vector<size_t> repeats, int axis, Tensor *dst, const Tensor &in, const size_t num, const size_t dst_offset = 0, const size_t src_offset = 0); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/python/singa/tensor.py ---------------------------------------------------------------------- diff --git a/python/singa/tensor.py b/python/singa/tensor.py index ff7206c..d559ecb 100644 --- a/python/singa/tensor.py +++ b/python/singa/tensor.py @@ -136,6 +136,10 @@ class Tensor(object): ''' return self.data.transpose() + # def transpose(self): + + # return self.data + def size(self): # TODO(wangwei) compute size ''' Returns: @@ -255,6 +259,28 @@ class Tensor(object): ''' return _call_singa_func(self.data.Clone) + def repeat_(self, repeats, axis): + ret = CTensor() + if isinstance(repeats, int): + if axis == 9999: + Repeats = [repeats,] + ret = self.data.Repeat(Repeats, axis) + else: + Repeats = [repeats,] + ret = self.data.Repeat(Repeats, axis) + return ret + + elif isinstance(repeats, tuple) or isinstance(repeats, list): + if axis == 9999: + ret = self.data.Repeat(list(repeats), axis) + + elif axis >= 0: + ret = self.data.Repeat(list(repeats), axis) + return ret + + + + def T(self): ''' shallow copy, negate the transpose field. @@ -1104,23 +1130,25 @@ def sum2(t, axis=None, out=None): t_ndim = t.ndim() if axis is None: - one = Tensor(t.shape, t.device, t.dtype) + one = Tensor(t.shape, t.device) one.set_value(1.0) ret = tensordot(t, one, t_ndim) if isinstance(axis,int): if axis < 0: - axis += 2 + axis += t_ndim axis_shape = t_shape[axis] - one = Tensor(axis_shape, t.device, t.dtype) + axis_shape = int(axis_shape) + one = Tensor(shape = (axis_shape, ), device = t.device) one.set_value(1.0) ret = tensordot(t, one, axes=([axis],[0])) if isinstance(axis,tuple): l_axis = list(axis) axis_shape = [t_shape[x] for x in axis] - one = Tensor(axis_shape, t.device, t.dtype) + axisshape = tuple(axis_shape) + one = Tensor(axisshape, t.device) one.set_value(1.0) one_axis = [x for x in range(one.ndim())] ret = tensordot(t, one, (l_axis,one_axis)) @@ -1133,31 +1161,59 @@ def sum2(t, axis=None, out=None): else: return ret -def repeat(t, repeats, axis = None): +def repeat (t, repeats, axis = None): + t_ndim = t.ndim() if isinstance(repeats, int): if repeats < 0: raise ValueError("'repeats' should not be negative: {}".format(repeats)) + if axis != None and axis < 0: + axis += t_ndim # broadcast = True if axis == None: axis = 9999 - if axis < 0: - axis += 2 - ret = singa.Repeat(t, list(repeats), axis) + ret = Tensor() + ret.shape = (product(t.shape)*repeats,) + # Repeats = [repeats,] + ret.data = t.repeat_(repeats, axis) + # ret.data = t.data.Repeat(Repeats, axis) + elif axis >= 0: + ret = Tensor() + t_shape = list(t.shape) + t_shape[axis] = t.shape[axis]*repeats + print(t_shape) + ret.shape = tuple(t_shape) + print(ret.shape) + # Repeats = [repeats,] + ret.data = t.repeat_(repeats, axis) + # ret.data = t.data.Repeat(Repeats, axis) + print(ret.shape) + elif isinstance(repeats, tuple) or isinstance(repeats, list): for rep in repeats: if rep < 0: raise ValueError("'repeats' should be int or sequence: {}".format(repeats)) + + if axis != None and axis < 0: + axis += t_ndim if axis == None: axis = 9999 - if axis < 0: - axis += 2 - ret = singa.Repeat(t, list(repeats), axis) - t_shape = t.shape - t_shape[axis] = sum(repeats) - ret = ret.reshape(t_shape) + ret = Tensor() + ret.shape = (sum(repeats), ) + t_shape = list(t.shape) + ret.data = t.repeat_(repeats, axis) + #ret = t.data.Repeat(list(repeats), axis) + + elif axis >= 0: + ret = Tensor() + t_shape = list(t.shape) + t_shape[axis] = sum(repeats) + ret.shape = tuple(t_shape) + ret.data = t.repeat_(repeats, axis) + #ret = t.data.Repeat(list(repeats), axis) else: raise ValueError('repeats should be int or sequence') return ret + def tensordot (A,B,axes=2): @@ -1188,7 +1244,7 @@ def tensordot (A,B,axes=2): # if A is in shape(3,2,4), B is in shape(4,2,5), it will return a matrix in shape(3,2,2,5) #when axes is 2 and A,B are shape (3,2,4) and (2,4,5), it will return a matrix in shape(3,5) - if type(axes) == int: + if type(axes) == int or type(axes) == long: axes_A = list(range(-axes, 0)) axes_B = list(range(0, axes)) axes_B = axes_B @@ -1257,11 +1313,24 @@ def tensordot (A,B,axes=2): newshape_b = (N2, N1) oldb = [b_shape[axis] for axis in notin] # do transpose and reshape to get the 2D matrix to do multiplication - at = A.transpose(newaxes_a).reshape(newshape_a) - bt = B.transpose(newaxes_b).reshape(newshape_b) - res = mult(at, bt) + A_ = to_numpy(A) + B_ = to_numpy(B) + at_ = np.transpose(A_,newaxes_a).reshape(newshape_a) + bt_ = np.transpose(B_,newaxes_b).reshape(newshape_b) + at = from_numpy(at_) + bt = from_numpy(bt_) + res = mult(at,bt) + if len(olda + oldb) == 0: + olda = [1] + oldb = [1] + res.reshape(tuple(olda + oldb)) + else: + res.reshape(tuple(olda + oldb)) + print(res.shape) + # res_ = np.dot(at_, bt_) + # res = from_numpy(res_.reshape(olda + oldb)) #reshape the result - return res.reshape(olda + oldb) + return res def div(lhs, rhs, ret=None): '''Elementi-wise division. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/src/api/core_tensor.i ---------------------------------------------------------------------- diff --git a/src/api/core_tensor.i b/src/api/core_tensor.i index 31562c9..756fe60 100644 --- a/src/api/core_tensor.i +++ b/src/api/core_tensor.i @@ -119,9 +119,12 @@ namespace singa{ %template(CopyIntDataFromHostPtr) CopyDataFromHostPtr<int>; void CopyData(const Tensor &other); + void RepeatData(std::vector<size_t> repeats, int axis, int total_repeats, const Tensor &src); Tensor Clone() const; + Tensor Repeat(std::vector<size_t> repeats, int axis); Tensor T() const; + #if USE_JAVA %rename(iAdd) operator+=(const Tensor &t); %rename(iSub) operator-=(const Tensor &t); @@ -157,6 +160,10 @@ namespace singa{ void CopyDataToFrom(Tensor *dst, const Tensor &src, size_t num, size_t src_offset = 0, size_t dst_offset = 0); + void RepeatDataToFrom(bool broadcast_flag, std::vector<size_t> repeats, int axis, + Tensor *dst, const Tensor &src, const size_t num, + const size_t dst_offset, const size_t src_offset); + Tensor Reshape(const Tensor &in, const std::vector<size_t> &s); Tensor Abs(const Tensor &t); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/src/core/device/device.cc ---------------------------------------------------------------------- diff --git a/src/core/device/device.cc b/src/core/device/device.cc index d569015..135ae3a 100644 --- a/src/core/device/device.cc +++ b/src/core/device/device.cc @@ -67,7 +67,7 @@ void Device::CopyDataToFrom(Block* dst, Block* src, size_t nBytes, void Device::RepeatDataToFrom(Block* dst, Block* src, size_t nBytes, CopyDirection direct, bool broadcast_flag, int axis_shape, int shape_outer, int chunk, - vector<int> repeats, int dst_offset, int src_offset) { + vector<size_t> repeats, int dst_offset, int src_offset) { const char *src_data = reinterpret_cast<const char*>(src->data()) + dst_offset; char *dst_data = reinterpret_cast<char*>(dst->mutable_data()) + src_offset; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/src/core/tensor/tensor.cc ---------------------------------------------------------------------- diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc index 22541df..b75ac40 100644 --- a/src/core/tensor/tensor.cc +++ b/src/core/tensor/tensor.cc @@ -216,8 +216,8 @@ void Tensor::CopyData(const Tensor &src) { } } -void Tensor::RepeatData(vector<int> repeats, int axis, int total_repeats, const Tensor &src) { - if(axis == Noaxis) { +void Tensor::RepeatData(vector<size_t> repeats, int axis, int total_repeats, const Tensor &src) { + if(repeats.size() == 1) { CHECK_EQ(Size(), src.Size()*total_repeats); } else { CHECK_EQ(Size(), src.Size()*total_repeats/src.shape()[axis]); @@ -345,25 +345,44 @@ Tensor Tensor::Clone(std::shared_ptr<Device> device) const { return t; } -Tensor Tensor::Repeat(vector<int> repeats, int axis, std::shared_ptr<Device> device) { +Tensor Tensor::Repeat(vector<size_t> repeats, int axis, std::shared_ptr<Device> device) { if (device == nullptr) device = device_; - Tensor t; + vector<size_t> tshape; int total_repeats = 0; if (axis == Noaxis) { total_repeats = repeats[0]; - t.shape_.push_back(Product(shape_)*total_repeats); + tshape.push_back(Product(shape_)*total_repeats); } else { - for (size_t i = 0; i < shape_[axis]; i++) { - if(repeats[i] < 0) { - LOG(FATAL) << "the repeats number is less than zero"; + if (repeats.size() == 1){ + total_repeats = repeats[0]; + for (int i = 0; i < shape_.size(); i++) { + if (i == axis) { + tshape.push_back(shape_[i] * total_repeats); + } else { + tshape.push_back(shape_[i]); + } + } + } else { + if (repeats.size() != shape_[axis]) { + LOG(FATAL) << "the repeats number doesn't match the axis"; + } + for (size_t i = 0; i < shape_[axis]; i++) { + if(repeats[i] < 0) { + LOG(FATAL) << "the repeats number is less than zero"; + } + total_repeats += repeats[i]; + } + for (int i = 0; i < shape_.size(); i++){ + if (i == axis) { + tshape.push_back(total_repeats); + } else{ + tshape.push_back(shape_[i]); + } } - total_repeats += repeats[i]; - t.shape_.push_back(Product(shape_)/shape_[axis]*total_repeats); } } - t.device_ = device_; - t.data_type_ = data_type_; - t.strides_.push_back(1); + Tensor t(tshape, device_); + //t.strides_.push_back(1); t.RepeatData(repeats, axis, total_repeats, *this); return t; } @@ -522,7 +541,7 @@ void CopyDataToFrom(Tensor *dst, const Tensor &src, const size_t num, } } -void RepeatDataToFrom(bool broadcast_flag, vector<int> repeats, int axis, +void RepeatDataToFrom(bool broadcast_flag, vector<size_t> repeats, int axis, Tensor *dst, const Tensor &src, const size_t num, const size_t dst_offset, const size_t src_offset) { if (repeats.size() == 1) { @@ -543,15 +562,20 @@ void RepeatDataToFrom(bool broadcast_flag, vector<int> repeats, int axis, auto s_offset = src_offset * width; int chunk = width; int axis_shape = 1; + int shape_outer = 1; if (axis == Noaxis){ axis_shape = 1; + shape_outer = Product(src.shape()); } else { + for (size_t i = 0; i < axis; i++) { + shape_outer *= src.shape()[i]; + } axis_shape = src.shape()[axis]; for(size_t i = axis + 1; i < src.nDim(); i++) { chunk *= src.shape()[i]; } } - int shape_outer = Product(src.shape()); + std::shared_ptr<Device> src_dev = src.device(), dst_dev = dst->device(); Block *from = src.block(), *to = dst->block(); if (dst_dev->lang() != src_dev->lang()) { @@ -667,6 +691,7 @@ void Tensor::SetValue(const SType x) { CHECK_EQ(sizeof(SType), SizeOf(data_type_)); //auto size = Size(); auto ptr = block_; + TYPE_LANG_SWITCH(data_type_, DType, device_->lang(), Lang, { // TODO(wangwei) cast x to DType device_->Exec([this, x, ptr](Context * ctx) { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/test/python/test_tensor.py ---------------------------------------------------------------------- diff --git a/test/python/test_tensor.py b/test/python/test_tensor.py index 0b3b85b..a47bbff 100644 --- a/test/python/test_tensor.py +++ b/test/python/test_tensor.py @@ -167,9 +167,10 @@ class TestTensorMethods(unittest.TestCase): def test_einsum(self): - a = np.arange(12).reshape(3, 2, 2) - + a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2]) + a = np.reshape(a,(2,3,2)) ta = tensor.from_numpy(a) + res1 = np.einsum('kij,kij->kij', a, a) tres1 = tensor.einsum('kij,kij->kij', ta, ta) Tres1 = tensor.to_numpy(tres1) @@ -177,9 +178,61 @@ class TestTensorMethods(unittest.TestCase): tres2 = tensor.einsum('kij,kih->kjh', ta, ta) Tres2 = tensor.to_numpy(tres2) - self.assertEqual(np.sum(Tres1 - res1), 0.) - self.assertEqual(np.sum(Tres2 - res2), 0.) + self.assertAlmostEqual(np.sum(Tres1 - res1), 0.,places=3) + self.assertAlmostEqual(np.sum(Tres2 - res2), 0.,places=3) + + def test_repeat(self): + + a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2]) + a = np.reshape(a,(2,3,2)) + ta = tensor.from_numpy(a) + + ta_repeat1 = tensor.repeat(ta,2,axis = None) + a_repeat1 = np.repeat(a,2,axis = None) + Ta_repeat1 = tensor.to_numpy(ta_repeat1) + ta_repeat2 = tensor.repeat(ta, 4, axis = 1) + a_repeat2 = np.repeat(a, 4, axis = 1) + Ta_repeat2 = tensor.to_numpy(ta_repeat2) + print(Ta_repeat2) + print(a_repeat2) + + self.assertAlmostEqual(np.sum(Ta_repeat1 - a_repeat1), 0., places=3) + self.assertAlmostEqual(np.sum(Ta_repeat2 - a_repeat2), 0., places=3) + + def test_sum2(self): + a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2]) + a = np.reshape(a,(2,3,2)) + ta = tensor.from_numpy(a) + + a_sum0 = np.sum(a) + ta_sum0 = tensor.sum2(ta) + Ta_sum0 = tensor.to_numpy(ta_sum0) + a_sum1 = np.sum(a, axis = 1) + ta_sum1 = tensor.sum2(ta, axis = 1) + Ta_sum1 = tensor.to_numpy(ta_sum1) + a_sum2 = np.sum(a, axis = 2) + ta_sum2 = tensor.sum2(ta, axis = 2) + Ta_sum2 = tensor.to_numpy(ta_sum2) + + self.assertAlmostEqual(np.sum(a_sum0 - Ta_sum0), 0., places=3) + self.assertAlmostEqual(np.sum(a_sum1 - Ta_sum1), 0., places=3) + self.assertAlmostEqual(np.sum(a_sum2 - Ta_sum2), 0., places=3) + + def test_tensordot(self): + a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2]) + a = np.reshape(a,(2,3,2)) + + ta = tensor.from_numpy(a) + + res1 = np.tensordot(a, a, axes = 1) + tres1 = tensor.tensordot(ta, ta, axes = 1) + Tres1 = tensor.to_numpy(tres1) + res2 = np.tensordot(a, a, axes = ([0,1],[2,1])) + tres2 = tensor.tensordot(ta, ta, axes = ([0,1],[2,1])) + Tres2 = tensor.to_numpy(tres2) + self.assertAlmostEqual(np.sum(Tres1 - res1), 0., places=3) + self.assertAlmostEqual(np.sum(Tres2 - res2), 0., places=3) if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d25ed93/test/singa/test_tensor.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_tensor.cc b/test/singa/test_tensor.cc index 316b996..c8df3ee 100644 --- a/test/singa/test_tensor.cc +++ b/test/singa/test_tensor.cc @@ -129,3 +129,34 @@ TEST(TensorClass, T) { EXPECT_EQ(t.shape()[1], o.shape()[0]); } +TEST(TensorClass, Repeat) { + float data[] = {1.0f, 2.0f, 3.0f}; + Tensor t(Shape{3}); + t.CopyDataFromHostPtr(data, 3); + + Tensor o = t.Repeat(vector <size_t>{2},9999); + const float* dptr = static_cast<const float*>(o.block()->data()); + EXPECT_FLOAT_EQ(1.0f, dptr[0]); + EXPECT_FLOAT_EQ(1.0f, dptr[1]); + EXPECT_FLOAT_EQ(2.0f, dptr[2]); + EXPECT_FLOAT_EQ(2.0f, dptr[3]); + EXPECT_FLOAT_EQ(3.0f, dptr[4]); + EXPECT_FLOAT_EQ(3.0f, dptr[5]); +} + +TEST(TensorCLass, RepeatData) { + float data[] = {1.0f, 2.0f, 3.0f}; + Tensor t(Shape{3}); + t.CopyDataFromHostPtr(data, 3); + + Tensor o(Shape{6}); + o.RepeatData({2},9999,2,t); + const float* dptr = static_cast<const float*>(o.block()->data()); + EXPECT_FLOAT_EQ(1.0f, dptr[0]); + EXPECT_FLOAT_EQ(1.0f, dptr[1]); + EXPECT_FLOAT_EQ(2.0f, dptr[2]); + EXPECT_FLOAT_EQ(2.0f, dptr[3]); + EXPECT_FLOAT_EQ(3.0f, dptr[4]); + EXPECT_FLOAT_EQ(3.0f, dptr[5]); +} +
