Re: [theano-users] Batched matrix operations in Theano

[Frédéric Bastien]

Do you want to make a PR or of this?

Le dim. 23 juil. 2017 08:20, Maxim Kochurov  a
écrit :

> class BatchedDiag(tt.Op):
> """
> Fast BatchedDiag allocation
> """
> __props__ = ()
>
> def make_node(self, diag):
> diag = tt.as_tensor_variable(diag)
> if diag.type.ndim != 2:
> raise TypeError('data argument must be a matrix', diag.type)
>
> return tt.Apply(self, [diag], [tt.tensor3(dtype=diag.dtype)])
>
> def perform(self, node, ins, outs, params=None):
> (C,) = ins
> (z,) = outs
>
> bc = C.shape[0]
> dim = C.shape[-1]
> Cd = np.zeros((bc, dim, dim), C.dtype)
> bidx = np.repeat(np.arange(bc), dim)
> didx = np.tile(np.arange(dim), bc)
> Cd[bidx, didx, didx] = C.flatten()
> z[0] = Cd
>
> def grad(self, inputs, gout):
> (gz,) = gout
> idx = tt.arange(gz.shape[-1])
> return [gz[..., idx, idx]]
>
> def infer_shape(self, nodes, shapes):
> return [(shapes[0][0], ) + (shapes[0][1],) * 2]
>
> Here is code for Custom Op that might work faster when taking gradients
>
>
> суббота, 7 мая 2016 г., 16:00:54 UTC+3 пользователь Tambet Matiisen
> написал:
>
>> OK, solved. I used Keras wrapper K.zeros(), but this created Numpy matrix
>> of zeros, which failed with Theano expression as dimension. After switching
>> to full Theano implementation the error went away. The final code looks
>> like this:
>>
>> # initialize with zeros
>> batch_size = x.shape[0]
>> a = T.zeros((batch_size, num_actuators, num_actuators))
>> # set diagonal elements
>> batch_idx = T.extra_ops.repeat(T.arange(batch_size), num_actuators)
>> diag_idx = T.tile(T.arange(num_actuators), batch_size)
>> b = T.set_subtensor(a[batch_idx, diag_idx, diag_idx],
>> T.flatten(T.exp(x[:, :num_actuators])))
>> # set lower triangle
>> cols = np.concatenate([np.array(range(i), dtype=np.uint) for i in
>> xrange(num_actuators)])
>> rows = np.concatenate([np.array([i]*i, dtype=np.uint) for i in
>> xrange(num_actuators)])
>> cols_idx = T.tile(T.as_tensor_variable(cols), batch_size)
>> rows_idx = T.tile(T.as_tensor_variable(rows), batch_size)
>> batch_idx = T.extra_ops.repeat(T.arange(batch_size), len(cols))
>> c = T.set_subtensor(b[batch_idx, rows_idx, cols_idx], T.flatten(x[:,
>> num_actuators:]))
>>
>> Thanks injecting me belief that it is possible!
>>
>>   Tambet
>>
>> reede, 6. mai 2016 17:57.02 UTC+3 kirjutas nouiz:
>>>
>>> what error do you get?
>>>
>>>
>>> On Fri, May 6, 2016 at 10:54 AM, Tambet Matiisen 
>>> wrote:
>>>
 I could not figure out how make broadcasting work here, so I
 implemented option 2.

 num_actuators=4
 x = K.variable([range(num_actuators*(num_actuators+1)/2)]*5)

 batch_size = K.shape(x)[0]
 a = K.zeros((batch_size.eval(), num_actuators, num_actuators))

 # populate diagonal
 batch_idx = T.extra_ops.repeat(T.arange(batch_size), num_actuators)
 diag_idx = T.tile(T.arange(num_actuators), batch_size)
 b = T.set_subtensor(a[batch_idx, diag_idx, diag_idx],
 T.flatten(K.exp(x[:, :num_actuators])))

 # populate lower triangle
 cols = np.concatenate([np.array(range(i), dtype=np.uint) for i in
 xrange(num_actuators)])
 rows = np.concatenate([np.array([i]*i, dtype=np.uint) for i in
 xrange(num_actuators)])
 cols_idx = T.tile(K.variable(cols, dtype=int), batch_size)
 rows_idx = T.tile(K.variable(rows, dtype=int), batch_size)
 batch_idx = T.extra_ops.repeat(T.arange(batch_size), len(cols))
 c = T.set_subtensor(b[batch_idx, rows_idx, cols_idx], T.flatten(x[:,
 num_actuators:]))

 It works nicely, but only because I eval() batch_size when creating all
 zeros array. In real application I don't know the batch size beforehand and
 using it without eval() gives an error. So the question is - can you create
 a matrix in Theano dynamically, depending on some value in computational
 graph?

   Tambet

 reede, 6. mai 2016 16:14.59 UTC+3 kirjutas nouiz:
>
> broadcasting could be in theory more efficient. So this would request
> that you try option 1.
>
> Otherwise, both should work.
>
> Fred
>
> On Fri, May 6, 2016 at 9:12 AM, Tambet Matiisen 
> wrote:
>
>> Actually I know the dimensions of the matrix beforehand, so I can do
>> those calculations in Python+Numpy. Following seems to do the trick:
>>
>> num_actuators = 3
>> x = [1,2,3,4,5,6]
>> a = K.zeros((num_actuators, num_actuators))
>>
>> # set diagonal elements
>> b = T.set_subtensor(a[range(num_actuators), range(num_actuators)],
>> K.exp(x[:num_actuators]))
>>
>> # set lower triangle
>> cols = np.concatenate([np.array(range(i), dtype=np.uint) for i in
>> xrange(num_actuators)])
>> 

Re: [theano-users] Batched matrix operations in Theano

[Maxim Kochurov]

class BatchedDiag(tt.Op):
"""
Fast BatchedDiag allocation
"""
__props__ = ()

def make_node(self, diag):
diag = tt.as_tensor_variable(diag)
if diag.type.ndim != 2:
raise TypeError('data argument must be a matrix', diag.type)

return tt.Apply(self, [diag], [tt.tensor3(dtype=diag.dtype)])

def perform(self, node, ins, outs, params=None):
(C,) = ins
(z,) = outs

bc = C.shape[0]
dim = C.shape[-1]
Cd = np.zeros((bc, dim, dim), C.dtype)
bidx = np.repeat(np.arange(bc), dim)
didx = np.tile(np.arange(dim), bc)
Cd[bidx, didx, didx] = C.flatten()
z[0] = Cd

def grad(self, inputs, gout):
(gz,) = gout
idx = tt.arange(gz.shape[-1])
return [gz[..., idx, idx]]

def infer_shape(self, nodes, shapes):
return [(shapes[0][0], ) + (shapes[0][1],) * 2]

Here is code for Custom Op that might work faster when taking gradients


суббота, 7 мая 2016 г., 16:00:54 UTC+3 пользователь Tambet Matiisen написал:
>
> OK, solved. I used Keras wrapper K.zeros(), but this created Numpy matrix 
> of zeros, which failed with Theano expression as dimension. After switching 
> to full Theano implementation the error went away. The final code looks 
> like this:
>
> # initialize with zeros
> batch_size = x.shape[0]
> a = T.zeros((batch_size, num_actuators, num_actuators))
> # set diagonal elements
> batch_idx = T.extra_ops.repeat(T.arange(batch_size), num_actuators)
> diag_idx = T.tile(T.arange(num_actuators), batch_size)
> b = T.set_subtensor(a[batch_idx, diag_idx, diag_idx], 
> T.flatten(T.exp(x[:, :num_actuators])))
> # set lower triangle
> cols = np.concatenate([np.array(range(i), dtype=np.uint) for i in 
> xrange(num_actuators)])
> rows = np.concatenate([np.array([i]*i, dtype=np.uint) for i in 
> xrange(num_actuators)])
> cols_idx = T.tile(T.as_tensor_variable(cols), batch_size)
> rows_idx = T.tile(T.as_tensor_variable(rows), batch_size)
> batch_idx = T.extra_ops.repeat(T.arange(batch_size), len(cols))
> c = T.set_subtensor(b[batch_idx, rows_idx, cols_idx], T.flatten(x[:, 
> num_actuators:]))
>
> Thanks injecting me belief that it is possible!
>
>   Tambet
>
> reede, 6. mai 2016 17:57.02 UTC+3 kirjutas nouiz:
>>
>> what error do you get?
>>
>>
>> On Fri, May 6, 2016 at 10:54 AM, Tambet Matiisen  
>> wrote:
>>
>>> I could not figure out how make broadcasting work here, so I implemented 
>>> option 2.
>>>
>>> num_actuators=4
>>> x = K.variable([range(num_actuators*(num_actuators+1)/2)]*5)
>>>
>>> batch_size = K.shape(x)[0]
>>> a = K.zeros((batch_size.eval(), num_actuators, num_actuators))
>>>
>>> # populate diagonal
>>> batch_idx = T.extra_ops.repeat(T.arange(batch_size), num_actuators)
>>> diag_idx = T.tile(T.arange(num_actuators), batch_size)
>>> b = T.set_subtensor(a[batch_idx, diag_idx, diag_idx], 
>>> T.flatten(K.exp(x[:, :num_actuators])))
>>>
>>> # populate lower triangle
>>> cols = np.concatenate([np.array(range(i), dtype=np.uint) for i in 
>>> xrange(num_actuators)])
>>> rows = np.concatenate([np.array([i]*i, dtype=np.uint) for i in 
>>> xrange(num_actuators)])
>>> cols_idx = T.tile(K.variable(cols, dtype=int), batch_size)
>>> rows_idx = T.tile(K.variable(rows, dtype=int), batch_size)
>>> batch_idx = T.extra_ops.repeat(T.arange(batch_size), len(cols))
>>> c = T.set_subtensor(b[batch_idx, rows_idx, cols_idx], T.flatten(x[:, 
>>> num_actuators:]))
>>>
>>> It works nicely, but only because I eval() batch_size when creating all 
>>> zeros array. In real application I don't know the batch size beforehand and 
>>> using it without eval() gives an error. So the question is - can you create 
>>> a matrix in Theano dynamically, depending on some value in computational 
>>> graph?
>>>
>>>   Tambet
>>>
>>> reede, 6. mai 2016 16:14.59 UTC+3 kirjutas nouiz:

 broadcasting could be in theory more efficient. So this would request 
 that you try option 1.

 Otherwise, both should work.

 Fred

 On Fri, May 6, 2016 at 9:12 AM, Tambet Matiisen  
 wrote:

> Actually I know the dimensions of the matrix beforehand, so I can do 
> those calculations in Python+Numpy. Following seems to do the trick:
>
> num_actuators = 3
> x = [1,2,3,4,5,6]
> a = K.zeros((num_actuators, num_actuators))
>
> # set diagonal elements
> b = T.set_subtensor(a[range(num_actuators), range(num_actuators)], 
> K.exp(x[:num_actuators]))
>
> # set lower triangle
> cols = np.concatenate([np.array(range(i), dtype=np.uint) for i in 
> xrange(num_actuators)])
> rows = np.concatenate([np.array([i]*i, dtype=np.uint) for i in 
> xrange(num_actuators)])
> c = T.set_subtensor(b[rows, cols], x[num_actuators:])
>
> K.eval(c)
>
>
> array([[  2.71828175,   0.,   0.],
>[  4.