I posted this in lasagne-users as well: Running into a strange error when
trying to calculate updates. The end
goal is a learned pruning/dropping layer, who's total number of activations
along with how binary those
activations are are part of the cost function - these activations from its
own weights are then multiplied by the
incoming layer in order to selectively turn off the least useful inputs.
Once the mask is learned, the layers
before and after can be resized. Seems sound in principle, but I'm having
trouble convincing theano of the idea.
The code:
------------------------------------------------------------------------------------------------------------------------------------------
from lasagne.layers import InputLayer, DenseLayer, Layer, get_output
from lasagne.nonlinearities import sigmoid, rectify, tanh
from numpy.random import random
from numpy.random import randint
from numpy import zeros
from theano import shared
import theano as th
import theano.tensor as T
import numpy as np
import lasagne as nn
floatX = th.config.floatX
batchsize = 4
Width = 10
Target = 5
np.set_printoptions(precision=2, suppress=True)
class Mask(Layer):
def __init__(self, incoming, A=10**4, **kwargs):
super(Mask, self).__init__(incoming, **kwargs)
num_inputs = self.input_shape[1]
self.A = A
self.num_units = num_inputs
W = ((random( (batchsize,num_inputs))-(1/2))*.001).astype(floatX)
for n in range(1,W.shape[0]):
W[n]=W[0]
W = shared(W )
self.W = self.add_param(W,(batchsize,num_inputs), name = 'W_Mask')
def get_output_for(self, input, **kwargs):
#return ((sigmoid(rectify(self.W*self.A) )-(1/2))*2)
return ((tanh(self.A*self.W)/2)+.5 +(input*0) ).astype(floatX)
def average(self):
Acc = zeros((1,self.num_units))
W = self.W.get_value()
for w in W:
Acc += w
Acc /= batchsize
W = zeros((batchsize,self.num_units))
for n in range(batchsize):
W[n] = Acc[:]
self.W.set_value(W)
class MaskLayer(Layer):
def __init__(self, incoming, W, A=10**6, **kwargs):
super(MaskLayer, self).__init__(incoming, **kwargs)
num_inputs = self.input_shape[1]
self.A = A
self.num_units = num_inputs
self.W = self.add_param(W,(1,num_inputs), name = 'W')
def get_output_for(self, input, **kwargs):
return (((tanh(self.A*self.W)/2)+.5)*input).astype(floatX)
In = InputLayer((3,Width),input_var=x)
M = Mask(In)
LM = MaskLayer(In,M.W)
LN = DenseLayer(LM,1,nonlinearity=sigmoid)
X = (random((batchsize,Width))-.5).astype(floatX)
x = T.matrix('x')
Out = get_output(LN,x)
M1 = get_output(M,x)
params = nn.layers.get_all_params(LN)
mask = th.function([x],M1)
cost = ((Target - (M1.sum().mean() / batchsize))**2 +
1000*T.sin(M1*np.pi).mean() - Out.mean()).astype(floatX)
Updates = nn.updates.sgd(cost,params,learning_rate=.001)
#train = th.function([x],[cost],updates=Updates)
run = th.function([x],[cost, Out, M1],on_unused_input='ignore',
updates=Updates)
print('In:\n',X)
print('Mask:\n',M.get_output_for(X).eval())
print('Masked:\n',LM.get_output_for(X).eval())
print('Dense:\n',LN.get_output_for(X).eval())
The Error:
--------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------TypeError
Traceback (most recent call
last)/usr/local/lib/python3.4/dist-packages/theano/compile/pfunc.py in
rebuild_collect_shared(outputs, inputs, replace, updates, rebuild_strict,
copy_inputs_over, no_default_updates) 192 update_val =
store_into.type.filter_variable(update_val,--> 193
allow_convert=False) 194 except
TypeError:
/usr/local/lib/python3.4/dist-packages/theano/tensor/type.py in
filter_variable(self, other, allow_convert) 233
other=other,--> 234 self=self)) 235
TypeError: Cannot convert Type TensorType(float64, matrix) (of Variable
Elemwise{sub,no_inplace}.0) into Type TensorType(float32, matrix). You can try
to manually convert Elemwise{sub,no_inplace}.0 into a TensorType(float32,
matrix).
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
<ipython-input-70-60c11801fab9> in <module>() 68 Updates =
nn.updates.sgd(cost,params,learning_rate=.001) 69 #train =
th.function([x],[cost],updates=Updates)---> 70 run = th.function([x],[cost,
Out, M1],on_unused_input='ignore', updates=Updates)
71 print('In:\n',X)
72 print('Mask:\n',M.get_output_for(X).eval())
/usr/local/lib/python3.4/dist-packages/theano/compile/function.py in
function(inputs, outputs, mode, updates, givens, no_default_updates,
accept_inplace, name, rebuild_strict, allow_input_downcast, profile,
on_unused_input) 320 on_unused_input=on_unused_input,
321 profile=profile,--> 322
output_keys=output_keys) 323 # We need to add the flag check_aliased
inputs if we have any mutable or
324 # borrowed used defined inputs
/usr/local/lib/python3.4/dist-packages/theano/compile/pfunc.py in pfunc(params,
outputs, mode, updates, givens, no_default_updates, accept_inplace, name,
rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys)
441 rebuild_strict=rebuild_strict,
442 copy_inputs_over=True,--> 443
no_default_updates=no_default_updates)
444 # extracting the arguments
445 input_variables, cloned_extended_outputs, other_stuff = output_vars
/usr/local/lib/python3.4/dist-packages/theano/compile/pfunc.py in
rebuild_collect_shared(outputs, inputs, replace, updates, rebuild_strict,
copy_inputs_over, no_default_updates) 206 ' function
to remove broadcastable dimensions.') 207 --> 208 raise
TypeError(err_msg, err_sug) 209 assert update_val.type ==
store_into.type 210
TypeError: ('An update must have the same type as the original shared variable
(shared_var=W_Mask, shared_var.type=TensorType(float32, matrix),
update_val=Elemwise{sub,no_inplace}.0, update_val.type=TensorType(float64,
matrix)).', 'If the difference is related to the broadcast pattern, you can
call the tensor.unbroadcast(var, axis_to_unbroadcast[, ...]) function to remove
broadcastable dimensions.')
--
---
You received this message because you are subscribed to the Google Groups
"theano-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
