Hi.
I have been trying to develop imagenet, I have the base of LENET and
modifying it and adding things I would like to get Imagenet. My problem is
that weights are not updated so the network does not learn and I can not
figure out where the problem is and I do not know what to do! I am really
stressed
and blocked. So any help or idea would be welcome!!
I have attached two files, "layers.py" where the layers are described and
"network.py" where the architecture and the learning/testing processing is
described and carried out.
Thank you very much in advance.
Regards.
Beatriz
--
---
You received this message because you are subscribed to the Google Groups
"theano-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
import numpy as np
import theano
import theano.tensor as T
from theano.tensor.signal import pool
from theano.tensor.nnet import conv2d
from pylearn2.expr.normalize import CrossChannelNormalization
class Fully_Connected_Dropout(object):
# http://christianherta.de/lehre/dataScience/machineLearning/neuralNetworks/Dropout.php
def __init__(self, rng, is_train, input, n_in, n_out, W=None, b=None, p=0.5):
self.input = input
# end-snippet-1
rng = np.random.RandomState(1234)
srng = T.shared_randomstreams.RandomStreams(rng.randint(999999))
# for a discussion of the initialization, see
# https://plus.google.com/+EricBattenberg/posts/f3tPKjo7LFa
if W is None:
W_values = np.asarray(
rng.uniform(
low=-np.sqrt(6. / (n_in + n_out)),
high=np.sqrt(6. / (n_in + n_out)),
size=(n_in, n_out)
),
dtype=theano.config.floatX
)
W = theano.shared(value=W_values, name='W', borrow=True)
# init biases to positive values, so we should be initially in the linear regime of the linear rectified function
if b is None:
b_values = np.ones((n_out,), dtype=theano.config.floatX) * np.cast[theano.config.floatX](0.01)
b = theano.shared(value=b_values, name='b', borrow=True)
self.W = W
self.b = b
lin_output = T.dot(input, self.W) + self.b
output = theano.tensor.nnet.relu(lin_output)
# multiply output and drop -> in an approximation the scaling effects cancel out
input_drop = np.cast[theano.config.floatX](1. / p) * output
mask = srng.binomial(n=1, p=p, size=input_drop.shape, dtype=theano.config.floatX)
train_output = input_drop * mask
# is_train is a pseudo boolean theano variable for switching between training and prediction
self.output = T.switch(T.neq(is_train, 0), train_output, output)
# parameters of the model
self.params = [self.W, self.b]
class Fully_Connected_Softmax(object):
def __init__(self, rng, input, n_in, n_out, W=None, b=None,
activation=theano.tensor.nnet.relu):
self.input = input
if W is None:
W_values = np.asarray(
rng.uniform(
low=-np.sqrt(6. / (n_in + n_out)),
high=np.sqrt(6. / (n_in + n_out)),
size=(n_in, n_out)
),
dtype=theano.config.floatX
)
if activation == theano.tensor.nnet.sigmoid:
W_values *= 4
W = theano.shared(value=W_values, name='W', borrow=True)
if b is None:
b_values = np.zeros((n_out,), dtype=theano.config.floatX)
b = theano.shared(value=b_values, name='b', borrow=True)
self.W = W
self.b = b
lin_output = T.nnet.softmax(T.dot(input, self.W) + self.b)
self.output = (
lin_output if activation is None
else activation(lin_output)
)
self.params = [self.W, self.b]
class LeNetConvPoolLRNLayer(object):
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2), stride=(1, 1), lrn=False):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
self.lrn = lrn
if self.lrn:
self.lrn_func = CrossChannelNormalization()
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = np.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) //
np.prod(poolsize))
# initialize weights with random weights
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(
np.asarray(
rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX
),
borrow=True
)
# the bias is a 1D tensor -- one bias per output feature map
b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv2d(
input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape,
subsample=stride
)
conv_out = conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
# ReLu
self.output = T.maximum(conv_out, 0)
# pool each feature map individually, using maxpooling
pooled_out = pool.pool_2d(
input=conv_out,
ds=poolsize,
ignore_border=True
)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = pooled_out
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
self.params = [self.W, self.b]
# keep track of model input
self.input = input
class LeNetConvPoolLayer(object):
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(1, 1)):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = np.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) //
np.prod(poolsize))
# initialize weights with random weights
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(
np.asarray(
rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX
),
borrow=True
)
# the bias is a 1D tensor -- one bias per output feature map
b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv2d(
input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape,
)
conv_out = conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
# ReLu
self.output = T.maximum(conv_out, 0)
# pool each feature map individually, using maxpooling
pooled_out = pool.pool_2d(
input=conv_out,
ds=poolsize,
ignore_border=True
)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = pooled_out
# store parameters of this layer
self.params = [self.W, self.b]
# keep track of model input
self.input = input
# En este archivo de python se va a utilizar la base de datos de FRAV y Casia de imagenes y se va a seguir la configuracion de la CNN del paper 'Learn convolutional neural network for face anti-spoofing'
import numpy
import timeit
from pylab import *
from logistic_sgd import LogisticRegression
from sklearn.svm import SVC
import matplotlib.pyplot as plt
from read_Data import *
from layers import *
import sys
#nkerns=[96, 256, 386, 384, 256]
def evaluate_lenet5(learning_rate=0.001, n_epochs=10, nkerns=[10, 10, 10, 10, 10], batch_size=20):
""" Demonstrates lenet on MNIST dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: path to the dataset used for training /testing (MNIST here)
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
"""
# orig_stdout = sys.stdout
# f = file('out.txt', 'w')
# sys.stdout = f
print ('Start reading the data...')
rng = numpy.random.RandomState(123456)
#train_index, test_index, validate_index, train_set_x, test_set_x, y_train, y_test, valid_set_x, y_val = read_images()
# open_file2 = open('C:\Users\FRAV\Desktop\Beatriz\FRAv_casia_ImageNet\data_casia_all.pkl', 'rb')
open_file2 = open('data_frav_all_origi.pkl', 'rb')
[train_index, test_index, validate_index, train_set_x, test_set_x, y_train, y_test, valid_set_x, y_val] = pickle.load(open_file2)
open_file2.close()
print (numpy.array(train_set_x).shape, numpy.array(test_set_x).shape, numpy.array(valid_set_x).shape)
print (numpy.array(y_train).shape, numpy.array(y_test).shape, numpy.array(y_val).shape)
train_set_x = theano.shared(numpy.array(train_set_x, dtype= 'float32'))
test_set_x = theano.shared(numpy.array(test_set_x, dtype= 'float32'))
train_set_y = theano.shared(numpy.array(y_train, dtype='int32'))
test_set_y = theano.shared(numpy.array(y_test, dtype='int32'))
valid_set_x = theano.shared(numpy.array(valid_set_x, dtype= 'float32'))
valid_set_y = theano.shared(numpy.array(y_val,dtype='int32'))
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
print("n_train_samples: %d" % n_train_batches)
print("n_valid_samples: %d" % n_valid_batches)
print("n_test_samples: %d" % n_test_batches)
print("n_batches:")
n_train_batches /= batch_size
n_valid_batches /= batch_size
n_test_batches /= batch_size
print("n_train_batches: %d" % n_train_batches)
print("n_valid_batches: %d" % n_valid_batches)
print("n_test_batches: %d" % n_test_batches)
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
variable_train_test = 0
# start-snippet-1
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of [int] labels
is_train = T.iscalar('is_train') # To differenciate between train and test
######################
# BUILD ACTUAL MODEL #
######################
print ('... building the model')
# Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
# (28, 28) is the size of MNIST images.
layer0_input = x.reshape((batch_size, 3, 128, 128))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to (28-5+1 , 28-5+1) = (24, 24)
# maxpooling reduces this further to (24/2, 24/2) = (12, 12)
# 4D output tensor is thus of shape(batch_size, nkerns[0], 12, 12)
layer0 = LeNetConvPoolLRNLayer(
rng,
input=layer0_input,
image_shape=(batch_size, 3, 128, 128),
filter_shape=(nkerns[0], 3, 11, 11),
stride=(1, 1),
lrn=True,
poolsize=(2, 2)
)
layer1 = LeNetConvPoolLRNLayer(
rng,
input=layer0.output,
# image_shape=(batch_size, nkerns[0], 27, 27),
image_shape=(batch_size, nkerns[0], 59, 59),
filter_shape=(nkerns[1], nkerns[0], 4, 4),
lrn=True,
poolsize=(2, 2)
)
layer2 = LeNetConvPoolLayer(
rng,
input=layer1.output,
image_shape=(batch_size, nkerns[1], 28, 28),
filter_shape=(nkerns[2], nkerns[1], 3, 3),
poolsize=(1, 1)
)
layer3 = LeNetConvPoolLayer(
rng,
input=layer2.output,
image_shape=(batch_size, nkerns[2], 26, 26),
filter_shape=(nkerns[3], nkerns[2], 3, 3),
poolsize=(1, 1)
)
layer4 = LeNetConvPoolLayer(
rng,
input=layer3.output,
image_shape=(batch_size, nkerns[3], 24, 24),
filter_shape=(nkerns[4], nkerns[3], 3, 3),
poolsize=(2, 2)
)
layer5_input = layer4.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer5 = Fully_Connected_Dropout(
rng,
input=layer5_input,
n_in=nkerns[4] * 11 * 11,
n_out=4096,
is_train=is_train
)
layer6 = Fully_Connected_Dropout(
rng,
input=layer5.output,
n_in=4096,
n_out=4096,
is_train=is_train
)
layer7 = Fully_Connected_Softmax(
rng,
input=layer6.output,
n_in=4096,
n_out=4096
)
# classify the values of the fully-connected sigmoidal layer
svm = SVC()
layer8 = LogisticRegression(input=layer7.output, n_in=4096, n_out=2)
salidas_capa8 = theano.function(
[index],
layer8.y_pred,
on_unused_input='ignore',
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size],
is_train: numpy.cast['int32'](1)
}
)
salidas_probabilidad = theano.function(
[index],
layer8.p_y_given_x,
on_unused_input='ignore',
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size],
is_train: numpy.cast['int32'](1)
}
)
# the cost we minimize during training is the NLL of the model
cost = layer8.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer8.errors(y),
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size],
is_train: numpy.cast['int32'](1)
}
)
validate_model = theano.function(
[index],
layer8.errors(y),
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size],
y: valid_set_y[index * batch_size: (index + 1) * batch_size],
is_train: numpy.cast['int32'](1)
}
)
# create a list of all model parameters to be fit by gradient descent
params = layer0.params + layer1.params + layer2.params + layer3.params + layer4.params + layer5.params + layer6.params + layer7.params + layer8.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [
(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)
]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + np.cast['int32'](1)) * batch_size],
y: train_set_y[index * batch_size: (index + np.cast['int32'](1)) * batch_size],
is_train: np.cast['int32'](0)
}
)
# end-snippet-1
###############
# TRAIN MODEL #
###############
print ('... training')
print (' ')
# early-stopping parameters
patience = 100000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
print("patience: %d" % patience)
print("patience_increase: %d" % patience_increase)
print("improvement threshold: %d" % improvement_threshold)
print("validation_frequency: %d" % validation_frequency)
print (' ')
# go through this many minibatche before checking the network
# on the validation set; in this case we check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
error_epoch = []
lista_coste = []
epoch = 0
done_looping = False
# save_file = open('/home/beaa/PycharmProjects/TFM/LearningTheano/salidas0.pkl', 'wb')
# save_file1 = open('/home/beaa/PycharmProjects/TFM/LearningTheano/wb0.pkl', 'wb')
# save_file2 = open('/home/beaa/PycharmProjects/TFM/LearningTheano/salidas1.pkl', 'wb')
# save_file3 = open('/home/beaa/PycharmProjects/TFM/LearningTheano/wb1.pkl', 'wb')
print ('n_train_batches', n_train_batches)
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print('training @ iter = ', iter)
cost_ij = train_model(minibatch_index)
lista_coste.append(cost_ij)
if (iter + 1) % validation_frequency == 0:
variable_train_test = 1
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i
in range(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
error_epoch.append(this_validation_loss * 100)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
# improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * \
improvement_threshold:
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
w0_test = layer0.W.get_value()
b0_test = layer0.b.get_value()
w1_test = layer1.W.get_value()
b1_test = layer1.b.get_value()
w2_test = layer2.W.get_value()
b2_test = layer2.b.get_value()
w3_test = layer3.W.get_value()
b3_test = layer3.b.get_value()
w4_test = layer4.W.get_value()
b4_test = layer4.b.get_value()
w5_test = layer5.W.get_value()
b5_test = layer5.b.get_value()
w6_test = layer6.W.get_value()
b6_test = layer6.b.get_value()
w7_test = layer7.W.get_value()
b7_test = layer7.b.get_value()
w8_test = layer8.W.get_value()
b8_test = layer8.b.get_value()
if patience <= iter:
done_looping = True
break
###############################
### TESTING MODEL ###
###############################
# Aqui se tiene que cargar la red
layer0.W.set_value(w0_test)
layer0.b.set_value(b0_test)
layer1.W.set_value(w1_test)
layer1.b.set_value(b1_test)
layer2.W.set_value(w2_test)
layer2.b.set_value(b2_test)
layer3.W.set_value(w3_test)
layer3.b.set_value(b3_test)
layer4.W.set_value(w4_test)
layer4.b.set_value(b4_test)
layer5.W.set_value(w5_test)
layer5.b.set_value(b5_test)
layer6.W.set_value(w6_test)
layer6.b.set_value(b6_test)
layer7.W.set_value(w7_test)
layer7.b.set_value(b7_test)
layer8.W.set_value(w8_test)
layer8.b.set_value(b8_test)
y_pred_junto = []
y_prob_junto = []
# test it on the test set
for i in range(n_test_batches):
test_losses = [test_model(i)]
y_pred_test = salidas_capa8(i)
y_probabilidad = salidas_probabilidad(i)
test_score = numpy.mean(test_losses)
for j in y_pred_test:
y_pred_junto.append(j)
for j in y_probabilidad:
y_prob_junto.append(j[0])
print((' test error of best model %f %%') % (test_score * 100.))
end_time = timeit.default_timer()
print('Optimization complete.')
print('Best validation score of %f %% obtained at iteration %i, '
'with test performance %f %%' %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
analize_results(y_test, y_pred_junto, y_prob_junto)
print(('The code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.)))
plt.clf()
plt.plot(error_epoch)
plt.ylabel('error')
plt.xlabel('epoch')
plt.savefig('error_frav.png')
plt.clf()
plt.plot(lista_coste)
plt.ylabel('cost_ij')
plt.xlabel('epoch')
plt.savefig('cost_frav.png')
# sys.stdout = orig_stdout
# f.close()
if __name__ == '__main__':
evaluate_lenet5()
def experiment(state, channel):
evaluate_lenet5(state.learning_rate, dataset=state.dataset)
