SINGA-316 Add SigmoidCrossEntropy Apply CrossEntropy loss for prediction (probability) generated from Sigmoid
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/be093f19 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/be093f19 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/be093f19 Branch: refs/heads/master Commit: be093f1916da3e0f7bf55f0780b5616565d22f53 Parents: ea078dc Author: wangwei <[email protected]> Authored: Tue May 23 14:27:42 2017 +0800 Committer: Wei Wang <[email protected]> Committed: Wed May 24 16:41:01 2017 +0800 ---------------------------------------------------------------------- python/singa/loss.py | 54 ++++++++++++++++++++++++++++++++++++++++++ python/singa/net.py | 2 +- test/python/test_loss.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/be093f19/python/singa/loss.py ---------------------------------------------------------------------- diff --git a/python/singa/loss.py b/python/singa/loss.py index 60835fc..7fd3d77 100644 --- a/python/singa/loss.py +++ b/python/singa/loss.py @@ -39,6 +39,7 @@ Example usage:: from . import singa_wrap as singa from proto import model_pb2 import tensor +import numpy as np class Loss(object): @@ -116,6 +117,59 @@ class SoftmaxCrossEntropy(Loss): self.swig_loss = singa.SoftmaxCrossEntropy() +class SigmoidCrossEntropy(Loss): + '''This loss evaluates the cross-entropy loss between the prediction and the + truth values with the prediction probability generated from Sigmoid. + ''' + def __init__(self, epsilon=1e-8): + super(SigmoidCrossEntropy, self).__init__() + self.truth = None + self.prob = None + self.epsilon = epsilon # to avoid log(x) with x being too small + + def forward(self, flag, x, y): + '''loss is -yi * log pi - (1-yi) log (1-pi), where pi=sigmoid(xi) + + Args: + flag (bool): true for training; false for evaluation + x (Tensor): the prediction Tensor + y (Tensor): the truth Tensor, a binary array value per sample + + Returns: + a Tensor with one error value per sample + ''' + p = tensor.sigmoid(x) + if flag: + self.truth = y + self.prob = p + np = 1 - p + p += (p < self.epsilon) * self.epsilon + np += (np < self.epsilon) * self.epsilon + l = (y-1) * tensor.log(np) - y * tensor.log(p) + # TODO(wangwei): add unary operation -Tensor + return tensor.average(l, axis=1) + + def backward(self): + ''' Compute the gradient of loss w.r.t to x. + + Returns: + dx = pi - yi. + ''' + assert self.truth is not None, 'must call forward in a prior' + dx = self.prob - self.truth + self.truth = None + return dx + + def evaluate(self, flag, x, y): + '''Compuate the averaged error. + + Returns: + a float value as the averaged error + ''' + l = self.forward(False, x, y) + return l.l1() + + class SquaredError(Loss): '''This loss evaluates the squared error between the prediction and the truth values. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/be093f19/python/singa/net.py ---------------------------------------------------------------------- diff --git a/python/singa/net.py b/python/singa/net.py index 96a9c79..e4c1de9 100644 --- a/python/singa/net.py +++ b/python/singa/net.py @@ -399,7 +399,7 @@ class FeedForwardNet(object): if cur.name in output: ret[cur.name] = outs # ret.update(output_of_layer) - yield (cur.param_names(), cur.param_values(), pgrads) + yield (cur.param_names(), cur.param_values(), pgrads, ret) def save(self, f, buffer_size=10, use_pickle=False): '''Save model parameters using io/snapshot. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/be093f19/test/python/test_loss.py ---------------------------------------------------------------------- diff --git a/test/python/test_loss.py b/test/python/test_loss.py new file mode 100644 index 0000000..78356f2 --- /dev/null +++ b/test/python/test_loss.py @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest + +import numpy as np + +from singa import loss +from singa import tensor + + +class TestLoss(unittest.TestCase): + def setUp(self): + self.x_np = np.asarray([[0.9, 0.2, 0.1], + [0.1, 0.4, 0.5], + [0.2, 0.4, 0.4]], + dtype=np.float32) + + self.y_np = np.asarray([[1, 0, 1], + [0, 1, 1], + [1, 0, 0]], + dtype=np.float32) + + self.x = tensor.from_numpy(self.x_np) + self.y = tensor.from_numpy(self.y_np) + + def test_sigmoid_cross_entropy(self): + sig = loss.SigmoidCrossEntropy() + l1 = sig.forward(True, self.x, self.y) + sig.backward() + l2 = sig.evaluate(True, self.x, self.y) + + p = 1.0 / (1 + np.exp(-self.x_np)) + l = - (self.y_np * np.log(p) + (1-self.y_np) * np.log(1-p)) + self.assertAlmostEqual(l1.l1(), l2) + self.assertAlmostEqual(l1.l1(), np.average(l)) + + +if __name__ == '__main__': + unittest.main()
