Repository: incubator-systemml Updated Branches: refs/heads/master 6d95c9f5e -> b14d55bed
Adding a smoothing term to the cross-entropy loss function for numerical stability in situations in which predictions are exactly equal to 0. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/ba60e73e Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/ba60e73e Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/ba60e73e Branch: refs/heads/master Commit: ba60e73eb1ba097eeeb003f8f297745f460b59ff Parents: 6d95c9f Author: Mike Dusenberry <[email protected]> Authored: Fri May 27 17:44:30 2016 -0700 Committer: Mike Dusenberry <[email protected]> Committed: Fri May 27 17:44:30 2016 -0700 ---------------------------------------------------------------------- .../nn/layers/cross_entropy_loss.dml | 6 ++-- scripts/staging/SystemML-NN/nn/test/test.dml | 32 ++++++++++++++++++-- scripts/staging/SystemML-NN/nn/test/tests.dml | 1 + 3 files changed, 35 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ba60e73e/scripts/staging/SystemML-NN/nn/layers/cross_entropy_loss.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/layers/cross_entropy_loss.dml b/scripts/staging/SystemML-NN/nn/layers/cross_entropy_loss.dml index 6b9840f..306ea96 100644 --- a/scripts/staging/SystemML-NN/nn/layers/cross_entropy_loss.dml +++ b/scripts/staging/SystemML-NN/nn/layers/cross_entropy_loss.dml @@ -41,7 +41,8 @@ forward = function(matrix[double] pred, matrix[double] y) * - loss: Scalar loss, of shape (1). */ N = nrow(y) - losses = rowSums(-y * log(pred)) + eps = 1e-10 # numerical stability to avoid log(0) + losses = rowSums(-y * log(pred+eps)) loss = sum(losses) / N } @@ -60,6 +61,7 @@ backward = function(matrix[double] pred, matrix[double] y) * - dpred: Gradient wrt pred, of shape (N, K). */ N = nrow(y) - dpred = (1/N) * -y * (1/pred) + eps = 1e-10 # numerical stability to avoid divide-by-zero + dpred = (1/N) * -y * (1/(pred+eps)) } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ba60e73e/scripts/staging/SystemML-NN/nn/test/test.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/test/test.dml b/scripts/staging/SystemML-NN/nn/test/test.dml index 58ee3e1..1ecff68 100644 --- a/scripts/staging/SystemML-NN/nn/test/test.dml +++ b/scripts/staging/SystemML-NN/nn/test/test.dml @@ -24,6 +24,7 @@ */ source("nn/layers/conv.dml") as conv source("nn/layers/conv_builtin.dml") as conv_builtin +source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss source("nn/layers/max_pool.dml") as max_pool source("nn/layers/max_pool_builtin.dml") as max_pool_builtin source("nn/test/conv_simple.dml") as conv_simple @@ -68,6 +69,31 @@ conv = function() { } } +cross_entropy_loss = function() { + /* + * Test for the `cross-entropy` loss function. + * + * Here we make sure that the cross-entropy loss function does + * not propagate `infinity` values in the case that a prediction is +` * exactly equal to 0. + */ + print("Testing the cross-entropy loss function with zero-valued predictions.") + + # Generate data + N = 3 # num examples + K = 10 # num targets + pred = matrix(0, rows=N, cols=K) + y = rand(rows=N, cols=K, min=0, max=1, pdf="uniform") + y = y / rowSums(y) # normalized probs + + loss = cross_entropy_loss::forward(pred, y) + + inf = 1/0 + if (loss == inf) { + print("ERROR: The cross-entropy loss function ouptuts infinity for all-zero predictions.") + } +} + im2col = function() { /* * Test for the `im2col` and `col2im` functions. @@ -97,8 +123,9 @@ im2col = function() { # Equivalency check equivalent = util::all_equal(x_pad, x_pad2) - if (!equivalent) + if (!equivalent) { print("ERROR: im2col and then col2im does not yield the original image.") + } } padding = function() { @@ -135,8 +162,9 @@ padding = function() { # Equivalency check equivalent = util::all_equal(x, x1) - if (!equivalent) + if (!equivalent) { print("ERROR: Padding and then unpadding does not yield the original image.") + } } max_pool = function() { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ba60e73e/scripts/staging/SystemML-NN/nn/test/tests.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/test/tests.dml b/scripts/staging/SystemML-NN/nn/test/tests.dml index 1b91967..cac56c2 100644 --- a/scripts/staging/SystemML-NN/nn/test/tests.dml +++ b/scripts/staging/SystemML-NN/nn/test/tests.dml @@ -63,6 +63,7 @@ print("---") tmp = test::im2col() tmp = test::padding() tmp = test::conv() +tmp = test::cross_entropy_loss() tmp = test::max_pool() print("---")
