Repository: incubator-systemml Updated Branches: refs/heads/master fb82482b0 -> 4a941e44e
[SYSTEMML-1516] Improve output size calculation in conv2d & max_pool2d The calculation of the output sizes (`Hout`, `Wout`) for conv2d and max_pool2d layers rely on a flooring operation to only allow for valid operations. Previously, we simply relied on the `as.integer` call to implicitly perform this flooring operation during truncation of the double to an integer. While this gave the correct results, it was less readable than it should be, and relied on truncation behavior rather than explicit mathematical behavior. This commit simply adds an explicit `floor` operation to each calculation. Closes #459. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/4a941e44 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/4a941e44 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/4a941e44 Branch: refs/heads/master Commit: 4a941e44e008a9e65e6a5422d727c720980b0a3d Parents: fb82482 Author: Mike Dusenberry <[email protected]> Authored: Wed Apr 12 14:49:09 2017 -0700 Committer: Mike Dusenberry <[email protected]> Committed: Wed Apr 12 14:49:09 2017 -0700 ---------------------------------------------------------------------- scripts/staging/SystemML-NN/nn/layers/batch_norm1d.dml | 2 +- scripts/staging/SystemML-NN/nn/layers/batch_norm2d.dml | 2 +- scripts/staging/SystemML-NN/nn/layers/conv2d.dml | 4 ++-- .../staging/SystemML-NN/nn/layers/conv2d_builtin.dml | 4 ++-- scripts/staging/SystemML-NN/nn/layers/max_pool2d.dml | 4 ++-- .../SystemML-NN/nn/layers/max_pool2d_builtin.dml | 4 ++-- scripts/staging/SystemML-NN/nn/test/conv2d_simple.dml | 6 ++---- scripts/staging/SystemML-NN/nn/test/grad_check.dml | 12 ++++++------ .../staging/SystemML-NN/nn/test/max_pool2d_simple.dml | 4 ++-- scripts/staging/SystemML-NN/nn/test/test.dml | 9 +++++++-- scripts/staging/SystemML-NN/nn/util.dml | 8 ++++---- 11 files changed, 31 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/layers/batch_norm1d.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/layers/batch_norm1d.dml b/scripts/staging/SystemML-NN/nn/layers/batch_norm1d.dml index 9ecbd77..2ccffdb 100644 --- a/scripts/staging/SystemML-NN/nn/layers/batch_norm1d.dml +++ b/scripts/staging/SystemML-NN/nn/layers/batch_norm1d.dml @@ -81,7 +81,7 @@ forward = function(matrix[double] X, matrix[double] gamma, matrix[double] beta, */ N = nrow(X) - if(mode == 'train') { + if (mode == 'train') { # Compute feature-wise mean and variance mean = colMeans(X) # shape (1, D) # var = (1/N) * colSums((X-mean)^2) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/layers/batch_norm2d.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/layers/batch_norm2d.dml b/scripts/staging/SystemML-NN/nn/layers/batch_norm2d.dml index fb25b2c..49c6746 100644 --- a/scripts/staging/SystemML-NN/nn/layers/batch_norm2d.dml +++ b/scripts/staging/SystemML-NN/nn/layers/batch_norm2d.dml @@ -88,7 +88,7 @@ forward = function(matrix[double] X, matrix[double] gamma, matrix[double] beta, */ N = nrow(X) - if(mode == 'train') { + if (mode == 'train') { # Compute channel-wise mean and variance # Since we don't have tensors, we will compute the means and variances in a piece-wise fashion. # - mean of total group is mean of subgroup means http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/layers/conv2d.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/layers/conv2d.dml b/scripts/staging/SystemML-NN/nn/layers/conv2d.dml index 7aeec16..9d03568 100644 --- a/scripts/staging/SystemML-NN/nn/layers/conv2d.dml +++ b/scripts/staging/SystemML-NN/nn/layers/conv2d.dml @@ -67,8 +67,8 @@ forward = function(matrix[double] X, matrix[double] W, matrix[double] b, */ N = nrow(X) F = nrow(W) - Hout = as.integer((Hin + 2*padh - Hf)/strideh + 1) - Wout = as.integer((Win + 2*padw - Wf)/stridew + 1) + Hout = as.integer(floor((Hin + 2*padh - Hf)/strideh + 1)) + Wout = as.integer(floor((Win + 2*padw - Wf)/stridew + 1)) # Create output volume out = matrix(0, rows=N, cols=F*Hout*Wout) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/layers/conv2d_builtin.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/layers/conv2d_builtin.dml b/scripts/staging/SystemML-NN/nn/layers/conv2d_builtin.dml index e7771ba..bda7a9c 100644 --- a/scripts/staging/SystemML-NN/nn/layers/conv2d_builtin.dml +++ b/scripts/staging/SystemML-NN/nn/layers/conv2d_builtin.dml @@ -66,8 +66,8 @@ forward = function(matrix[double] X, matrix[double] W, matrix[double] b, */ N = nrow(X) F = nrow(W) - Hout = as.integer((Hin + 2*padh - Hf)/strideh + 1) - Wout = as.integer((Win + 2*padw - Wf)/stridew + 1) + Hout = as.integer(floor((Hin + 2*padh - Hf)/strideh + 1)) + Wout = as.integer(floor((Win + 2*padw - Wf)/stridew + 1)) # Convolution - built-in implementation out = conv2d(X, W, input_shape=[N,C,Hin,Win], filter_shape=[F,C,Hf,Wf], http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/layers/max_pool2d.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/layers/max_pool2d.dml b/scripts/staging/SystemML-NN/nn/layers/max_pool2d.dml index ef1499a..fba1a4c 100644 --- a/scripts/staging/SystemML-NN/nn/layers/max_pool2d.dml +++ b/scripts/staging/SystemML-NN/nn/layers/max_pool2d.dml @@ -57,8 +57,8 @@ forward = function(matrix[double] X, int C, int Hin, int Win, int Hf, int Wf, * - Wout: Output width. */ N = nrow(X) - Hout = as.integer((Hin + 2*padh - Hf)/strideh + 1) - Wout = as.integer((Win + 2*padw - Wf)/stridew + 1) + Hout = as.integer(floor((Hin + 2*padh - Hf)/strideh + 1)) + Wout = as.integer(floor((Win + 2*padw - Wf)/stridew + 1)) pad_value = -1/0 # in max pooling we pad with -infinity # Create output volume http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/layers/max_pool2d_builtin.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/layers/max_pool2d_builtin.dml b/scripts/staging/SystemML-NN/nn/layers/max_pool2d_builtin.dml index 65ba71f..880f818 100644 --- a/scripts/staging/SystemML-NN/nn/layers/max_pool2d_builtin.dml +++ b/scripts/staging/SystemML-NN/nn/layers/max_pool2d_builtin.dml @@ -56,8 +56,8 @@ forward = function(matrix[double] X, int C, int Hin, int Win, int Hf, int Wf, * - Wout: Output width. */ N = nrow(X) - Hout = as.integer((Hin-Hf)/strideh + 1) - Wout = as.integer((Win-Wf)/stridew + 1) + Hout = as.integer(floor((Hin + 2*padh - Hf)/strideh + 1)) + Wout = as.integer(floor((Win + 2*padw - Wf)/stridew + 1)) # Max pooling - built-in implementation out = max_pool(X, input_shape=[N,C,Hin,Win], pool_size=[Hf,Wf], http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/test/conv2d_simple.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/test/conv2d_simple.dml b/scripts/staging/SystemML-NN/nn/test/conv2d_simple.dml index 05f0f7d..9f126d0 100644 --- a/scripts/staging/SystemML-NN/nn/test/conv2d_simple.dml +++ b/scripts/staging/SystemML-NN/nn/test/conv2d_simple.dml @@ -57,8 +57,8 @@ forward = function(matrix[double] X, matrix[double] W, matrix[double] b, */ N = nrow(X) F = nrow(W) - Hout = as.integer((Hin + 2*padh - Hf)/strideh + 1) - Wout = as.integer((Win + 2*padw - Wf)/stridew + 1) + Hout = as.integer(floor((Hin + 2*padh - Hf)/strideh + 1)) + Wout = as.integer(floor((Win + 2*padw - Wf)/stridew + 1)) # Create output volume out = matrix(0, rows=N, cols=F*Hout*Wout) @@ -131,8 +131,6 @@ backward = function(matrix[double] dout, int Hout, int Wout, */ N = nrow(X) F = nrow(W) - Hout = as.integer((Hin + 2*padh - Hf)/strideh + 1) - Wout = as.integer((Win + 2*padw - Wf)/stridew + 1) # Create gradient volumes dX = matrix(0, rows=N, cols=C*Hin*Win) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/test/grad_check.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/test/grad_check.dml b/scripts/staging/SystemML-NN/nn/test/grad_check.dml index 516fe2a..f3bc9a7 100644 --- a/scripts/staging/SystemML-NN/nn/test/grad_check.dml +++ b/scripts/staging/SystemML-NN/nn/test/grad_check.dml @@ -1026,8 +1026,8 @@ max_pool2d = function() { for (pad in 0:1) { print(" - Grad checking w/ pad="+pad+".") - Hout = as.integer((Hin + 2*pad - Hf)/stride + 1) - Wout = as.integer((Win + 2*pad - Wf)/stride + 1) + Hout = as.integer(floor((Hin + 2*pad - Hf)/stride + 1)) + Wout = as.integer(floor((Win + 2*pad - Wf)/stride + 1)) y = rand(rows=N, cols=C*Hout*Wout) # Compute analytical gradients of loss wrt parameters @@ -1075,8 +1075,8 @@ max_pool2d_builtin = function() { for (pad in 0:1) { print(" - Grad checking w/ pad="+pad+".") - Hout = as.integer((Hin + 2 * pad - Hf) / stride + 1) - Wout = as.integer((Win + 2 * pad - Wf) / stride + 1) + Hout = as.integer(floor((Hin + 2 * pad - Hf) / stride + 1)) + Wout = as.integer(floor((Win + 2 * pad - Wf) / stride + 1)) y = rand(rows=N, cols=C*Hout*Wout) # Compute analytical gradients of loss wrt parameters @@ -1128,8 +1128,8 @@ max_pool2d_simple = function() { for (pad in 0:1) { print(" - Grad checking w/ pad="+pad+".") - Hout = as.integer((Hin + 2*pad - Hf)/stride + 1) - Wout = as.integer((Win + 2*pad - Wf)/stride + 1) + Hout = as.integer(floor((Hin + 2*pad - Hf)/stride + 1)) + Wout = as.integer(floor((Win + 2*pad - Wf)/stride + 1)) y = rand(rows=N, cols=C*Hout*Wout) # Compute analytical gradients of loss wrt parameters http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/test/max_pool2d_simple.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/test/max_pool2d_simple.dml b/scripts/staging/SystemML-NN/nn/test/max_pool2d_simple.dml index dee1a48..188bd6e 100644 --- a/scripts/staging/SystemML-NN/nn/test/max_pool2d_simple.dml +++ b/scripts/staging/SystemML-NN/nn/test/max_pool2d_simple.dml @@ -55,8 +55,8 @@ forward = function(matrix[double] X, int C, int Hin, int Win, int Hf, int Wf, * - Wout: Output width. */ N = nrow(X) - Hout = as.integer((Hin + 2*padh - Hf)/strideh + 1) - Wout = as.integer((Win + 2*padw - Wf)/stridew + 1) + Hout = as.integer(floor((Hin + 2*padh - Hf)/strideh + 1)) + Wout = as.integer(floor((Win + 2*padw - Wf)/stridew + 1)) # Create output volume out = matrix(0, rows=N, cols=C*Hout*Wout) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/test/test.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/test/test.dml b/scripts/staging/SystemML-NN/nn/test/test.dml index 3928fac..5a0390f 100644 --- a/scripts/staging/SystemML-NN/nn/test/test.dml +++ b/scripts/staging/SystemML-NN/nn/test/test.dml @@ -147,8 +147,8 @@ im2col = function() { Wf = 3 # filter width stride = 2 pad = (Hin * stride - Hin + Hf - stride) / 2 - Hout = as.integer((Hin + 2 * pad - Hf) / stride + 1) - Wout = as.integer((Win + 2 * pad - Wf) / stride + 1) + Hout = as.integer(floor((Hin + 2*pad - Hf)/stride + 1)) + Wout = as.integer(floor((Win + 2*pad - Wf)/stride + 1)) x = rand(rows=C, cols=Hin*Win) # pad @@ -157,6 +157,11 @@ im2col = function() { # im2col x_cols = util::im2col(x_pad, Hin+2*pad, Win+2*pad, Hf, Wf, stride, stride) + if (ncol(x_cols) != Hout*Wout) { + print("ERROR: im2col does not yield the correct output size: " + + ncol(x_cols)+" (actual) vs. "+Hout*Wout+" (correct).") + } + # col2im x_pad2 = util::col2im(x_cols, C, Hin+2*pad, Win+2*pad, Hf, Wf, stride, stride, "none") http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4a941e44/scripts/staging/SystemML-NN/nn/util.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/nn/util.dml b/scripts/staging/SystemML-NN/nn/util.dml index 62a90f2..3a73f08 100644 --- a/scripts/staging/SystemML-NN/nn/util.dml +++ b/scripts/staging/SystemML-NN/nn/util.dml @@ -64,8 +64,8 @@ im2col = function(matrix[double] img, int Hin, int Win, int Hf, int Wf, int stri * out into columns, of shape (C*Hf*Wf, Hout*Wout). */ C = nrow(img) - Hout = as.integer((Hin-Hf)/strideh + 1) - Wout = as.integer((Win-Wf)/stridew + 1) + Hout = as.integer(floor((Hin-Hf)/strideh + 1)) + Wout = as.integer(floor((Win-Wf)/stridew + 1)) # Note: We start with `img_cols` transposed to allow for row-major # left-indexing inside the loop, which is more performant. @@ -119,8 +119,8 @@ col2im = function(matrix[double] img_cols, int C, int Hin, int Win, int Hf, int * Outputs: * - img: Input image, of shape (C, Hin*Win). */ - Hout = as.integer((Hin-Hf)/strideh + 1) - Wout = as.integer((Win-Wf)/stridew + 1) + Hout = as.integer(floor((Hin-Hf)/strideh + 1)) + Wout = as.integer(floor((Win-Wf)/stridew + 1)) img = matrix(0, rows=C, cols=Hin*Win) # zeros for (hout in 1:Hout) { # all output rows
