This is an automated email from the ASF dual-hosted git repository.
sebwrede pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 6b25434106 [MINOR] Edit U-Net To Use One-Hot Encoded Labels
6b25434106 is described below
commit 6b25434106e4793a15bdc16097fec51e11f7ab44
Author: sebwrede <[email protected]>
AuthorDate: Thu Feb 2 12:32:26 2023 +0100
[MINOR] Edit U-Net To Use One-Hot Encoded Labels
Closes #1780.
---
scripts/nn/examples/u-net.dml | 31 +++++++++++++---------
.../paramserv/EncryptedFederatedParamservTest.java | 2 +-
.../paramserv/FederatedParamservTest.java | 2 +-
.../paramserv/EncryptedFederatedParamservTest.dml | 11 +++++---
.../federated/paramserv/FederatedParamservTest.dml | 11 +++++---
5 files changed, 35 insertions(+), 22 deletions(-)
diff --git a/scripts/nn/examples/u-net.dml b/scripts/nn/examples/u-net.dml
index 9110b85ab7..5ca255a6d2 100644
--- a/scripts/nn/examples/u-net.dml
+++ b/scripts/nn/examples/u-net.dml
@@ -31,7 +31,7 @@ source("scripts/nn/layers/dropout.dml") as dropout
source("scripts/nn/layers/l2_reg.dml") as l2_reg
source("scripts/nn/layers/max_pool2d_builtin.dml") as max_pool2d
source("scripts/nn/layers/relu.dml") as relu
-source("scripts/nn/layers/softmax.dml") as softmax
+source("scripts/nn/layers/softmax2d.dml") as softmax2d
source("scripts/nn/optim/sgd_momentum.dml") as sgd_momentum
source("scripts/nn/layers/dropout.dml") as dropout
source("scripts/utils/image_utils.dml") as img_utils
@@ -210,6 +210,8 @@ extrapolate_channel = function(matrix[double] img, int Hin,
int Win, int pad_siz
* - scheme: Parameter server training scheme
* - learning_rate: The learning rate for the SGD with momentum
* - seed: Seed for the initialization of the convolution weights. Default is
-1 meaning that the seeds are random.
+ * - M: Size of the segmentation map (C*(Hin-pad)*(Win-pad))
+ * - K: Number of output categories (for each element of segmentation map)
* - he: Homomorphic encryption activated (boolean)
* - F1: Number of filters of the top layer of the U-Net model. Default is 64.
*
@@ -220,10 +222,9 @@ train_paramserv = function(matrix[double] X,
matrix[double] y,
matrix[double] X_val, matrix[double] y_val,
int C, int Hin, int Win, int epochs, int workers,
string utype, string freq, int batch_size, string scheme,
double learning_rate,
- int seed = -1, boolean he = FALSE, int F1 = 64)
+ int seed = -1, int M, int K, boolean he = FALSE, int F1 = 64)
return (list[unknown] model_trained) {
N = nrow(X) # Number of inputs
- K = ncol(y) # Number of target classes
# Define model network constants
Hf = 3 # convolution filter height
@@ -285,7 +286,7 @@ train_paramserv = function(matrix[double] X, matrix[double]
y,
[W21, b21] = conv2d::init(F1, F2, Hf, Wf, seed =
as.integer(as.scalar(lseed[21])))
[W22, b22] = conv2d::init(F1, F1, Hf, Wf, seed =
as.integer(as.scalar(lseed[22])))
# Segmentation map
- [W23, b23] = conv2d::init(C, F1, 1, 1, seed =
as.integer(as.scalar(lseed[23])))
+ [W23, b23] = conv2d::init(K*C, F1, 1, 1, seed =
as.integer(as.scalar(lseed[23])))
# Initialize SGD with momentum
vW1 = sgd_momentum::init(W1); vb1 = sgd_momentum::init(b1)
@@ -328,7 +329,7 @@ train_paramserv = function(matrix[double] X, matrix[double]
y,
# Create the hyper parameter list
params = list(
- learning_rate=learning_rate, mu=mu, decay=decay, C=C, Hin=Hin, Win=Win,
Hf=Hf, Wf=Wf,
+ learning_rate=learning_rate, mu=mu, decay=decay, M=M, K=K, C=C, Hin=Hin,
Win=Win, Hf=Hf, Wf=Wf,
conv_stride=conv_stride, pool_stride=pool_stride, pool_HWf=pool_HWf,
conv_t_HWf=conv_t_HWf, conv_t_stride=conv_t_stride,
pad=pad, lambda=lambda, F1=F1, F2=F2, F3=F3, F4=F4, F5=F5,
dropProb=dropProb, dropSeed=dropSeed)
@@ -351,13 +352,14 @@ train_paramserv = function(matrix[double] X,
matrix[double] y,
* - Win: Input width
* - batch_size: Batch size
* - model: List of weights of the model (23 weights, 23 biases)
-* - K: Size of the segmentation map (C*(Hin-pad)*(Win-pad))
+* - M: Size of the segmentation map (C*(Hin-pad)*(Win-pad))
+* - K: Number of output categories (for each element of segmentation map)
* - F1: Number of filters of the top layer of the U-Net model. Default is 64.
*
* Output:
* - probs: Segmentation map probabilities generated by the forward pass of
the U-Net model
*/
-predict = function(matrix[double] X, int C, int Hin, int Win, int batch_size,
list[unknown] model, int K, int F1 = 64)
+predict = function(matrix[double] X, int C, int Hin, int Win, int batch_size,
list[unknown] model, int M, int K, int F1 = 64)
return (matrix[double] probs) {
W1 = as.matrix(model[1])
W2 = as.matrix(model[2])
@@ -423,7 +425,7 @@ predict = function(matrix[double] X, int C, int Hin, int
Win, int batch_size, li
dropSeed = -1
# Compute predictions over mini-batches
- probs = matrix(0, rows=N, cols=K)
+ probs = matrix(0, rows=N, cols=K*M)
iters = ceil(N / batch_size)
for(i in 1:iters, check=0) {
# Get next batch
@@ -490,7 +492,7 @@ predict = function(matrix[double] X, int C, int Hin, int
Win, int batch_size, li
[outc23, Houtc23, Woutc23] = conv2d::forward(outr18, W23, b23, F1,
Houtc22, Woutc22, 1, 1, conv_stride, conv_stride, pad, pad)
# Store predictions
- probs[beg:end,] = softmax::forward(outc23)
+ probs[beg:end,] = softmax2d::forward(outc23, K)
}
}
@@ -515,6 +517,8 @@ predict = function(matrix[double] X, int C, int Hin, int
Win, int batch_size, li
* - (scalar[integer]) F1, F2, F3, F4, F5: Number of filters
of the convolutions in the five layers
* - (scalar[double]) dropProb: Dropout probability
* - (scalar[integer]) dropSeed: Dropout seed
+* - (scalar[integer]) M: Size of the segmentation map
(C*(Hin-pad)*(Win-pad))
+* - (scalar[integer]) K: Number of output categories (for each
element of segmentation map)
* - features: Features of size C*Hin*Win. The features need to be padded with
mirrored data.
* The input feature size should result in an output size of the
U-Net equal to the label size.
* See extrapolate function for how to pad the features by
extrapolating.
@@ -546,6 +550,8 @@ gradients = function(list[unknown] model,
F5 = as.integer(as.scalar(hyperparams["F5"]))
dropProb = as.double(as.scalar(hyperparams["dropProb"]))
dropSeed = as.integer(as.scalar(hyperparams["dropSeed"]))
+ M = as.integer(as.scalar(hyperparams["M"]))
+ K = as.integer(as.scalar(hyperparams["K"]))
W1 = as.matrix(model[1])
W2 = as.matrix(model[2])
W3 = as.matrix(model[3])
@@ -650,7 +656,7 @@ gradients = function(list[unknown] model,
# This last conv2d needs to create the segmentation map (1x1 filter):
[outc23, Houtc23, Woutc23] = conv2d::forward(outr18, W23, b23, F1,
Houtc22, Woutc22, 1, 1, conv_stride, conv_stride, pad, pad)
- probs = softmax::forward(outc23)
+ probs = softmax2d::forward(outc23, K)
# Compute loss & accuracy for training data
loss = cross_entropy_loss::forward(probs, labels)
@@ -661,7 +667,7 @@ gradients = function(list[unknown] model,
## loss
dprobs = cross_entropy_loss::backward(probs, labels)
- doutc23 = softmax::backward(dprobs, outc23)
+ doutc23 = softmax2d::backward(dprobs, outc23, K)
# Up-Convolution
# conv2d parameters: (previous_gradient, output height, output width,
input to original layer, layer weight, layer bias, layer input channel number,
input height, input width, filter height, filter width, stride height, stride
width, pad height, pad width)
@@ -1041,7 +1047,8 @@ validate = function(matrix[double] val_features,
matrix[double] val_labels,
C = as.integer(as.scalar(hyperparams["C"]))
Hin = as.integer(as.scalar(hyperparams["Hin"]))
Win = as.integer(as.scalar(hyperparams["Win"]))
+ M = as.integer(as.scalar(hyperparams["M"]))
K = as.integer(as.scalar(hyperparams["K"]))
- predictions = predict(val_features, C, Hin, Win, batch_size, model, K, F1)
+ predictions = predict(val_features, C, Hin, Win, batch_size, model, M, K, F1)
[loss, accuracy] = eval(predictions, val_labels)
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
index 4ef5d0b45a..f2fe84fdfc 100644
---
a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
@@ -147,7 +147,7 @@ public class EncryptedFederatedParamservTest extends
AutomatedTestBase {
int C = 1, Hin = 28, Win = 28;
int numLabels = 10;
if (Objects.equals(_networkType, "UNet")){
- C = 3; Hin = 340; Win = 340;
+ C = 3; Hin = 196; Win = 196;
numLabels = C * Hin * Win;
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/FederatedParamservTest.java
b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/FederatedParamservTest.java
index 894c7ca548..0bcc117f76 100644
---
a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/FederatedParamservTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/FederatedParamservTest.java
@@ -141,7 +141,7 @@ public class FederatedParamservTest extends
AutomatedTestBase {
int C = 1, Hin = 28, Win = 28;
int numLabels = 10;
if (_networkType.equals("UNet")){
- C = 3; Hin = 340; Win = 340;
+ C = 3; Hin = 196; Win = 196;
numLabels = C * Hin * Win;
}
diff --git
a/src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml
b/src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml
index 7b00f52eff..e321534b1b 100644
---
a/src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml
+++
b/src/test/scripts/functions/federated/paramserv/EncryptedFederatedParamservTest.dml
@@ -67,12 +67,15 @@ else if($network_type == "UNet") {
x_hw = $hin + 184 # Padded input height and width
x_val = matrix(0, rows=numRows, cols=$channels*x_hw*x_hw)
- y_val = matrix(0, rows=numRows, cols=numFeatures)
+ y_val = matrix(0, rows=numRows, cols=numFeatures*2)
+ K = 2
+ M = numFeatures
+ labels_one_hot = cbind((labels - 1) * -1, labels) # (N,KCHW) encoded labels
- model = UNet::train_paramserv(features, labels, x_val, y_val, $channels,
x_hw, x_hw, $epochs, 0, $utype, $freq, $batch_size, $scheme, $eta, $seed, TRUE,
F1)
+ model = UNet::train_paramserv(features, labels_one_hot, x_val, y_val,
$channels, x_hw, x_hw, $epochs, 0, $utype, $freq, $batch_size, $scheme, $eta,
$seed, M, K, TRUE, F1)
print("Test results:")
- hyperparams = list(learning_rate=$eta, C=$channels, Hin=x_hw, Win=x_hw,
K=numFeatures)
- [loss_test, accuracy_test] = UNet::validate(matrix(0, rows=numRows,
cols=$channels*x_hw*x_hw), matrix(0, rows=numRows, cols=numFeatures), model,
hyperparams, F1, $batch_size)
+ hyperparams = list(learning_rate=$eta, C=$channels, Hin=x_hw, Win=x_hw,
M=numFeatures, K=K)
+ [loss_test, accuracy_test] = UNet::validate(matrix(0, rows=numRows,
cols=$channels*x_hw*x_hw), matrix(0, rows=numRows, cols=numFeatures*2), model,
hyperparams, F1, $batch_size)
print("[+] test loss: " + loss_test + ", test accuracy: " + accuracy_test
+ "\n")
}
else {
diff --git
a/src/test/scripts/functions/federated/paramserv/FederatedParamservTest.dml
b/src/test/scripts/functions/federated/paramserv/FederatedParamservTest.dml
index 009d5d3dec..d88af80ca0 100644
--- a/src/test/scripts/functions/federated/paramserv/FederatedParamservTest.dml
+++ b/src/test/scripts/functions/federated/paramserv/FederatedParamservTest.dml
@@ -47,12 +47,15 @@ else if($network_type == "UNet"){
x_hw = $hin + 184 # Padded input height and width
x_val = matrix(0, rows=numRows, cols=$channels*x_hw*x_hw)
- y_val = matrix(0, rows=numRows, cols=numFeatures)
+ y_val = matrix(0, rows=numRows, cols=numFeatures*2)
+ K = 2
+ M = numFeatures
+ labels_one_hot = cbind((labels - 1) * -1, labels) # (N,KCHW) encoded labels
- model = UNet::train_paramserv(features, labels, x_val, y_val, $channels,
x_hw, x_hw, $epochs, 2, $utype, $freq, $batch_size, $scheme, $eta, $seed,
FALSE, F1)
+ model = UNet::train_paramserv(features, labels_one_hot, x_val, y_val,
$channels, x_hw, x_hw, $epochs, 0, $utype, $freq, $batch_size, $scheme, $eta,
$seed, M, K, FALSE, F1)
print("Test results:")
- hyperparams = list(learning_rate=$eta, C=$channels, Hin=x_hw, Win=x_hw,
K=numFeatures)
- [loss_test, accuracy_test] = UNet::validate(matrix(0, rows=numRows,
cols=$channels*x_hw*x_hw), matrix(0, rows=numRows, cols=numFeatures), model,
hyperparams, F1, numRows)
+ hyperparams = list(learning_rate=$eta, C=$channels, Hin=x_hw, Win=x_hw,
M=numFeatures, K=K)
+ [loss_test, accuracy_test] = UNet::validate(matrix(0, rows=numRows,
cols=$channels*x_hw*x_hw), matrix(0, rows=numRows, cols=numFeatures*2), model,
hyperparams, F1, $batch_size)
print("[+] test loss: " + loss_test + ", test accuracy: " + accuracy_test +
"\n")
}
else {
