IGNITE-7174: Local MLP this closes #3246
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e4f19215 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e4f19215 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e4f19215 Branch: refs/heads/ignite-zk Commit: e4f19215de8d20311e16a6b12d0ef68d711b3462 Parents: 661ada6 Author: artemmalykh <[email protected]> Authored: Fri Dec 22 18:07:44 2017 +0300 Committer: Yury Babak <[email protected]> Committed: Fri Dec 22 18:07:44 2017 +0300 ---------------------------------------------------------------------- .../KNNClassificationExample.java | 2 +- .../ml/knn/regression/KNNRegressionExample.java | 2 +- .../DistributedRegressionModelExample.java | 2 +- .../main/java/org/apache/ignite/ml/Model.java | 10 +- .../ignite/ml/clustering/FuzzyCMeansModel.java | 2 +- .../ignite/ml/clustering/KMeansModel.java | 2 +- .../apache/ignite/ml/estimators/Estimators.java | 4 +- .../apache/ignite/ml/knn/models/KNNModel.java | 2 +- .../regression/KNNMultipleLinearRegression.java | 2 +- .../org/apache/ignite/ml/math/VectorUtils.java | 96 +++ ...iteDifferentiableDoubleToDoubleFunction.java | 31 + ...iteDifferentiableVectorToDoubleFunction.java | 33 ++ .../ml/math/functions/IgniteTriConsumer.java | 40 ++ .../ml/math/impls/matrix/AbstractMatrix.java | 5 + .../apache/ignite/ml/math/util/MatrixUtil.java | 88 +++ .../org/apache/ignite/ml/nn/Activators.java | 61 ++ .../ignite/ml/nn/LocalBatchTrainerInput.java | 41 ++ .../org/apache/ignite/ml/nn/LossFunctions.java | 47 ++ .../java/org/apache/ignite/ml/nn/MLPLayer.java | 47 ++ .../java/org/apache/ignite/ml/nn/MLPState.java | 73 +++ .../ignite/ml/nn/MultilayerPerceptron.java | 565 ++++++++++++++++++ .../ignite/ml/nn/ReplicatedVectorMatrix.java | 583 +++++++++++++++++++ .../ml/nn/architecture/LayerArchitecture.java | 45 ++ .../ml/nn/architecture/MLPArchitecture.java | 147 +++++ .../TransformationLayerArchitecture.java | 68 +++ .../ignite/ml/nn/architecture/package-info.java | 22 + .../ml/nn/initializers/MLPInitializer.java | 40 ++ .../ml/nn/initializers/RandomInitializer.java | 51 ++ .../ignite/ml/nn/initializers/package-info.java | 22 + .../org/apache/ignite/ml/nn/package-info.java | 22 + .../ml/nn/trainers/local/LocalBatchTrainer.java | 180 ++++++ .../nn/trainers/local/MLPLocalBatchTrainer.java | 78 +++ .../ml/nn/trainers/local/package-info.java | 22 + .../ignite/ml/nn/trainers/package-info.java | 22 + .../ml/nn/updaters/BaseSmoothParametrized.java | 64 ++ .../ignite/ml/nn/updaters/NesterovUpdater.java | 76 +++ .../ml/nn/updaters/NesterovUpdaterParams.java | 67 +++ .../ignite/ml/nn/updaters/ParameterUpdater.java | 51 ++ .../ignite/ml/nn/updaters/RPropUpdater.java | 148 +++++ .../ml/nn/updaters/RPropUpdaterParams.java | 134 +++++ .../ignite/ml/nn/updaters/SimpleGDParams.java | 65 +++ .../ignite/ml/nn/updaters/SimpleGDUpdater.java | 60 ++ .../ml/nn/updaters/SmoothParametrized.java | 24 + .../ignite/ml/nn/updaters/UpdaterParams.java | 32 + .../ignite/ml/nn/updaters/package-info.java | 22 + .../OLSMultipleLinearRegressionModel.java | 2 +- .../ml/trees/models/DecisionTreeModel.java | 2 +- .../java/org/apache/ignite/ml/util/Utils.java | 27 + .../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +- .../ignite/ml/knn/KNNClassificationTest.java | 14 +- .../ml/knn/KNNMultipleLinearRegressionTest.java | 16 +- .../ignite/ml/nn/MLPConstInitializer.java | 67 +++ .../ignite/ml/nn/MLPLocalTrainerTest.java | 97 +++ .../java/org/apache/ignite/ml/nn/MLPTest.java | 207 +++++++ .../org/apache/ignite/ml/nn/MLPTestSuite.java | 33 ++ .../ml/nn/SimpleMLPLocalBatchTrainerInput.java | 95 +++ .../apache/ignite/ml/nn/performance/Mnist.java | 140 +++++ .../OLSMultipleLinearRegressionModelTest.java | 2 +- .../ml/trees/ColumnDecisionTreeTrainerTest.java | 4 +- .../ColumnDecisionTreeTrainerBenchmark.java | 4 +- .../trees/columntrees.manualrun.properties | 8 +- .../yardstick/ml/trees/SplitDataGenerator.java | 2 +- 62 files changed, 3880 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java index a92e9af..fb7eebd 100644 --- a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java +++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java @@ -89,7 +89,7 @@ public class KNNClassificationExample { // Save predicted classes to test dataset for (int i = 0; i < test.rowSize(); i++) { - double predictedCls = knnMdl.predict(test.getRow(i).features()); + double predictedCls = knnMdl.apply(test.getRow(i).features()); test.setLabel(i, predictedCls); } http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java index f4a9e1c..6ed0dd6 100644 --- a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java +++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java @@ -94,7 +94,7 @@ public class KNNRegressionExample { // Save predicted classes to test dataset for (int i = 0; i < test.rowSize(); i++) { - double predictedCls = knnMdl.predict(test.getRow(i).features()); + double predictedCls = knnMdl.apply(test.getRow(i).features()); test.setLabel(i, predictedCls); } http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java ---------------------------------------------------------------------- diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java index ab1b17d..38de97e 100644 --- a/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java +++ b/examples/src/main/ml/org/apache/ignite/examples/ml/regression/DistributedRegressionModelExample.java @@ -123,7 +123,7 @@ public class DistributedRegressionModelExample { Tracer.showAscii(val); System.out.println(">>> Trained model prediction results:"); - Tracer.showAscii(mdl.predict(val)); + Tracer.showAscii(mdl.apply(val)); }); igniteThread.start(); http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/Model.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java index 05ce774..f0e6cc6 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java @@ -17,15 +17,11 @@ package org.apache.ignite.ml; -import java.io.Serializable; import java.util.function.BiFunction; +import org.apache.ignite.ml.math.functions.IgniteFunction; /** Basic interface for all models. */ -@FunctionalInterface -public interface Model<T, V> extends Serializable { - /** Predict a result for value. */ - V predict(T val); - +public interface Model<T, V> extends IgniteFunction<T, V> { /** * Combines this model with other model via specified combiner * @@ -34,6 +30,6 @@ public interface Model<T, V> extends Serializable { * @return Combination of models. */ default <X, W> Model<T, X> combine(Model<T, W> other, BiFunction<V, W, X> combiner) { - return v -> combiner.apply(predict(v), other.predict(v)); + return v -> combiner.apply(apply(v), other.apply(v)); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java index 83fbf1f..70009cb 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java @@ -64,7 +64,7 @@ public class FuzzyCMeansModel implements ClusterizationModel<Vector, Integer>, E * @param val Vector. * @return Index of the closest center or -1 if it can't be found. */ - @Override public Integer predict(Vector val) { + @Override public Integer apply(Vector val) { int idx = -1; double minDistance = Double.POSITIVE_INFINITY; http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java index 381f976..e1d783f 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java @@ -65,7 +65,7 @@ public class KMeansModel implements ClusterizationModel<Vector, Integer>, Export * * @param vec Vector. */ - public Integer predict(Vector vec) { + public Integer apply(Vector vec) { int res = -1; double minDist = Double.POSITIVE_INFINITY; http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java b/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java index 13331d1..b2731ff 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java @@ -29,7 +29,7 @@ public class Estimators { /** Simple implementation of mean squared error estimator. */ public static <T, V> IgniteTriFunction<Model<T, V>, Stream<IgniteBiTuple<T, V>>, Function<V, Double>, Double> MSE() { return (model, stream, f) -> stream.mapToDouble(dp -> { - double diff = f.apply(dp.get2()) - f.apply(model.predict(dp.get1())); + double diff = f.apply(dp.get2()) - f.apply(model.apply(dp.get1())); return diff * diff; }).average().orElse(0); } @@ -41,7 +41,7 @@ public class Estimators { long cnt = stream. peek((ib) -> total.incrementAndGet()). - filter(dp -> !model.predict(dp.get1()).equals(dp.get2())). + filter(dp -> !model.apply(dp.get1()).equals(dp.get2())). count(); return (double)cnt / total.get(); http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java index 44955c8..d3dff8c 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModel.java @@ -75,7 +75,7 @@ public class KNNModel implements Model<Vector, Double>, Exportable<KNNModelForma } /** {@inheritDoc} */ - @Override public Double predict(Vector v) { + @Override public Double apply(Vector v) { LabeledVector[] neighbors = findKNearestNeighbors(v, true); return classify(neighbors, v, stgy); http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java index 90392a3..1796eeb 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java @@ -43,7 +43,7 @@ public class KNNMultipleLinearRegression extends KNNModel { } /** {@inheritDoc} */ - @Override public Double predict(Vector v) { + @Override public Double apply(Vector v) { LabeledVector[] neighbors = findKNearestNeighbors(v, true); return predictYBasedOn(neighbors, v); http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java index 6bb0276..7268365 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java @@ -18,8 +18,10 @@ package org.apache.ignite.ml.math; import java.util.Map; +import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; import org.apache.ignite.ml.math.impls.vector.MapWrapperVector; +import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector; /** * Some utils for {@link Vector}. @@ -39,4 +41,98 @@ public class VectorUtils { public static Vector fromMap(Map<Integer, Double> val, boolean cp) { return new MapWrapperVector(val); } + + /** + * Turn number into a local Vector of given size with one-hot encoding. + * + * @param num Number to turn into vector. + * @param vecSize Vector size of output vector. + * @return One-hot encoded number. + */ + public static Vector num2Vec(int num, int vecSize) { + return num2Vec(num, vecSize, false); + } + + /** + * Turn number into Vector of given size with one-hot encoding. + * + * @param num Number to turn into vector. + * @param vecSize Vector size of output vector. + * @param isDistributed Flag indicating if distributed vector should be created. + * @return One-hot encoded number. + */ + public static Vector num2Vec(int num, int vecSize, boolean isDistributed) { + Vector res = isDistributed ? new SparseDistributedVector(vecSize) : new DenseLocalOnHeapVector(vecSize); + return res.setX(num, 1); + } + + /** + * Turn Vector into number by looking at index of maximal element in vector. + * + * @param vec Vector to be turned into number. + * @return Number. + */ + public static double vec2Num(Vector vec) { + int max = 0; + double maxVal = Double.NEGATIVE_INFINITY; + + for (int i = 0; i < vec.size(); i++) { + double curVal = vec.getX(i); + if (curVal > maxVal) { + max = i; + maxVal = curVal; + } + } + + return max; + } + + /** + * Performs in-place vector multiplication. + * + * @param vec1 Operand to be changed and first multiplication operand. + * @param vec2 Second multiplication operand. + * @return Updated first operand. + */ + public static Vector elementWiseTimes(Vector vec1, Vector vec2) { + vec1.map(vec2, (a, b) -> a * b); + + return vec1; + } + + /** + * Performs in-place vector subtraction. + * + * @param vec1 Operand to be changed and subtracted from. + * @param vec2 Operand to subtract. + * @return Updated first operand. + */ + public static Vector elementWiseMinus(Vector vec1, Vector vec2) { + vec1.map(vec2, (a, b) -> a - b); + + return vec1; + } + + /** + * Zip two vectors with given binary function + * (i.e. apply binary function to both vector elementwise and construct vector from results). + * + * Example zipWith({0, 2, 4}, {1, 3, 5}, plus) = {0 + 1, 2 + 3, 4 + 5}. + * Length of result is length of shortest of vectors. + * + * @param v1 First vector. + * @param v2 Second vector. + * @param f Function to zip with. + * @return Result of zipping. + */ + public static Vector zipWith(Vector v1, Vector v2, IgniteBiFunction<Double, Double, Double> f) { + int size = Math.min(v1.size(), v2.size()); + + Vector res = v1.like(size); + + for (int row = 0; row < size; row++) + res.setX(row, f.apply(v1.getX(row), v2.getX(row))); + + return res; + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java new file mode 100644 index 0000000..e97ee41 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableDoubleToDoubleFunction.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.functions; + +/** + * Interface for differentiable functions from double to double. + */ +public interface IgniteDifferentiableDoubleToDoubleFunction extends IgniteDoubleFunction<Double> { + /** + * Get function differential at a given point. + * + * @param pnt Point to calculate differential at. + * @return Function differential at a given point. + */ + double differential(double pnt); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java new file mode 100644 index 0000000..3132c63 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteDifferentiableVectorToDoubleFunction.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.functions; + +import org.apache.ignite.ml.math.Vector; + +/** + * Interface for differentiable functions from vector to double. + */ +public interface IgniteDifferentiableVectorToDoubleFunction extends IgniteFunction<Vector, Double> { + /** + * Get function differential at a given point. + * + * @param pnt Point to calculate differential at. + * @return Function differential at a given point. + */ + Vector differential(Vector pnt); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java new file mode 100644 index 0000000..af304c9 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriConsumer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.functions; + +import java.io.Serializable; +import java.util.function.Consumer; + +/** + * Serializable tri-consumer. + * + * @param <A> First parameter type. + * @param <B> Second parameter type. + * @param <C> Third parameter type. + */ +@FunctionalInterface +public interface IgniteTriConsumer<A, B, C> extends Serializable { + /** + * Analogous to 'accept' in {@link Consumer} version, but with three parameters. + * + * @param first First parameter. + * @param second Second parameter. + * @param third Third parameter. + */ + void accept(A first, B second, C third); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java index ef6bc05..5b6c988 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java @@ -996,4 +996,9 @@ public abstract class AbstractMatrix implements Matrix { return maxAmountOfCols; } + + /** {@inheritDoc} */ + @Override public String toString() { + return "Matrix [rows=" + rowSize() + ", cols=" + columnSize() + "]"; + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java index 9b670e5..9f14bc7 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java @@ -22,6 +22,8 @@ import org.apache.ignite.internal.util.GridArgumentCheck; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.functions.IgniteBiFunction; +import org.apache.ignite.ml.math.functions.IgniteTriFunction; import org.apache.ignite.ml.math.impls.matrix.CacheMatrix; import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; import org.apache.ignite.ml.math.impls.matrix.MatrixView; @@ -209,6 +211,66 @@ public class MatrixUtil { mtx.setX(i, j, fArr[!isRowMode ? i * colsCnt + j : j * rowsCnt + i]); } + /** + * Zip two vectors with given tri-function taking as third argument position in vector + * (i.e. apply binary function to both vector elementwise and construct vector from results). + * Example zipWith({200, 400, 600}, {100, 300, 500}, plusAndMultiplyByIndex) = {(200 + 100) * 0, (400 + 300) * 1, (600 + 500) * 3}. + * Length of result is length of shortest of vectors. + * + * @param v1 First vector. + * @param v2 Second vector. + * @param f Function to zip with. + * @return Result of zipping. + */ + public static Vector zipWith(Vector v1, Vector v2, IgniteTriFunction<Double, Double, Integer, Double> f) { + int size = Math.min(v1.size(), v2.size()); + + Vector res = v1.like(size); + + for (int row = 0; row < size; row++) + res.setX(row, f.apply(v1.getX(row), v2.getX(row), row)); + + return res; + } + + /** + * Zips two matrices by column-by-column with specified function. Result is matrix same flavour as first matrix. + * + * @param mtx1 First matrix. + * @param mtx2 Second matrix. + * @param fun Function to zip with. + * @return Vector consisting from values resulted from zipping column-by-column. + */ + public static Vector zipFoldByColumns(Matrix mtx1, Matrix mtx2, IgniteBiFunction<Vector, Vector, Double> fun) { + int cols = Math.min(mtx1.columnSize(), mtx2.columnSize()); + + Vector vec = mtx1.likeVector(cols); + + for (int i = 0; i < cols; i++) + vec.setX(i, fun.apply(mtx1.getCol(i), mtx2.getCol(i))); + + return vec; + } + + /** + * Zips two matrices by row-by-row with specified function. Result is matrix same flavour as first matrix. + * + * @param mtx1 First matrix. + * @param mtx2 Second matrix. + * @param fun Function to zip with. + * @return Vector consisting from values resulted from zipping row-by-row. + */ + public static Vector zipFoldByRows(Matrix mtx1, Matrix mtx2, IgniteBiFunction<Vector, Vector, Double> fun) { + int rows = Math.min(mtx1.rowSize(), mtx2.rowSize()); + + Vector vec = mtx1.likeVector(rows); + + for (int i = 0; i < rows; i++) + vec.setX(i, fun.apply(mtx1.viewRow(i), mtx2.viewRow(i))); + + return vec; + } + /** */ public static double[] flatten(double[][] arr, int stoMode) { assert arr != null; @@ -231,4 +293,30 @@ public class MatrixUtil { return res; } + + /** + * Performs in-place matrix subtraction. + * + * @param mtx1 Operand to be changed and subtracted from. + * @param mtx2 Operand to subtract. + * @return Updated first operand. + */ + public static Matrix elementWiseMinus(Matrix mtx1, Matrix mtx2) { + mtx1.map(mtx2, (a, b) -> a - b); + + return mtx1; + } + + /** + * Performs in-place matrix multiplication. + * + * @param mtx1 Operand to be changed and first multiplication operand. + * @param mtx2 Second multiplication operand. + * @return Updated first operand. + */ + public static Matrix elementWiseTimes(Matrix mtx1, Matrix mtx2) { + mtx1.map(mtx2, (a, b) -> a * b); + + return mtx1; + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java new file mode 100644 index 0000000..f05bde8 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn; + +import org.apache.ignite.ml.math.functions.IgniteDifferentiableDoubleToDoubleFunction; + +/** + * Class containing some common activation functions. + */ +public class Activators { + /** + * Sigmoid activation function. + */ + public static IgniteDifferentiableDoubleToDoubleFunction SIGMOID = new IgniteDifferentiableDoubleToDoubleFunction() { + /** {@inheritDoc} */ + @Override public double differential(double pnt) { + double v = apply(pnt); + return v * (1 - v); + } + + /** {@inheritDoc} */ + @Override public Double apply(double val) { + return 1 / (1 + Math.exp(-val)); + } + }; + + /** + * Rectified linear unit (ReLU) activation function. + */ + public static IgniteDifferentiableDoubleToDoubleFunction RELU = new IgniteDifferentiableDoubleToDoubleFunction() { + /** + * Differential of ReLU at pnt. Formally, function is not differentiable at 0, but we let differential at 0 be 0. + * + * @param pnt Point to differentiate at. + * @return Differential at pnt. + */ + @Override public double differential(double pnt) { + return pnt > 0 ? 1 : 0; + } + + /** {@inheritDoc} */ + @Override public Double apply(double val) { + return Math.max(val, 0); + } + }; +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java new file mode 100644 index 0000000..4574841 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LocalBatchTrainerInput.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn; + +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.ml.Model; +import org.apache.ignite.ml.math.Matrix; + +/** + * Interface for classes containing input parameters for LocalBatchTrainer. + */ +public interface LocalBatchTrainerInput<M extends Model<Matrix, Matrix>> { + /** + * Get next batch in form of matrix of inputs and matrix of outputs. + * + * @return Next batch. + */ + IgniteBiTuple<Matrix, Matrix> getBatch(); + + /** + * Model to train. + * + * @return Model to train. + */ + M mdl(); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java new file mode 100644 index 0000000..652072c --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/LossFunctions.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn; + +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteFunction; + +/** + * Class containing popular loss functions. + */ +public class LossFunctions { + /** + * Mean squared error loss function. + */ + public static IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> MSE = groundTruth -> + new IgniteDifferentiableVectorToDoubleFunction() { + /** {@inheritDoc} */ + @Override public Vector differential(Vector pnt) { + double multiplier = 2.0 / pnt.size(); + return pnt.minus(groundTruth).times(multiplier); + } + + /** {@inheritDoc} */ + @Override public Double apply(Vector vector) { + return groundTruth.copy().map(vector, (a, b) -> { + double diff = a - b; + return diff * diff; + }).sum() / (vector.size()); + } + }; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java new file mode 100644 index 0000000..621dc9f --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPLayer.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn; + +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.Vector; + +/** + * Class containing information about layer. + */ +public class MLPLayer { + /** + * Weights matrix. + */ + protected Matrix weights; + + /** + * Biases vector. + */ + protected Vector biases; + + /** + * Construct MLPLayer from weights and biases. + * + * @param weights Weights. + * @param biases Biases. + */ + public MLPLayer(Matrix weights, Vector biases) { + this.weights = weights; + this.biases = biases; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java new file mode 100644 index 0000000..5884543 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPState.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn; + +import java.util.ArrayList; +import java.util.List; +import org.apache.ignite.ml.math.Matrix; + +/** + * State of MLP after computation. + */ +public class MLPState { + /** + * Output of activators. + */ + protected List<Matrix> activatorsOutput; + + /** + * Output of linear transformations. + */ + protected List<Matrix> linearOutput; + + /** + * Input. + */ + protected Matrix input; + + /** + * Construct MLP state. + * + * @param input Matrix of inputs. + */ + public MLPState(Matrix input) { + this.input = input != null ? input.copy() : null; + linearOutput = new ArrayList<>(); + activatorsOutput = new ArrayList<>(); + } + + /** + * Output of activators of given layer. If layer index is 0, inputs are returned. + * + * @param layer Index of layer to get activators outputs from. + * @return Activators output. + */ + public Matrix activatorsOutput(int layer) { + return layer > 0 ? activatorsOutput.get(layer - 1) : input; + } + + /** + * Output of linear transformation of given layer. If layer index is 0, inputs are returned. + * + * @param layer Index of layer to get linear transformation outputs from. + * @return Linear transformation output. + */ + public Matrix linearOutput(int layer) { + return layer == 0 ? input : linearOutput.get(layer - 1); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java new file mode 100644 index 0000000..e372d96 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MultilayerPerceptron.java @@ -0,0 +1,565 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.ml.Model; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.functions.IgniteDifferentiableDoubleToDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteDifferentiableVectorToDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteFunction; +import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; +import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; +import org.apache.ignite.ml.nn.architecture.MLPArchitecture; +import org.apache.ignite.ml.nn.architecture.TransformationLayerArchitecture; +import org.apache.ignite.ml.nn.initializers.MLPInitializer; +import org.apache.ignite.ml.nn.initializers.RandomInitializer; +import org.apache.ignite.ml.nn.updaters.SmoothParametrized; + +import static org.apache.ignite.ml.math.util.MatrixUtil.elementWiseTimes; + +/** + * Class encapsulating logic of multilayer perceptron. + */ +public class MultilayerPerceptron implements Model<Matrix, Matrix>, SmoothParametrized<MultilayerPerceptron> { + /** + * This MLP architecture. + */ + protected MLPArchitecture architecture; + + /** + * List containing layers parameters. + */ + protected List<MLPLayer> layers; + + /** + * MLP which is 'below' this MLP (i.e. below output goes to this MLP as input). + */ + protected MultilayerPerceptron below; + + /** + * Construct MLP from given architecture and parameters initializer. + * + * @param arch Architecture. + * @param initializer Parameters initializer. + */ + public MultilayerPerceptron(MLPArchitecture arch, MLPInitializer initializer) { + layers = new ArrayList<>(arch.layersCount() + 1); + architecture = arch; + below = null; + + initLayers(initializer); + } + + /** + * Construct MLP from given architecture. + * + * @param arch Architecture. + */ + public MultilayerPerceptron(MLPArchitecture arch) { + layers = new ArrayList<>(arch.layersCount() + 1); + architecture = arch; + below = null; + + initLayers(new RandomInitializer(new Random())); + } + + /** + * Init layers parameters with initializer. + * + * @param initializer Parameters initializer. + */ + private void initLayers(MLPInitializer initializer) { + int prevSize = architecture.inputSize(); + + for (int i = 1; i < architecture.layersCount(); i++) { + TransformationLayerArchitecture layerCfg = architecture.transformationLayerArchitecture(i); + int neuronsCnt = layerCfg.neuronsCount(); + DenseLocalOnHeapMatrix weights = new DenseLocalOnHeapMatrix(neuronsCnt, prevSize); + initializer.initWeights(weights); + DenseLocalOnHeapVector biases = null; + if (layerCfg.hasBias()) { + biases = new DenseLocalOnHeapVector(neuronsCnt); + initializer.initBiases(biases); + } + layers.add(new MLPLayer(weights, biases)); + prevSize = layerCfg.neuronsCount(); + } + } + + /** + * Create MLP from two MLPs: first stacked on second. + * + * @param above MLP to be above. + * @param below MLP to be below. + */ + protected MultilayerPerceptron(MultilayerPerceptron above, MultilayerPerceptron below) { + this.layers = above.layers; + this.architecture = above.architecture; + this.below = below; + } + + /** + * Perform forward pass and return state of outputs of each layer. + * + * @param val Value to perform computation on. + * @return MLP state after computation. + */ + public MLPState computeState(Matrix val) { + MLPState res = new MLPState(val); + + forwardPass(val, res, true); + + return res; + } + + /** + * Perform forward pass and if needed write state of outputs of each layer. + * + * @param val Value to perform computation on. + * @param state State object to write state into. + * @param writeState Flag indicating need to write state. + */ + public Matrix forwardPass(Matrix val, MLPState state, boolean writeState) { + Matrix res = val; + + if (below != null) + res = below.forwardPass(val, state, writeState); + + for (int i = 1; i < architecture.layersCount(); i++) { + MLPLayer curLayer = layers.get(i - 1); + res = curLayer.weights.times(res); + + TransformationLayerArchitecture layerCfg = this.architecture.transformationLayerArchitecture(i); + + if (layerCfg.hasBias()) { + ReplicatedVectorMatrix biasesMatrix = new ReplicatedVectorMatrix(biases(i), res.columnSize(), true); + res = res.plus(biasesMatrix); + } + + state.linearOutput.add(res); + + // If we do not write state, we can overwrite result. + if (writeState) + res = res.copy(); + + res = res.map(layerCfg.activationFunction()); + + state.activatorsOutput.add(res); + } + + return res; + } + + /** + * Predict values on inputs given as columns in a given matrix. + * + * @param val Matrix containing inputs as columns. + * @return Matrix with predicted vectors stored in columns with column indexes corresponding to column indexes in + * the input matrix. + */ + @Override public Matrix apply(Matrix val) { + MLPState state = new MLPState(null); + forwardPass(val, state, false); + return state.activatorsOutput.get(state.activatorsOutput.size() - 1); + } + + /** + * Create MLP where this MLP output is fed as input to added MLP. + * + * @param above Added MLP. + * @return New MLP where this MLP output is fed as input to added MLP. + */ + public MultilayerPerceptron add(MultilayerPerceptron above) { + return new MultilayerPerceptron(above, this); + } + + /** + * Get weights of layer with given index. Proper indexes are in [1, layersCount). + * + * @param layerIdx Layer index. + * @return Weights of layer with given index. + */ + public Matrix weights(int layerIdx) { + assert layerIdx >= 1; + assert layerIdx < architecture.layersCount() || below != null; + + if (layerIdx < belowLayersCount()) + return below.weights(layerIdx - architecture.layersCount()); + else + return layers.get(layerIdx - belowLayersCount() - 1).weights; + } + + /** + * Get biases of layer with given index. Proper indexes are in [1, layersCount). + * + * @param layerIdx Layer index. + * @return Biases of layer with given index. + */ + public Vector biases(int layerIdx) { + assert layerIdx >= 0; + assert layerIdx < architecture.layersCount() || below != null; + + if (layerIdx < belowLayersCount()) + return below.biases(layerIdx - architecture.layersCount()); + else + return layers.get(layerIdx - belowLayersCount() - 1).biases; + } + + /** + * Checks if layer with given index has biases. + * + * @param layerIdx Layer index. + * @return Value of predicate 'layer with layerIdx has biases'. + */ + public boolean hasBiases(int layerIdx) { + return layerIdx != 0 && biases(layerIdx) != null; + + } + + /** + * Sets the biases of layer with a given index. + * + * @param layerIdx Layer index. + * @param bias New values for biases. + * @return This MLP with updated biases. + */ + public MultilayerPerceptron setBiases(int layerIdx, Vector bias) { + biases(layerIdx).assign(bias); + + return this; + } + + /** + * Set the bias of given neuron in given layer. + * + * @param layerIdx Layer index. + * @param neuronIdx Neuron index. + * @param val New value of bias. + * @return This MLP with updated biases. + */ + public MultilayerPerceptron setBias(int layerIdx, int neuronIdx, double val) { + // Should be transformation layer. + assert layerIdx > 0; + assert architecture.transformationLayerArchitecture(layerIdx).hasBias(); + + biases(layerIdx).setX(neuronIdx, val); + + return this; + } + + /** + * Get the bias of given neuron in given layer. + * + * @param layerIdx Layer index. + * @param neuronIdx Neuron index. + * @return Bias with specified coordinates. + */ + public double bias(int layerIdx, int neuronIdx) { + // Should be transformation layer. + assert layerIdx > 0; + assert architecture.transformationLayerArchitecture(layerIdx).hasBias(); + + return biases(layerIdx).getX(neuronIdx); + } + + /** + * Sets the weighs of layer with a given index. + * + * @param layerIdx Layer index. + * @param weights New values for weights. + * @return This MLP with updated weights. + */ + public MultilayerPerceptron setWeights(int layerIdx, Matrix weights) { + weights(layerIdx).assign(weights); + + return this; + } + + /** + * Set the weight of neuron with given index in previous layer to neuron with given index in given layer. + * + * @param layerIdx Layer index. + * @param fromNeuron Neuron index in previous layer. + * @param toNeuron Neuron index in current layer. + * @param val New value of weight. + * @return This MLP with updated weights. + */ + public MultilayerPerceptron setWeight(int layerIdx, int fromNeuron, int toNeuron, double val) { + // Should be transformation layer. + assert layerIdx > 0; + + weights(layerIdx).setX(toNeuron, fromNeuron, val); + + return this; + } + + /** + * Get the weight of neuron with given index in previous layer to neuron with given index in given layer. + * + * @param layerIdx Layer index. + * @param fromNeuron Neuron index in previous layer. + * @param toNeuron Neuron index in current layer. + * @return Weight with specified coordinates. + */ + public double weight(int layerIdx, int fromNeuron, int toNeuron) { + // Should be transformation layer. + assert layerIdx > 0; + assert architecture.transformationLayerArchitecture(layerIdx).hasBias(); + + return weights(layerIdx).getX(fromNeuron, toNeuron); + } + + /** + * Get count of layers in this MLP. + * + * @return Count of layers in this MLP. + */ + public int layersCount() { + return architecture.layersCount() + (below != null ? below.layersCount() : 0); + } + + /** Count of layers in below MLP. */ + protected int belowLayersCount() { + return below != null ? below.layersCount() : 0; + } + + /** + * Get architecture of this MLP. + * + * @return Architecture of this MLP. + */ + public MLPArchitecture architecture() { + if (below != null) + return below.architecture().add(architecture()); + return architecture; + } + + /** {@inheritDoc} */ + public Vector differentiateByParameters(IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss, + Matrix inputsBatch, Matrix truthBatch) { + // Backpropagation algorithm is used here. + int batchSize = inputsBatch.columnSize(); + double invBatchSize = 1 / (double)batchSize; + int lastLayer = layersCount() - 1; + MLPState mlpState = computeState(inputsBatch); + Matrix dz = null; + + List<MLPLayer> layersParameters = new LinkedList<>(); + + for (int layer = lastLayer; layer > 0; layer--) { + Matrix z = mlpState.linearOutput(layer).copy(); + Matrix dSigmaDz = differentiateNonlinearity(z, + architecture().transformationLayerArchitecture(layer).activationFunction()); + + if (layer == lastLayer) { + Matrix sigma = mlpState.activatorsOutput(lastLayer).copy(); + Matrix dLossDSigma = differentiateLoss(truthBatch, sigma, loss); + dz = elementWiseTimes(dLossDSigma, dSigmaDz); + } + else { + dz = weights(layer + 1).transpose().times(dz); + dz = elementWiseTimes(dz, dSigmaDz); + } + + Matrix a = mlpState.activatorsOutput(layer - 1); + Matrix dw = dz.times(a.transpose()).times(invBatchSize); + + Vector db = null; + if (hasBiases(layer)) + db = dz.foldRows(Vector::sum).times(invBatchSize); + + // Because we go from last layer, add each layer to the begining. + layersParameters.add(0, new MLPLayer(dw, db)); + } + + return paramsAsVector(layersParameters); + } + + /** {@inheritDoc} */ + public Vector parameters() { + return paramsAsVector(layers); + } + + /** + * Flatten this MLP parameters as vector. + * + * @param layersParams List of layers parameters. + * @return This MLP parameters as vector. + */ + protected Vector paramsAsVector(List<MLPLayer> layersParams) { + int off = 0; + Vector res = new DenseLocalOnHeapVector(architecture().parametersCount()); + + for (MLPLayer layerParams : layersParams) { + off = writeToVector(res, layerParams.weights, off); + + if (layerParams.biases != null) + off = writeToVector(res, layerParams.biases, off); + } + + return res; + } + + /** {@inheritDoc} */ + public MultilayerPerceptron setParameters(Vector vector) { + int off = 0; + + for (int l = 1; l < layersCount(); l++) { + MLPLayer layer = layers.get(l - 1); + + IgniteBiTuple<Integer, Matrix> readRes = readFromVector(vector, layer.weights.rowSize(), + layer.weights.columnSize(), off); + + off = readRes.get1(); + layer.weights = readRes.get2(); + + if (hasBiases(l)) { + IgniteBiTuple<Integer, Vector> readRes1 = readFromVector(vector, layer.biases.size(), off); + off = readRes1.get1(); + + layer.biases = readRes1.get2(); + } + } + + return this; + } + + /** {@inheritDoc} */ + @Override public int parametersCount() { + return architecture().parametersCount(); + } + + /** + * Read matrix with given dimensions from vector starting with offset and return new offset position + * which is last matrix entry position + 1. + * + * @param v Vector to read from. + * @param rows Count of rows of matrix to read. + * @param cols Count of columns of matrix to read. + * @param off Start read position. + * @return New offset position which is last matrix entry position + 1. + */ + private IgniteBiTuple<Integer, Matrix> readFromVector(Vector v, int rows, int cols, int off) { + Matrix mtx = new DenseLocalOnHeapMatrix(rows, cols); + + int size = rows * cols; + for (int i = 0; i < size; i++) + mtx.setX(i / cols, i % cols, v.getX(off + i)); + + return new IgniteBiTuple<>(off + size, mtx); + } + + /** + * Read vector of given size from vector and return new offset position which is last read vector entry position + 1. + * + * @param v Vector to read from. + * @param size Size of vector to read. + * @param off Start read position. + * @return New offset position which is last read vector entry position + 1. + */ + private IgniteBiTuple<Integer, Vector> readFromVector(Vector v, int size, int off) { + Vector vec = new DenseLocalOnHeapVector(size); + + for (int i = 0; i < size; i++) + vec.setX(i, v.getX(off + i)); + + return new IgniteBiTuple<>(off + size, vec); + } + + /** + * Write matrix into vector starting from offset and return new offset position which is last written entry position + 1. + * + * @param vec Vector to write into. + * @param mtx Matrix to write. + * @param off Start write position. + * @return New offset position which is last written entry position + 1. + */ + private int writeToVector(Vector vec, Matrix mtx, int off) { + int rows = mtx.rowSize(); + int cols = mtx.columnSize(); + + for (int r = 0; r < rows; r++) { + for (int c = 0; c < cols; c++) { + vec.setX(off, mtx.getX(r, c)); + off++; + } + } + + return off; + } + + /** + * Write vector into vector starting from offset and return new offset position which is last written entry position + 1. + * + * @param vec Vector to write into. + * @param v Vector to write. + * @param off Start write position. + * @return New offset position which is last written entry position + 1. + */ + private int writeToVector(Vector vec, Vector v, int off) { + for (int i = 0; i < v.size(); i++) { + vec.setX(off, v.getX(i)); + off++; + } + + return off; + } + + /** + * Differentiate loss. + * + * @param groundTruth Ground truth values. + * @param lastLayerOutput Last layer output. + * @param loss Loss function. + * @return Gradients matrix. + */ + private Matrix differentiateLoss(Matrix groundTruth, Matrix lastLayerOutput, + IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) { + Matrix diff = groundTruth.like(groundTruth.rowSize(), groundTruth.columnSize()); + + for (int col = 0; col < groundTruth.columnSize(); col++) { + // TODO: IGNITE-7155 Couldn't use views here because copy on views doesn't do actual copy and all changes are propagated to original. + Vector gtCol = groundTruth.getCol(col); + Vector predCol = lastLayerOutput.getCol(col); + diff.assignColumn(col, loss.apply(gtCol).differential(predCol)); + } + + return diff; + } + + /** + * Differentiate nonlinearity. + * + * @param linearOut Linear output of current layer. + * @param nonlinearity Nonlinearity of current layer. + * @return Gradients matrix. + */ + private Matrix differentiateNonlinearity(Matrix linearOut, IgniteDifferentiableDoubleToDoubleFunction nonlinearity) { + Matrix diff = linearOut.copy(); + + diff.map(nonlinearity::differential); + + return diff; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java new file mode 100644 index 0000000..559206d --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/ReplicatedVectorMatrix.java @@ -0,0 +1,583 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Map; +import java.util.Spliterator; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.MatrixStorage; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.exceptions.CardinalityException; +import org.apache.ignite.ml.math.functions.IgniteBiConsumer; +import org.apache.ignite.ml.math.functions.IgniteBiFunction; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteFunction; +import org.apache.ignite.ml.math.functions.IgniteTriFunction; +import org.apache.ignite.ml.math.functions.IntIntToDoubleFunction; +import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; + +/** + * Convenient way to create matrix of replicated columns or rows from vector. + * This class should be considered as utility class: not all matrix methods are implemented here, only those which + * were necessary for MLPs. + */ +class ReplicatedVectorMatrix implements Matrix { + /** + * Vector to replicate. + */ + private Vector vector; + + /** + * Flag determining is vector replicated as column or row. + */ + private boolean asCol; + + /** + * Count of vector replications. + */ + private int replicationCnt; + + /** + * Construct ReplicatedVectorMatrix. + * + * @param vector Vector to replicate. + * @param replicationCnt Count of replications. + * @param asCol Should vector be replicated as a column or as a row. + */ + ReplicatedVectorMatrix(Vector vector, int replicationCnt, boolean asCol) { + this.vector = vector; + this.asCol = asCol; + this.replicationCnt = replicationCnt; + } + + /** + * Constructor for externalization. + */ + public ReplicatedVectorMatrix() { + // No-op. + } + + /** {@inheritDoc} */ + @Override public boolean isSequentialAccess() { + return vector.isSequentialAccess(); + } + + /** {@inheritDoc} */ + @Override public boolean isRandomAccess() { + return vector.isRandomAccess(); + } + + /** {@inheritDoc} */ + @Override public boolean isDense() { + return vector.isDense(); + } + + /** {@inheritDoc} */ + @Override public boolean isArrayBased() { + return vector.isArrayBased(); + } + + /** {@inheritDoc} */ + @Override public boolean isDistributed() { + return vector.isDistributed(); + } + + /** {@inheritDoc} */ + @Override public double maxValue() { + return vector.maxValue(); + } + + /** {@inheritDoc} */ + @Override public double minValue() { + return vector.minValue(); + } + + /** {@inheritDoc} */ + @Override public Element maxElement() { + return new Element() { + @Override public double get() { + return vector.maxElement().get(); + } + + @Override public int row() { + return asCol ? vector.maxElement().index() : 0; + } + + @Override public int column() { + return asCol ? 0 : vector.maxElement().index(); + } + + @Override public void set(double val) { + + } + }; + } + + /** {@inheritDoc} */ + @Override public Element minElement() { + return new Element() { + @Override public double get() { + return vector.minElement().get(); + } + + @Override public int row() { + return asCol ? vector.minElement().index() : 0; + } + + @Override public int column() { + return asCol ? 0 : vector.minElement().index(); + } + + @Override public void set(double val) { + + } + }; + } + + /** {@inheritDoc} */ + @Override public Element getElement(int row, int col) { + Vector.Element el = asCol ? vector.getElement(row) : vector.getElement(col); + int r = asCol ? el.index() : 0; + int c = asCol ? 0 : el.index(); + + return new Element() { + @Override public double get() { + return el.get(); + } + + @Override public int row() { + return r; + } + + @Override public int column() { + return c; + } + + @Override public void set(double val) { + + } + }; + } + + /** {@inheritDoc} */ + @Override public Matrix swapRows(int row1, int row2) { + return asCol ? new ReplicatedVectorMatrix(swap(row1, row2), replicationCnt, asCol) : this; + } + + /** {@inheritDoc} */ + private Vector swap(int idx1, int idx2) { + double val = vector.getX(idx1); + + vector.setX(idx1, vector.getX(idx2)); + vector.setX(idx2, val); + + return vector; + } + + /** {@inheritDoc} */ + @Override public Matrix swapColumns(int col1, int col2) { + return asCol ? this : new ReplicatedVectorMatrix(swap(col1, col2), replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public Matrix assign(double val) { + return new ReplicatedVectorMatrix(vector.assign(val), replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public Matrix assign(double[][] vals) { + return new DenseLocalOnHeapMatrix(vals); + } + + /** {@inheritDoc} */ + @Override public Matrix assign(Matrix mtx) { + return mtx.copy(); + } + + /** {@inheritDoc} */ + @Override public Matrix assign(IntIntToDoubleFunction fun) { + Vector vec = asCol ? this.vector.assign(idx -> fun.apply(idx, 0)) : this.vector.assign(idx -> fun.apply(0, idx)); + return new ReplicatedVectorMatrix(vec, replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public Matrix map(IgniteDoubleFunction<Double> fun) { + Vector vec = vector.map(fun); + return new ReplicatedVectorMatrix(vec, replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public Matrix map(Matrix mtx, IgniteBiFunction<Double, Double, Double> fun) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public int nonZeroElements() { + return vector.nonZeroElements() * (asCol ? columnSize() : rowSize()); + } + + /** {@inheritDoc} */ + @Override public Spliterator<Double> allSpliterator() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Spliterator<Double> nonZeroSpliterator() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix assignColumn(int col, Vector vec) { + int rows = asCol ? vector.size() : replicationCnt; + int cols = asCol ? replicationCnt : vector.size(); + int times = asCol ? cols : rows; + + Matrix res = new DenseLocalOnHeapMatrix(rows, cols); + + IgniteBiConsumer<Integer, Vector> replicantAssigner = asCol ? res::assignColumn : res::assignRow; + IgniteBiConsumer<Integer, Vector> assigner = res::assignColumn; + + assign(replicantAssigner, assigner, vector, vec, times, col); + + return res; + } + + /** {@inheritDoc} */ + @Override public Matrix assignRow(int row, Vector vec) { + int rows = asCol ? vector.size() : replicationCnt; + int cols = asCol ? replicationCnt : vector.size(); + int times = asCol ? cols : rows; + + Matrix res = new DenseLocalOnHeapMatrix(rows, cols); + + IgniteBiConsumer<Integer, Vector> replicantAssigner = asCol ? res::assignColumn : res::assignRow; + IgniteBiConsumer<Integer, Vector> assigner = res::assignRow; + + assign(replicantAssigner, assigner, vector, vec, times, row); + + return res; + } + + /** */ + private void assign(IgniteBiConsumer<Integer, Vector> replicantAssigner, + IgniteBiConsumer<Integer, Vector> assigner, Vector replicant, Vector vector, int times, int idx) { + for (int i = 0; i < times; i++) + replicantAssigner.accept(i, replicant); + assigner.accept(idx, vector); + } + + /** {@inheritDoc} */ + @Override public Vector foldRows(IgniteFunction<Vector, Double> fun) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Vector foldColumns(IgniteFunction<Vector, Double> fun) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public <T> T foldMap(IgniteBiFunction<T, Double, T> foldFun, IgniteDoubleFunction<Double> mapFun, + T zeroVal) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public boolean density(double threshold) { + return false; + } + + /** {@inheritDoc} */ + @Override public int columnSize() { + return asCol ? replicationCnt : vector.size(); + } + + /** {@inheritDoc} */ + @Override public int rowSize() { + return asCol ? vector.size() : replicationCnt; + } + + /** {@inheritDoc} */ + @Override public double determinant() { + // If matrix is not square throw exception. + checkCardinality(vector.size(), replicationCnt); + + // If matrix is 1x1 then determinant is its single element otherwise there are linear dependence and determinant is 0. + return vector.size() > 0 ? 0 : vector.get(1); + } + + /** {@inheritDoc} */ + @Override public Matrix inverse() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix divide(double x) { + return new ReplicatedVectorMatrix(vector.divide(x), replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public double get(int row, int col) { + return asCol ? vector.get(row) : vector.get(col); + } + + /** {@inheritDoc} */ + @Override public double getX(int row, int col) { + return asCol ? vector.getX(row) : vector.getX(col); + } + + /** {@inheritDoc} */ + @Override public MatrixStorage getStorage() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix copy() { + Vector cp = vector.copy(); + return new ReplicatedVectorMatrix(cp, replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public Matrix like(int rows, int cols) { + Vector lk = vector.like(vector.size()); + return new ReplicatedVectorMatrix(lk, replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public Vector likeVector(int crd) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix minus(Matrix mtx) { + throw new UnsupportedOperationException(); + } + + /** + * Specialized optimized version of minus for ReplicatedVectorMatrix. + * + * @param mtx Matrix to be subtracted. + * @return new ReplicatedVectorMatrix resulting from subtraction. + */ + public Matrix minus(ReplicatedVectorMatrix mtx) { + if (isColumnReplicated() == mtx.isColumnReplicated()) { + checkCardinality(mtx.rowSize(), mtx.columnSize()); + + Vector minus = vector.minus(mtx.replicant()); + + return new ReplicatedVectorMatrix(minus, replicationCnt, asCol); + } + + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix plus(double x) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix plus(Matrix mtx) { + throw new UnsupportedOperationException(); + } + + /** + * Specialized optimized version of plus for ReplicatedVectorMatrix. + * + * @param mtx Matrix to be added. + * @return new ReplicatedVectorMatrix resulting from addition. + */ + public Matrix plus(ReplicatedVectorMatrix mtx) { + if (isColumnReplicated() == mtx.isColumnReplicated()) { + checkCardinality(mtx.rowSize(), mtx.columnSize()); + + Vector plus = vector.plus(mtx.replicant()); + + return new ReplicatedVectorMatrix(plus, replicationCnt, asCol); + } + + throw new UnsupportedOperationException(); + } + + /** + * Checks that dimensions of this matrix are equal to given dimensions. + * + * @param rows Rows. + * @param cols Columns. + */ + private void checkCardinality(int rows, int cols) { + if (rows != rowSize()) + throw new CardinalityException(rowSize(), rows); + + if (cols != columnSize()) + throw new CardinalityException(columnSize(), cols); + } + + /** {@inheritDoc} */ + @Override public IgniteUuid guid() { + return null; + } + + /** {@inheritDoc} */ + @Override public Matrix set(int row, int col, double val) { + vector.set(asCol ? row : col, val); + + return this; + } + + /** {@inheritDoc} */ + @Override public Matrix setRow(int row, double[] data) { + return null; + } + + /** {@inheritDoc} */ + @Override public Vector getRow(int row) { + return null; + } + + /** {@inheritDoc} */ + @Override public Matrix setColumn(int col, double[] data) { + return null; + } + + /** {@inheritDoc} */ + @Override public Vector getCol(int col) { + return null; + } + + /** {@inheritDoc} */ + @Override public Matrix setX(int row, int col, double val) { + return null; + } + + /** {@inheritDoc} */ + @Override public Matrix times(double x) { + return new ReplicatedVectorMatrix(vector.times(x), replicationCnt, asCol); + } + + /** {@inheritDoc} */ + @Override public Matrix times(Matrix mtx) { + if (!asCol) { + Vector row = vector.like(mtx.columnSize()); + + for (int i = 0; i < mtx.columnSize(); i++) + row.setX(i, vector.dot(mtx.getCol(i))); + + return new ReplicatedVectorMatrix(row, replicationCnt, false); + + } + else + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Vector times(Vector vec) { + Vector res = vec.like(vec.size()); + if (asCol) { + for (int i = 0; i < rowSize(); i++) + res.setX(i, vec.sum() * vector.getX(i)); + } + else { + double val = vector.dot(vec); + + for (int i = 0; i < rowSize(); i++) + res.setX(i, val); + } + return res; + } + + /** {@inheritDoc} */ + @Override public double maxAbsRowSumNorm() { + return 0; + } + + /** {@inheritDoc} */ + @Override public double sum() { + return vector.sum() * replicationCnt; + } + + /** {@inheritDoc} */ + @Override public Matrix transpose() { + return new ReplicatedVectorMatrix(vector, replicationCnt, !asCol); + } + + /** {@inheritDoc} */ + @Override public Matrix viewPart(int[] off, int[] size) { + return null; + } + + /** {@inheritDoc} */ + @Override public Matrix viewPart(int rowOff, int rows, int colOff, int cols) { + return null; + } + + /** {@inheritDoc} */ + @Override public Vector viewRow(int row) { + return null; + } + + /** {@inheritDoc} */ + @Override public Vector viewColumn(int col) { + return null; + } + + /** {@inheritDoc} */ + @Override public Vector viewDiagonal() { + return null; + } + + /** {@inheritDoc} */ + @Override public void compute(int row, int col, IgniteTriFunction<Integer, Integer, Double, Double> f) { + // This operation cannot be performed because computing function depends on both indexes and therefore + // result of compute will be in general case not ReplicatedVectorMatrix. + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Map<String, Object> getMetaStorage() { + return null; + } + + /** + * Returns true if matrix constructed by replicating vector as column and false otherwise. + */ + public boolean isColumnReplicated() { + return asCol; + } + + /** + * Returns replicated vector. + */ + public Vector replicant() { + return vector; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java new file mode 100644 index 0000000..31a3e9a --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/LayerArchitecture.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn.architecture; + +/** + * Layer architecture. + */ +public class LayerArchitecture { + /** + * Count of neurons on layer. + */ + private final int neuronsCnt; + + /** + * Construct LayerArchitecture. + * + * @param neuronsCnt Count of neurons in layer. + */ + public LayerArchitecture(int neuronsCnt) { + this.neuronsCnt = neuronsCnt; + } + + /** + * Get count of neurons in layer. + * @return Count of neurons in layer. + */ + public int neuronsCount() { + return neuronsCnt; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/e4f19215/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java new file mode 100644 index 0000000..3ef7b61 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/architecture/MLPArchitecture.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.nn.architecture; + +import java.util.ArrayList; +import java.util.List; +import org.apache.ignite.ml.math.functions.IgniteDifferentiableDoubleToDoubleFunction; + +/** + * Class containing information about architecture of MLP. + */ +public class MLPArchitecture { + /** + * List of layers architectures. + */ + private final List<LayerArchitecture> layers; + + /** + * Construct an MLP architecture. + * + * @param inputSize Size of input to MLP. + */ + public MLPArchitecture(int inputSize) { + layers = new ArrayList<>(); + layers.add(new LayerArchitecture(inputSize)); + } + + /** + * Construct an MLP architecture. + * + * @param layers List of layers architectures. + */ + private MLPArchitecture(List<LayerArchitecture> layers) { + this.layers = layers; + } + + /** + * Count of layers in MLP. + * @return Layers count. + */ + public int layersCount() { + return layers.size(); + } + + /** + * Size of input of MLP. + * @return Size of input. + */ + public int inputSize() { + return layers.get(0).neuronsCount(); + } + + /** + * Size of output of MLP. + * @return Size of output. + */ + public int outputSize() { + return layers.get(layersCount() - 1).neuronsCount(); + } + + /** + * Constructs new MLP architecture with new layer added on top of all this architecture layers. + * + * @param neuronsCnt Count of neurons in new layer. + * @param hasBias Flag indicating presence of bias in added layer. + * @param f Activation function of a new layer. + * @return New MLP architecture with new layer added on top of all this architecture layers. + */ + public MLPArchitecture withAddedLayer(int neuronsCnt, boolean hasBias, IgniteDifferentiableDoubleToDoubleFunction f) { + ArrayList<LayerArchitecture> newLayers = new ArrayList<>(layers); + + newLayers.add(new TransformationLayerArchitecture(neuronsCnt, hasBias, f)); + + return new MLPArchitecture(newLayers); + } + + /** + * Get architecture of layer with given index. + * + * @param layer Index of layer to get architecture from. + * @return Architecture of layer with given index. + */ + public LayerArchitecture layerArchitecture(int layer) { + return layers.get(layer); + } + + /** + * Get architecture of transformation layer (i.e. non-input layer) with given index. + * + * @param layer Index of layer to get architecture from. + * @return Architecture of transformation layer with given index. + */ + public TransformationLayerArchitecture transformationLayerArchitecture(int layer) { + return (TransformationLayerArchitecture)layers.get(layer); + } + + /** + * Creates config describing network where first goes this config and after goes this method's argument. + * + * @param second Config to add after this config. + * @return New combined configuration. + */ + public MLPArchitecture add(MLPArchitecture second) { + assert second.inputSize() == outputSize(); + + MLPArchitecture res = new MLPArchitecture(inputSize()); + res.layers.addAll(layers); + res.layers.addAll(second.layers); + + return res; + } + + /** + * Count of parameters in this MLP architecture. + * + * @return Parameters in this MLP architecture. + */ + public int parametersCount() { + int res = 0; + + for (int i = 1; i < layersCount(); i++) { + TransformationLayerArchitecture la = transformationLayerArchitecture(i); + res += layerArchitecture(i - 1).neuronsCount() * la.neuronsCount(); + + if (la.hasBias()) + res += la.neuronsCount(); + + } + + return res; + } +}
