Repository: ignite Updated Branches: refs/heads/master a15b45bdf -> 106f57b89
IGNITE-6640: Introduction of models import/export. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/106f57b8 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/106f57b8 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/106f57b8 Branch: refs/heads/master Commit: 106f57b894933f0426f85aa78bb9ec389f06d50d Parents: a15b45b Author: Yury Babak <[email protected]> Authored: Wed Nov 1 17:16:38 2017 +0300 Committer: Igor Sapego <[email protected]> Committed: Wed Nov 1 17:16:38 2017 +0300 ---------------------------------------------------------------------- .../java/org/apache/ignite/ml/Exportable.java | 16 ++++ .../java/org/apache/ignite/ml/Exporter.java | 38 ++++++++++ .../java/org/apache/ignite/ml/FileExporter.java | 74 ++++++++++++++++++ .../org/apache/ignite/ml/KMeansModelFormat.java | 77 +++++++++++++++++++ .../main/java/org/apache/ignite/ml/Model.java | 3 +- .../ignite/ml/clustering/KMeansModel.java | 43 ++++++++++- .../ignite/ml/math/EuclideanDistance.java | 11 +++ .../storage/matrix/BlockMatrixStorage.java | 37 ++++----- .../org/apache/ignite/ml/LocalModelsTest.java | 80 ++++++++++++++++++++ 9 files changed, 357 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/Exportable.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Exportable.java b/modules/ml/src/main/java/org/apache/ignite/ml/Exportable.java new file mode 100644 index 0000000..f30f46d --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/Exportable.java @@ -0,0 +1,16 @@ +package org.apache.ignite.ml; + +/** + * Interface for exportable models({@link Model}). + * + * @see Exporter + */ +public interface Exportable<D> { + /** + * Save model by the given path. + * + * @param exporter Exporter. + * @param path Path to saved model. + */ + public <P> void saveModel(Exporter<D, P> exporter, P path); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/Exporter.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Exporter.java b/modules/ml/src/main/java/org/apache/ignite/ml/Exporter.java new file mode 100644 index 0000000..de796bd --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/Exporter.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml; + +/** + * Exporter for models, D - model representation object, P - path to resources. + */ +public interface Exporter<D, P> { + /** + * Save model by path p + * + * @param d Model representation object. + * @param p Path to resource. + */ + public void save(D d, P p); + + /** + * Load model representation object from p. + * + * @param p Path to resource. + */ + public D load(P p); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/FileExporter.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/FileExporter.java b/modules/ml/src/main/java/org/apache/ignite/ml/FileExporter.java new file mode 100644 index 0000000..79bb71c --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/FileExporter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import org.apache.ignite.IgniteLogger; + +/** + * Implementation of exporter to/from file. + * + * Default implementation of {@link Exporter}. + */ +public class FileExporter<D> implements Exporter<D, String> { + /** */ + private IgniteLogger log = null; + + /** + * @param log Logger. + */ + public void setLog(IgniteLogger log) { + this.log = log; + } + + /** {@inheritDoc} */ + @Override public void save(D d, String path) { + try (FileOutputStream fos = new FileOutputStream(path)){ + try (ObjectOutputStream outStream = new ObjectOutputStream(fos)){ + outStream.writeObject(d); + } + } catch (IOException e) { + if (log != null) + log.error("Error opening file.", e); + } + } + + /** {@inheritDoc} */ + @Override public D load(String path) { + D mdl = null; + + try (FileInputStream fis = new FileInputStream(path)){ + try (ObjectInputStream inputStream = new ObjectInputStream(fis)) { + mdl = (D)inputStream.readObject(); + } + catch (ClassNotFoundException e) { + if (log != null) + log.error("Object creation failed.", e); + } + } + catch (IOException e) { + if (log != null) + log.error("Error opening file.", e); + } + return mdl; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/KMeansModelFormat.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/KMeansModelFormat.java b/modules/ml/src/main/java/org/apache/ignite/ml/KMeansModelFormat.java new file mode 100644 index 0000000..4f5b143 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/KMeansModelFormat.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml; + +import java.io.Serializable; +import java.util.Arrays; +import org.apache.ignite.ml.math.DistanceMeasure; +import org.apache.ignite.ml.math.Vector; + +/** + * K-means model representation. + * + * @see Exportable + * @see Exporter + */ +public class KMeansModelFormat implements Serializable { + /** Centers of clusters. */ + private final Vector[] centers; + + /** Distance measure. */ + private final DistanceMeasure distance; + + /** */ + public KMeansModelFormat(Vector[] centers, DistanceMeasure distance) { + this.centers = centers; + this.distance = distance; + } + + /** */ + public DistanceMeasure getDistance() { + return distance; + } + + /** */ + public Vector[] getCenters() { + return centers; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + res = res * 37 + distance.hashCode(); + res = res * 37 + Arrays.hashCode(centers); + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (this == obj) + return true; + + if (obj == null || getClass() != obj.getClass()) + return false; + + KMeansModelFormat that = (KMeansModelFormat)obj; + + return distance.equals(that.distance) && Arrays.deepEquals(centers, that.centers); + } + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/Model.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java index 4dc1aca..3c60bfa 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java @@ -17,11 +17,12 @@ package org.apache.ignite.ml; +import java.io.Serializable; import java.util.function.BiFunction; /** Basic interface for all models. */ @FunctionalInterface -public interface Model<T, V> { +public interface Model<T, V> extends Serializable { /** Predict a result for value. */ public V predict(T val); http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java index 6584273..7f6f877 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java @@ -18,18 +18,21 @@ package org.apache.ignite.ml.clustering; import java.util.Arrays; +import org.apache.ignite.ml.Exportable; +import org.apache.ignite.ml.Exporter; +import org.apache.ignite.ml.KMeansModelFormat; import org.apache.ignite.ml.math.DistanceMeasure; import org.apache.ignite.ml.math.Vector; /** * This class encapsulates result of clusterization. */ -public class KMeansModel implements ClusterizationModel<Vector, Integer> { +public class KMeansModel implements ClusterizationModel<Vector, Integer>, Exportable<KMeansModelFormat> { /** Centers of clusters. */ - private Vector[] centers; + private final Vector[] centers; /** Distance measure. */ - private DistanceMeasure distance; + private final DistanceMeasure distance; /** * Construct KMeans model with given centers and distance measure. @@ -37,7 +40,7 @@ public class KMeansModel implements ClusterizationModel<Vector, Integer> { * @param centers Centers. * @param distance Distance measure. */ - KMeansModel(Vector[] centers, DistanceMeasure distance) { + public KMeansModel(Vector[] centers, DistanceMeasure distance) { this.centers = centers; this.distance = distance; } @@ -76,4 +79,36 @@ public class KMeansModel implements ClusterizationModel<Vector, Integer> { return res; } + + /** {@inheritDoc} */ + @Override public <P> void saveModel(Exporter<KMeansModelFormat, P> exporter, P path) { + KMeansModelFormat mdlData = new KMeansModelFormat(centers, distance); + + exporter.save(mdlData, path); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + res = res * 37 + distance.hashCode(); + res = res * 37 + Arrays.hashCode(centers); + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (this == obj) + return true; + + if (obj == null || getClass() != obj.getClass()) + return false; + + KMeansModel that = (KMeansModel)obj; + + return distance.equals(that.distance) && Arrays.deepEquals(centers, that.centers); + } + + } http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java index edc11dc..5d5a64e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java @@ -44,4 +44,15 @@ public class EuclideanDistance implements DistanceMeasure { @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { // No-op } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (this == obj) + return true; + + if (obj == null || getClass() != obj.getClass()) + return false; + + return true; + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java index 0978e5a..2fb4b30 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java @@ -271,30 +271,38 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto return res; } + /** */ + public IgnitePair<Long> getBlockId(int x, int y) { + return new IgnitePair<>((long)x / maxBlockEdge, (long)y / maxBlockEdge); + } + /** {@inheritDoc} */ @Override public int hashCode() { - int res = 1; + int res = blocksInCol; - res = res * 37 + cols; - res = res * 37 + rows; - res = res * 37 + uuid.hashCode(); - res = res * 37 + cache.hashCode(); + res = 31 * res + blocksInRow; + res = 31 * res + rows; + res = 31 * res + cols; + res = 31 * res + uuid.hashCode(); + res = 31 * res + maxBlockEdge; + res = 31 * res + cache.getName().hashCode(); return res; } /** {@inheritDoc} */ - @Override public boolean equals(Object obj) { - if (this == obj) + @Override public boolean equals(Object o) { + if (this == o) return true; - - if (obj == null || getClass() != obj.getClass()) + if (o == null || getClass() != o.getClass()) return false; - BlockMatrixStorage that = (BlockMatrixStorage)obj; + BlockMatrixStorage that = (BlockMatrixStorage)o; + + return blocksInCol == that.blocksInCol && blocksInRow == that.blocksInRow && rows == that.rows + && cols == that.cols && maxBlockEdge == that.maxBlockEdge && uuid.equals(that.uuid) + && cache.getName().equals(that.cache.getName()); - return rows == that.rows && cols == that.cols && uuid.equals(that.uuid) - && (cache != null ? cache.equals(that.cache) : that.cache == null); } /** @@ -358,11 +366,6 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto } /** */ - public IgnitePair<Long> getBlockId(int x, int y) { - return new IgnitePair<>((long)x / maxBlockEdge, (long)y / maxBlockEdge); - } - - /** */ private BlockEntry initBlockFor(int x, int y) { int blockRows = 0; int blockCols = 0; http://git-wip-us.apache.org/repos/asf/ignite/blob/106f57b8/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java new file mode 100644 index 0000000..8b12501 --- /dev/null +++ b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.ignite.ml.clustering.KMeansLocalClusterer; +import org.apache.ignite.ml.clustering.KMeansModel; +import org.apache.ignite.ml.math.EuclideanDistance; +import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for models import/export functionality. + */ +public class LocalModelsTest { + /** */ + private String mdlFilePath = "model.mlmod"; + + /** + * + */ + @After + public void cleanUp() throws IOException { + Files.deleteIfExists(Paths.get(mdlFilePath)); + } + + /** + * + */ + @Test + public void importExportKMeansModelTest(){ + Path mdlPath = Paths.get(mdlFilePath); + + KMeansModel mdl = getClusterModel(); + + Exporter<KMeansModelFormat, String> exporter = new FileExporter<>(); + mdl.saveModel(exporter, mdlFilePath); + + Assert.assertTrue(String.format("File %s not found.", mdlPath.toString()), Files.exists(mdlPath)); + + KMeansModelFormat load = exporter.load(mdlFilePath); + KMeansModel importedMdl = new KMeansModel(load.getCenters(), load.getDistance()); + + Assert.assertTrue("", mdl.equals(importedMdl)); + } + + /** + * + */ + private KMeansModel getClusterModel(){ + KMeansLocalClusterer clusterer = new KMeansLocalClusterer(new EuclideanDistance(), 1, 1L); + + double[] v1 = new double[] {1959, 325100}; + double[] v2 = new double[] {1960, 373200}; + + DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(new double[][] {v1, v2}); + + return clusterer.cluster(points, 1); + } +}
