http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java index cd76e5a..411b038 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java @@ -17,14 +17,6 @@ package org.apache.ignite.ml.math.impls.storage.matrix; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.UUID; import org.apache.ignite.IgniteCache; import org.apache.ignite.Ignition; import org.apache.ignite.cache.CacheAtomicityMode; @@ -38,16 +30,22 @@ import org.apache.ignite.ml.math.MatrixStorage; import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.distributed.CacheUtils; import org.apache.ignite.ml.math.distributed.DistributedStorage; -import org.apache.ignite.ml.math.distributed.keys.impl.BlockMatrixKey; -import org.apache.ignite.ml.math.impls.matrix.BlockEntry; +import org.apache.ignite.ml.math.distributed.keys.impl.MatrixBlockKey; +import org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry; import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; -import static org.apache.ignite.ml.math.impls.matrix.BlockEntry.MAX_BLOCK_SIZE; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.*; + +import static org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry.MAX_BLOCK_SIZE; /** * Storage for {@link SparseBlockDistributedMatrix}. */ -public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants, DistributedStorage<BlockMatrixKey> { +public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants, DistributedStorage<MatrixBlockKey> { /** Cache name used for all instances of {@link BlockMatrixStorage}. */ private static final String CACHE_NAME = "ML_BLOCK_SPARSE_MATRICES_CONTAINER"; /** */ @@ -65,8 +63,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto /** Actual distributed storage. */ private IgniteCache< - BlockMatrixKey /* Matrix block number with uuid. */, - BlockEntry /* Block of matrix, local sparse matrix. */ + MatrixBlockKey /* Matrix block number with uuid. */, + MatrixBlockEntry /* Block of matrix, local sparse matrix. */ > cache = null; /** @@ -98,7 +96,7 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto /** * */ - public IgniteCache<BlockMatrixKey, BlockEntry> cache() { + public IgniteCache<MatrixBlockKey, MatrixBlockEntry> cache() { return cache; } @@ -132,20 +130,6 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto return RANDOM_ACCESS_MODE; } - /** - * @return Blocks in column. - */ - public int blocksInCol() { - return blocksInCol; - } - - /** - * @return Blocks in row. - */ - public int blocksInRow() { - return blocksInRow; - } - /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeInt(rows); @@ -210,8 +194,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto * * NB: NOT cell indices. */ - public BlockMatrixKey getCacheKey(long blockIdRow, long blockIdCol) { - return new BlockMatrixKey(blockIdRow, blockIdCol, uuid, getAffinityKey(blockIdRow, blockIdCol)); + public MatrixBlockKey getCacheKey(long blockIdRow, long blockIdCol) { + return new MatrixBlockKey(blockIdRow, blockIdCol, uuid, getAffinityKey(blockIdRow, blockIdCol)); } /** @@ -219,17 +203,17 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto * * NB: NOT cell indices. */ - public BlockMatrixKey getCacheKey(IgnitePair<Long> blockId) { - return new BlockMatrixKey(blockId.get1(), blockId.get2(), uuid, getAffinityKey(blockId.get1(), blockId.get2())); + private MatrixBlockKey getCacheKey(IgnitePair<Long> blockId) { + return new MatrixBlockKey(blockId.get1(), blockId.get2(), uuid, getAffinityKey(blockId.get1(), blockId.get2())); } /** {@inheritDoc} */ - @Override public Set<BlockMatrixKey> getAllKeys() { + @Override public Set<MatrixBlockKey> getAllKeys() { int maxRowIdx = rows - 1; int maxColIdx = cols - 1; IgnitePair<Long> maxBlockId = getBlockId(maxRowIdx, maxColIdx); - Set<BlockMatrixKey> keyset = new HashSet<>(); + Set<MatrixBlockKey> keyset = new HashSet<>(); for(int i = 0; i <= maxBlockId.get1(); i++) for(int j = 0; j <= maxBlockId.get2(); j++) @@ -249,8 +233,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto * @param blockId block id. * @return The list of block entries. */ - public List<BlockEntry> getRowForBlock(IgnitePair<Long> blockId) { - List<BlockEntry> res = new LinkedList<>(); + public List<MatrixBlockEntry> getRowForBlock(IgnitePair<Long> blockId) { + List<MatrixBlockEntry> res = new LinkedList<>(); for (int i = 0; i < blocksInCol; i++) res.add(getEntryById(new IgnitePair<>(blockId.get1(), (long) i))); @@ -265,8 +249,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto * @param blockId block id. * @return The list of block entries. */ - public List<BlockEntry> getColForBlock(IgnitePair<Long> blockId) { - List<BlockEntry> res = new LinkedList<>(); + public List<MatrixBlockEntry> getColForBlock(IgnitePair<Long> blockId) { + List<MatrixBlockEntry> res = new LinkedList<>(); for (int i = 0; i < blocksInRow; i++) res.add(getEntryById(new IgnitePair<>((long) i, blockId.get2()))); @@ -308,10 +292,10 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto * @param blockId blockId * @return BlockEntry */ - private BlockEntry getEntryById(IgnitePair<Long> blockId) { - BlockMatrixKey key = getCacheKey(blockId.get1(), blockId.get2()); + private MatrixBlockEntry getEntryById(IgnitePair<Long> blockId) { + MatrixBlockKey key = getCacheKey(blockId.get1(), blockId.get2()); - BlockEntry entry = cache.localPeek(key, CachePeekMode.PRIMARY); + MatrixBlockEntry entry = cache.localPeek(key, CachePeekMode.PRIMARY); entry = entry != null ? entry : cache.get(key); if (entry == null) @@ -325,8 +309,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto * @param blockId blockId * @return Empty BlockEntry */ - private BlockEntry getEmptyBlockEntry(IgnitePair<Long> blockId) { - BlockEntry entry; + private MatrixBlockEntry getEmptyBlockEntry(IgnitePair<Long> blockId) { + MatrixBlockEntry entry; int rowMod = rows % maxBlockEdge; int colMod = cols % maxBlockEdge; @@ -345,7 +329,7 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto else colSize = blockId.get2() != (blocksInCol - 1) ? maxBlockEdge : colMod; - entry = new BlockEntry(rowSize, colSize); + entry = new MatrixBlockEntry(rowSize, colSize); return entry; } @@ -354,7 +338,7 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto * * Get affinity key for the given id. */ - private IgniteUuid getAffinityKey(long blockIdRow, long blockIdCol) { + private UUID getAffinityKey(long blockIdRow, long blockIdCol) { return null; } @@ -368,13 +352,13 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto private void matrixSet(int a, int b, double v) { IgnitePair<Long> blockId = getBlockId(a, b); // Remote set on the primary node (where given row or column is stored locally). - ignite().compute(groupForKey(CACHE_NAME, blockId)).run(() -> { - IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); + ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, blockId)).run(() -> { + IgniteCache<MatrixBlockKey, MatrixBlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); - BlockMatrixKey key = getCacheKey(blockId.get1(), blockId.get2()); + MatrixBlockKey key = getCacheKey(blockId.get1(), blockId.get2()); // Local get. - BlockEntry block = getEntryById(blockId); + MatrixBlockEntry block = getEntryById(blockId); block.set(a % block.rowSize(), b % block.columnSize(), v); @@ -402,13 +386,13 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto */ private double matrixGet(int a, int b) { // Remote get from the primary node (where given row or column is stored locally). - return ignite().compute(groupForKey(CACHE_NAME, getBlockId(a, b))).call(() -> { - IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); + return ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, getBlockId(a, b))).call(() -> { + IgniteCache<MatrixBlockKey, MatrixBlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); - BlockMatrixKey key = getCacheKey(getBlockId(a, b)); + MatrixBlockKey key = getCacheKey(getBlockId(a, b)); // Local get. - BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY); + MatrixBlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY); if (block == null) block = cache.get(key); @@ -420,8 +404,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto /** * Create new ML cache if needed. */ - private IgniteCache<BlockMatrixKey, BlockEntry> newCache() { - CacheConfiguration<BlockMatrixKey, BlockEntry> cfg = new CacheConfiguration<>(); + private IgniteCache<MatrixBlockKey, MatrixBlockEntry> newCache() { + CacheConfiguration<MatrixBlockKey, MatrixBlockEntry> cfg = new CacheConfiguration<>(); // Write to primary. cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockVectorStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockVectorStorage.java new file mode 100644 index 0000000..a44ed8e --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockVectorStorage.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.matrix; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.util.lang.IgnitePair; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.VectorStorage; +import org.apache.ignite.ml.math.distributed.CacheUtils; +import org.apache.ignite.ml.math.distributed.DistributedStorage; +import org.apache.ignite.ml.math.distributed.keys.impl.VectorBlockKey; +import org.apache.ignite.ml.math.impls.vector.SparseBlockDistributedVector; +import org.apache.ignite.ml.math.impls.vector.VectorBlockEntry; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.*; + +import static org.apache.ignite.ml.math.impls.matrix.MatrixBlockEntry.MAX_BLOCK_SIZE; + +/** + * Storage for {@link SparseBlockDistributedVector}. + */ +public class BlockVectorStorage extends CacheUtils implements VectorStorage, StorageConstants, DistributedStorage<VectorBlockKey> { + /** Cache name used for all instances of {@link BlockVectorStorage}. */ + private static final String CACHE_NAME = "ML_BLOCK_SPARSE_MATRICES_CONTAINER"; + /** */ + private int blocks; + /** Amount of columns in the vector. */ + private int size; + /** Matrix uuid. */ + private UUID uuid; + /** Block size about 8 KB of data. */ + private int maxBlockEdge = MAX_BLOCK_SIZE; + + /** Actual distributed storage. */ + private IgniteCache< + VectorBlockKey /* Matrix block number with uuid. */, + VectorBlockEntry /* Block of matrix, local sparse matrix. */ + > cache = null; + + /** + * + */ + public BlockVectorStorage() { + // No-op. + } + + /** + * @param size Amount of columns in the vector. + */ + public BlockVectorStorage(int size) { + + assert size > 0; + + this.size = size; + + this.blocks = size % maxBlockEdge == 0 ? size / maxBlockEdge : size / maxBlockEdge + 1; + + cache = newCache(); + + uuid = UUID.randomUUID(); + } + + /** + * + */ + public IgniteCache<VectorBlockKey, VectorBlockEntry> cache() { + return cache; + } + + /** {@inheritDoc} */ + @Override public double get(int x) { + return matrixGet(x); + } + + /** {@inheritDoc} */ + @Override public void set(int x, double v) { + matrixSet(x, v); + } + + /** {@inheritDoc} */ + @Override public int size() { + return size; + } + + + /** + * @return Blocks in row. + */ + public int blocksInRow() { + return blocks; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(size); + out.writeInt(blocks); + out.writeObject(uuid); + out.writeUTF(cache.getName()); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + size = in.readInt(); + blocks = in.readInt(); + uuid = (UUID) in.readObject(); + + cache = ignite().getOrCreateCache(in.readUTF()); + } + + /** {@inheritDoc} */ + @Override public boolean isSequentialAccess() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean isDense() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean isRandomAccess() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isDistributed() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isArrayBased() { + return false; + } + + /** Delete all data from cache. */ + @Override public void destroy() { + cache.clearAll(getAllKeys()); + } + + /** + * Get storage UUID. + * + * @return storage UUID. + */ + public UUID getUUID() { + return uuid; + } + + /** + * Build the cache key for the given blocks id. + * + * NB: NOT cell indices. + */ + public VectorBlockKey getCacheKey(long blockId) { + return new VectorBlockKey(blockId, uuid, getAffinityKey(blockId)); + } + + + /** {@inheritDoc} */ + @Override public Set<VectorBlockKey> getAllKeys() { + int maxIndex = size - 1; + long maxBlockId = getBlockId(maxIndex); + + Set<VectorBlockKey> keyset = new HashSet<>(); + + for (int i = 0; i <= maxBlockId; i++) + keyset.add(getCacheKey(i)); + + return keyset; + } + + /** {@inheritDoc} */ + @Override public String cacheName() { + return CACHE_NAME; + } + + + /** + * Get column for current block. + * + * @param blockId block id. + * @return The list of block entries. + */ + public List<VectorBlockEntry> getColForBlock(long blockId) { + List<VectorBlockEntry> res = new LinkedList<>(); + + for (int i = 0; i < blocks; i++) + res.add(getEntryById(i)); + + return res; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + res = res * 37 + size; + res = res * 37 + uuid.hashCode(); + res = res * 37 + cache.hashCode(); + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (this == obj) + return true; + + if (obj == null || getClass() != obj.getClass()) + return false; + + BlockVectorStorage that = (BlockVectorStorage) obj; + + return size == that.size && uuid.equals(that.uuid) + && (cache != null ? cache.equals(that.cache) : that.cache == null); + } + + /** + * + */ + private VectorBlockEntry getEntryById(long blockId) { + VectorBlockKey key = getCacheKey(blockId); + + VectorBlockEntry entry = cache.localPeek(key, CachePeekMode.PRIMARY); + entry = entry != null ? entry : cache.get(key); + + if (entry == null) + entry = getEmptyBlockEntry(blockId); + + return entry; + } + + @NotNull + private VectorBlockEntry getEmptyBlockEntry(long blockId) { + VectorBlockEntry entry; + int colMod = size % maxBlockEdge; + + int colSize; + + if (colMod == 0) + colSize = maxBlockEdge; + else + colSize = blockId != (blocks - 1) ? maxBlockEdge : colMod; + + entry = new VectorBlockEntry(colSize); + return entry; + } + + /** + * TODO: IGNITE-5646, WIP + * + * Get affinity key for the given id. + */ + private UUID getAffinityKey(long blockId) { + return null; + } + + /** + * Distributed matrix set. + * + * @param idx Row or column index. + * @param v New value to set. + */ + private void matrixSet(int idx, double v) { + long blockId = getBlockId(idx); + // Remote set on the primary node (where given row or column is stored locally). + ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, blockId)).run(() -> { + IgniteCache<VectorBlockKey, VectorBlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); + + VectorBlockKey key = getCacheKey(blockId); + + // Local get. + VectorBlockEntry block = getEntryById(blockId); + + block.set(idx % block.size(), v); + + // Local put. + cache.put(key, block); + }); + } + + /** */ + private long getBlockId(int x) { + return (long) x / maxBlockEdge; + } + + /** + * Distributed vector get. + * + * @param idx index. + * @return Vector value at (idx) index. + */ + private double matrixGet(int idx) { + // Remote get from the primary node (where given row or column is stored locally). + return ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, getBlockId(idx))).call(() -> { + IgniteCache<VectorBlockKey, VectorBlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); + + VectorBlockKey key = getCacheKey(getBlockId(idx)); + + // Local get. + VectorBlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY); + + if (block == null) + block = cache.get(key); + + return block == null ? 0.0 : block.get(idx % block.size()); + }); + } + + /** + * Create new ML cache if needed. + */ + private IgniteCache<VectorBlockKey, VectorBlockEntry> newCache() { + CacheConfiguration<VectorBlockKey, VectorBlockEntry> cfg = new CacheConfiguration<>(); + + // Write to primary. + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC); + + // Atomic transactions only. + cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + + // No eviction. + cfg.setEvictionPolicy(null); + + // No copying of values. + cfg.setCopyOnRead(false); + + // Cache is partitioned. + cfg.setCacheMode(CacheMode.PARTITIONED); + + // Random cache name. + cfg.setName(CACHE_NAME); + + return Ignition.localIgnite().getOrCreateCache(cfg); + } + + /** + * Avoid this method for large vectors + * + * @return data presented as array + */ + @Override public double[] data() { + double[] res = new double[this.size]; + for (int i = 0; i < this.size; i++) res[i] = this.get(i); + return res; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java index 4648421..91db30e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java @@ -17,13 +17,14 @@ package org.apache.ignite.ml.math.impls.storage.matrix; +import org.apache.ignite.internal.util.GridArgumentCheck; +import org.apache.ignite.ml.math.VectorStorage; + import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.Map; import java.util.Set; -import org.apache.ignite.internal.util.GridArgumentCheck; -import org.apache.ignite.ml.math.VectorStorage; /** * Storage for wrapping given map. http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java index c40e73d..e976899 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java @@ -19,14 +19,6 @@ package org.apache.ignite.ml.math.impls.storage.matrix; import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2DoubleRBTreeMap; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.ignite.IgniteCache; import org.apache.ignite.Ignition; import org.apache.ignite.cache.CacheAtomicityMode; @@ -42,6 +34,15 @@ import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey; import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey; import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + /** * {@link MatrixStorage} implementation for {@link SparseDistributedMatrix}. */ @@ -161,7 +162,7 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix */ private double matrixGet(int a, int b) { // Remote get from the primary node (where given row or column is stored locally). - return ignite().compute(groupForKey(CACHE_NAME, a)).call(() -> { + return ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, a)).call(() -> { IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); // Local get. @@ -183,7 +184,7 @@ public class SparseDistributedMatrixStorage extends CacheUtils implements Matrix */ private void matrixSet(int a, int b, double v) { // Remote set on the primary node (where given row or column is stored locally). - ignite().compute(groupForKey(CACHE_NAME, a)).run(() -> { + ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, a)).run(() -> { IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); // Local get. http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorage.java new file mode 100644 index 0000000..8f79413 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorage.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.vector; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.VectorStorage; +import org.apache.ignite.ml.math.distributed.CacheUtils; +import org.apache.ignite.ml.math.distributed.DistributedStorage; +import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey; +import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * {@link VectorStorage} implementation for {@link /*SparseDistributedVector}. + */ +public class SparseDistributedVectorStorage extends CacheUtils implements VectorStorage, StorageConstants, DistributedStorage<RowColMatrixKey> { + /** Cache name used for all instances of {@link SparseDistributedVectorStorage}. */ + private static final String CACHE_NAME = "ML_SPARSE_VECTORS_CONTAINER"; + /** Amount of elements in the vector. */ + private int size; + /** Random or sequential access mode. */ + private int acsMode; + /** Matrix uuid. */ + private UUID uuid; + + /** Actual distributed storage. */ + private IgniteCache<RowColMatrixKey, Double> cache = null; + + /** + * + */ + public SparseDistributedVectorStorage() { + // No-op. + } + + /** + * @param size Amount of elements in the vector. + * @param acsMode Random or sequential access mode. + */ + public SparseDistributedVectorStorage(int size, int acsMode) { + + assert size > 0; + assertAccessMode(acsMode); + + this.size = size; + this.acsMode = acsMode; + + cache = newCache(); + + uuid = UUID.randomUUID(); + } + + /** + * Create new ML cache if needed. + */ + private IgniteCache<RowColMatrixKey, Double> newCache() { + CacheConfiguration<RowColMatrixKey, Double> cfg = new CacheConfiguration<>(); + + // Write to primary. + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC); + + // Atomic transactions only. + cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + + // No eviction. + cfg.setEvictionPolicy(null); + + // No copying of values. + cfg.setCopyOnRead(false); + + // Cache is partitioned. + cfg.setCacheMode(CacheMode.PARTITIONED); + + // Random cache name. + cfg.setName(CACHE_NAME); + + return Ignition.localIgnite().getOrCreateCache(cfg); + } + + /** + * Gets cache + * + * @return cache + */ + public IgniteCache<RowColMatrixKey, Double> cache() { + return cache; + } + + /** + * Gets access mode + * + * @return code of access mode + */ + public int accessMode() { + return acsMode; + } + + /** + * Gets vector element by element index + * + * @param i Vector element index. + * @return vector element + */ + @Override public double get(int i) { + // Remote get from the primary node (where given row or column is stored locally). + return ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, getCacheKey(i))).call(() -> { + IgniteCache<RowColMatrixKey, Double> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); + Double res = cache.get(getCacheKey(i)); + if (res == null) return 0.0; + return res; + }); + } + + /** + * Sets vector element by index + * + * @param i Vector element index. + * @param v Value to set at given index. + */ + @Override public void set(int i, double v) { + // Remote set on the primary node (where given row or column is stored locally). + ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, getCacheKey(i))).run(() -> { + IgniteCache<RowColMatrixKey, Double> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME); + + RowColMatrixKey cacheKey = getCacheKey(i); + + if (v != 0.0) + cache.put(cacheKey, v); + else if (cache.containsKey(cacheKey)) // remove zero elements + cache.remove(cacheKey); + + }); + } + + + /** {@inheritDoc} */ + @Override public int size() { + return size; + } + + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(size); + out.writeInt(acsMode); + out.writeObject(uuid); + out.writeUTF(cache.getName()); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + size = in.readInt(); + acsMode = in.readInt(); + uuid = (UUID) in.readObject(); + cache = ignite().getOrCreateCache(in.readUTF()); + } + + /** {@inheritDoc} */ + @Override public boolean isSequentialAccess() { + return acsMode == SEQUENTIAL_ACCESS_MODE; + } + + /** {@inheritDoc} */ + @Override public boolean isDense() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean isRandomAccess() { + return acsMode == RANDOM_ACCESS_MODE; + } + + /** {@inheritDoc} */ + @Override public boolean isDistributed() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isArrayBased() { + return false; + } + + /** Delete all data from cache. */ + @Override public void destroy() { + Set<RowColMatrixKey> keyset = IntStream.range(0, size).mapToObj(this::getCacheKey).collect(Collectors.toSet()); + cache.clearAll(keyset); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + res = res * 37 + size; + res = res * 37 + acsMode; + res = res * 37 + uuid.hashCode(); + res = res * 37 + cache.hashCode(); + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (this == obj) + return true; + + if (obj == null || getClass() != obj.getClass()) + return false; + + SparseDistributedVectorStorage that = (SparseDistributedVectorStorage) obj; + + return size == that.size && acsMode == that.acsMode + && uuid.equals(that.uuid) && (cache != null ? cache.equals(that.cache) : that.cache == null); + } + + /** + * Builds cache key for vector element + * + * @param idx Index + * @return RowColMatrixKey + */ + public RowColMatrixKey getCacheKey(int idx) { + return new SparseMatrixKey(idx, uuid, null); + } + + /** {@inheritDoc} */ + @Override public Set<RowColMatrixKey> getAllKeys() { + int range = size; + + return IntStream.range(0, range).mapToObj(i -> new SparseMatrixKey(i, getUUID(), null)).collect(Collectors.toSet()); + } + + /** {@inheritDoc} */ + @Override public String cacheName() { + return CACHE_NAME; + } + + /** */ + public UUID getUUID() { + return uuid; + } + + @Override + public double[] data() { + double[] result = new double[this.size]; + for (int i = 0; i < this.size; i++) result[i] = this.get(i); + return result; + } + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVector.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVector.java new file mode 100644 index 0000000..e460f9f --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVector.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.vector; + +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.distributed.CacheUtils; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockVectorStorage; +import org.apache.ignite.ml.math.impls.storage.vector.SparseDistributedVectorStorage; + +import java.util.UUID; + +/** + * Sparse distributed vector implementation based on data grid. + * <p> + * Unlike {@link CacheVector} that is based on existing cache, this implementation creates distributed + * cache internally and doesn't rely on pre-existing cache.</p> + * <p> + * You also need to call {@link #destroy()} to remove the underlying cache when you no longer need this + * vector.</p> + * <p> + * <b>Currently fold supports only commutative operations.<b/></p> + */ +public class SparseBlockDistributedVector extends AbstractVector implements StorageConstants { + /** + * + */ + public SparseBlockDistributedVector() { + // No-op. + } + + /** + * @param size Vector size + */ + public SparseBlockDistributedVector(int size) { + + assert size > 0; + setStorage(new BlockVectorStorage(size)); + } + + + /** + * @param data Data to fill storage + */ + public SparseBlockDistributedVector(double[] data) { + setStorage(new BlockVectorStorage(data.length)); + for (int i = 0; i < data.length; i++) { + double val = data[i]; + if (val != 0.0) storage().set(i, val); + } + } + + + /** */ + public BlockVectorStorage storage() { + return (BlockVectorStorage) getStorage(); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param d Value to divide to. + */ + @Override public Vector divide(double d) { + return mapOverValues(v -> v / d); + } + + @Override public Vector like(int size) { + return new SparseBlockDistributedVector(size); + } + + @Override public Matrix likeMatrix(int rows, int cols) { + return new SparseBlockDistributedMatrix(rows, cols); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to add. + */ + @Override public Vector plus(double x) { + return mapOverValues(v -> v + x); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to multiply. + */ + @Override public Vector times(double x) { + return mapOverValues(v -> v * x); + } + + + /** {@inheritDoc} */ + @Override public Vector assign(double val) { + return mapOverValues(v -> val); + } + + /** {@inheritDoc} */ + @Override public Vector map(IgniteDoubleFunction<Double> fun) { + return mapOverValues(fun); + } + + /** + * @param mapper Mapping function. + * @return Vector with mapped values. + */ + private Vector mapOverValues(IgniteDoubleFunction<Double> mapper) { + CacheUtils.sparseMapForVector(getUUID(), mapper, storage().cacheName()); + + return this; + } + + /** */ + public UUID getUUID() { + return ((BlockVectorStorage) getStorage()).getUUID(); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVector.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVector.java new file mode 100644 index 0000000..b773bfa --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVector.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.vector; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.distributed.CacheUtils; +import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey; +import org.apache.ignite.ml.math.exceptions.CardinalityException; +import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.impls.matrix.*; +import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; +import org.apache.ignite.ml.math.impls.storage.vector.SparseDistributedVectorStorage; + +import java.util.Collection; +import java.util.Map; +import java.util.UUID; + +/** + * Sparse distributed vector implementation based on data grid. + * <p> + * Unlike {@link CacheVector} that is based on existing cache, this implementation creates distributed + * cache internally and doesn't rely on pre-existing cache.</p> + * <p> + * You also need to call {@link #destroy()} to remove the underlying cache when you no longer need this + * vector.</p> + * <p> + * <b>Currently fold supports only commutative operations.<b/></p> + */ +public class SparseDistributedVector extends AbstractVector implements StorageConstants { + /** + * + */ + public SparseDistributedVector() { + // No-op. + } + + /** + * @param size Vector size. + * @param acsMode Vector elements access mode.. + */ + public SparseDistributedVector(int size, int acsMode) { + + assert size > 0; + assertAccessMode(acsMode); + + + setStorage(new SparseDistributedVectorStorage(size, acsMode)); + } + + public SparseDistributedVector(int size) { + this(size, StorageConstants.RANDOM_ACCESS_MODE); + } + + /** + * @param data + */ + public SparseDistributedVector(double[] data) { + setStorage(new SparseDistributedVectorStorage(data.length, StorageConstants.RANDOM_ACCESS_MODE)); + for (int i = 0; i < data.length; i++) { + double value = data[i]; + if (value != 0.0) storage().set(i, value); + } + } + + + /** */ + public SparseDistributedVectorStorage storage() { + return (SparseDistributedVectorStorage) getStorage(); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param d Value to divide to. + */ + @Override public Vector divide(double d) { + return mapOverValues(v -> v / d); + } + + @Override + public Vector like(int size) { + return new SparseDistributedVector(size, storage().accessMode()); + } + + @Override + public Matrix likeMatrix(int rows, int cols) { + return new SparseDistributedMatrix(rows, cols); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to add. + */ + @Override public Vector plus(double x) { + return mapOverValues(v -> v + x); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to multiply. + */ + @Override public Vector times(double x) { + return mapOverValues(v -> v * x); + } + + + /** {@inheritDoc} */ + @Override public Vector assign(double val) { + return mapOverValues(v -> val); + } + + /** {@inheritDoc} */ + @Override public Vector map(IgniteDoubleFunction<Double> fun) { + return mapOverValues(fun); + } + + /** + * @param mapper Mapping function. + * @return Vector with mapped values. + */ + private Vector mapOverValues(IgniteDoubleFunction<Double> mapper) { + CacheUtils.sparseMapForVector(getUUID(), mapper, storage().cacheName()); + + return this; + } + + /** */ + public UUID getUUID() { + return ((SparseDistributedVectorStorage) getStorage()).getUUID(); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorBlockEntry.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorBlockEntry.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorBlockEntry.java new file mode 100644 index 0000000..ad795c4 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorBlockEntry.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.vector; + + +import org.apache.ignite.ml.math.Vector; + + +/** + * Block for {@link SparseBlockDistributedVector}. + */ +public final class VectorBlockEntry extends SparseLocalVector { + /** Max block size. */ + public static final int MAX_BLOCK_SIZE = 32; + + /** */ + public VectorBlockEntry() { + // No-op. + } + + /** */ + public VectorBlockEntry(int size) { + super(size, RANDOM_ACCESS_MODE); + assert size <= MAX_BLOCK_SIZE; + } + + /** */ + public VectorBlockEntry(Vector v) { + assert v.size() <= MAX_BLOCK_SIZE; + + setStorage(v.getStorage()); + } + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java index c0a57d7..0ab568c 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java @@ -71,7 +71,7 @@ public class MatrixUtil { * @return Like matrix. */ public static Matrix like(Matrix matrix, int rows, int cols) { - if (isCopyLikeSupport(matrix) || isDistributed(matrix)) + if (isCopyLikeSupport(matrix)) return new DenseLocalOnHeapMatrix(rows, cols); else return matrix.like(rows, cols); @@ -85,7 +85,7 @@ public class MatrixUtil { * @return Like vector. */ public static Vector likeVector(Matrix matrix, int crd) { - if (isCopyLikeSupport(matrix) || isDistributed(matrix)) + if (isCopyLikeSupport(matrix)) return new DenseLocalOnHeapVector(crd); else return matrix.likeVector(crd); http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java index 5dc860c..784a455 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java @@ -21,7 +21,10 @@ import org.apache.ignite.ml.math.impls.matrix.CacheMatrixTest; import org.apache.ignite.ml.math.impls.matrix.SparseDistributedBlockMatrixTest; import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrixTest; import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorageTest; +import org.apache.ignite.ml.math.impls.storage.vector.SparseDistributedVectorStorageTest; import org.apache.ignite.ml.math.impls.vector.CacheVectorTest; +import org.apache.ignite.ml.math.impls.vector.SparseBlockDistributedVectorTest; +import org.apache.ignite.ml.math.impls.vector.SparseDistributedVectorTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -30,11 +33,14 @@ import org.junit.runners.Suite; */ @RunWith(Suite.class) @Suite.SuiteClasses({ - CacheVectorTest.class, - CacheMatrixTest.class, - SparseDistributedMatrixStorageTest.class, - SparseDistributedMatrixTest.class, - SparseDistributedBlockMatrixTest.class + CacheVectorTest.class, + CacheMatrixTest.class, + SparseDistributedMatrixStorageTest.class, + SparseDistributedMatrixTest.class, + SparseDistributedBlockMatrixTest.class, + SparseDistributedVectorStorageTest.class, + SparseDistributedVectorTest.class, + SparseBlockDistributedVectorTest.class }) public class MathImplDistributedTestSuite { // No-op. http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java index fd6ed78..b4f5c2d 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java @@ -17,26 +17,23 @@ package org.apache.ignite.ml.math.impls.matrix; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.util.Collection; -import java.util.Set; -import java.util.UUID; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.distributed.DistributedStorage; -import org.apache.ignite.ml.math.distributed.keys.impl.BlockMatrixKey; -import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; +import org.apache.ignite.ml.math.distributed.keys.impl.MatrixBlockKey; import org.apache.ignite.ml.math.impls.MathTestConstants; import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage; +import org.apache.ignite.ml.math.impls.vector.SparseBlockDistributedVector; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.testframework.junits.common.GridCommonTest; +import java.io.*; +import java.util.Collection; +import java.util.Set; + import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL; /** @@ -207,14 +204,14 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); - try { - cacheMatrix.copy(); - fail("UnsupportedOperationException expected."); - } - catch (UnsupportedOperationException e) { - return; - } - fail("UnsupportedOperationException expected."); + cacheMatrix.set(rows-1, cols -1, 1); + + + Matrix newMatrix = cacheMatrix.copy(); + assert newMatrix.columnSize() == cols; + assert newMatrix.rowSize() == rows; + assert newMatrix.get(rows-1,cols-1) == 1; + } /** Test cache behaviour for matrix with different blocks */ @@ -226,7 +223,7 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { /** Test cache behaviour for matrix with homogeneous blocks */ public void testCacheBehaviourWithHomogeneousBlocks(){ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); - int size = BlockEntry.MAX_BLOCK_SIZE * 3; + int size = MatrixBlockEntry.MAX_BLOCK_SIZE * 3; cacheBehaviorLogic(size); } @@ -242,10 +239,10 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { assert cacheNames.contains(((DistributedStorage)cacheMatrix1.getStorage()).cacheName()); - IgniteCache<BlockMatrixKey, Object> cache = ignite.getOrCreateCache(((DistributedStorage)cacheMatrix1.getStorage()).cacheName()); + IgniteCache<MatrixBlockKey, Object> cache = ignite.getOrCreateCache(((DistributedStorage)cacheMatrix1.getStorage()).cacheName()); - Set<BlockMatrixKey> keySet1 = buildKeySet(cacheMatrix1); - Set<BlockMatrixKey> keySet2 = buildKeySet(cacheMatrix2); + Set<MatrixBlockKey> keySet1 = buildKeySet(cacheMatrix1); + Set<MatrixBlockKey> keySet2 = buildKeySet(cacheMatrix2); assert cache.containsKeys(keySet1); assert cache.containsKeys(keySet2); @@ -275,14 +272,10 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); - try { - cacheMatrix.likeVector(1); - fail("UnsupportedOperationException expected."); - } - catch (UnsupportedOperationException e) { - return; - } - fail("UnsupportedOperationException expected."); + Vector v = cacheMatrix.likeVector(1); + assert v.size() == 1; + assert v instanceof SparseBlockDistributedVector; + } /** @@ -298,7 +291,7 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { * Simple test for two square matrices with size which is proportional to MAX_BLOCK_SIZE constant */ public void testSquareMatrixTimesWithHomogeneousBlocks(){ - int size = BlockEntry.MAX_BLOCK_SIZE * 3; + int size = MatrixBlockEntry.MAX_BLOCK_SIZE * 3; squareMatrixTimesLogic(size); } @@ -331,8 +324,8 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { public void testNonSquareMatrixTimes(){ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); - int size = BlockEntry.MAX_BLOCK_SIZE + 1; - int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1; + int size = MatrixBlockEntry.MAX_BLOCK_SIZE + 1; + int size2 = MatrixBlockEntry.MAX_BLOCK_SIZE * 2 + 1; Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size2, size); Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size, size2); @@ -358,8 +351,8 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { public void testNonSquareMatrixTimes2(){ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); - int size = BlockEntry.MAX_BLOCK_SIZE + 1; - int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1; + int size = MatrixBlockEntry.MAX_BLOCK_SIZE + 1; + int size2 = MatrixBlockEntry.MAX_BLOCK_SIZE * 2 + 1; Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size, size2); Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size2, size); @@ -379,6 +372,26 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION); } + public void testMatrixVectorTimes(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + SparseBlockDistributedMatrix a = new SparseBlockDistributedMatrix(new double[][] {{2.0, 4.0, 0.0}, {-2.0, 1.0, 3.0}, {-1.0, 0.0, 1.0}}); + SparseBlockDistributedVector b = new SparseBlockDistributedVector(new double[] {1.0, 2.0, -1.0}); + SparseBlockDistributedVector result = new SparseBlockDistributedVector(new double[] {10, -3.0, -2.0}); + + + Vector calculatedResult = a.times(b); + + for(int i = 0; i < calculatedResult.size(); i++) + assertEquals(UNEXPECTED_VAL + " for "+ i, result.get(i), calculatedResult.get(i), PRECISION); + + + } + + + + + /** */ private void initMtx(Matrix m) { for (int i = 0; i < m.rowSize(); i++) @@ -387,7 +400,8 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { } /** Build key set for SparseBlockDistributedMatrix. */ - private Set<BlockMatrixKey> buildKeySet(SparseBlockDistributedMatrix m){ + private Set<MatrixBlockKey> buildKeySet(SparseBlockDistributedMatrix m){ + BlockMatrixStorage storage = (BlockMatrixStorage)m.getStorage(); return storage.getAllKeys(); http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java index 1955588..a9343f4 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java @@ -30,11 +30,14 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.distributed.DistributedStorage; import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey; import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; import org.apache.ignite.ml.math.impls.MathTestConstants; import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; +import org.apache.ignite.ml.math.impls.vector.SparseBlockDistributedVector; +import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.testframework.junits.common.GridCommonTest; @@ -212,14 +215,13 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest { cacheMatrix = new SparseDistributedMatrix(rows, cols, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE); - try { - cacheMatrix.copy(); - fail("UnsupportedOperationException expected."); - } - catch (UnsupportedOperationException e) { - return; + Matrix copiedMtx = cacheMatrix.copy(); + + for (int i = 0; i < cacheMatrix.rowSize(); i++) { + for (int j = 0; j < cacheMatrix.columnSize(); j++) { + assert copiedMtx.get(i,j) == cacheMatrix.get(i,j); + } } - fail("UnsupportedOperationException expected."); } /** */ @@ -274,14 +276,9 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest { cacheMatrix = new SparseDistributedMatrix(rows, cols, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE); - try { - cacheMatrix.likeVector(1); - fail("UnsupportedOperationException expected."); - } - catch (UnsupportedOperationException e) { - return; - } - fail("UnsupportedOperationException expected."); + Vector v = cacheMatrix.likeVector(1); + assert v.size() == 1; + assert v instanceof SparseDistributedVector; } /** */ http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorageTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorageTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorageTest.java new file mode 100644 index 0000000..9b6aa32 --- /dev/null +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/vector/SparseDistributedVectorStorageTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.vector; + +import org.apache.ignite.Ignite; +import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.impls.MathTestConstants; +import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.junits.common.GridCommonTest; + +/** + * Tests for {@link SparseDistributedVectorStorage}. + */ +@GridCommonTest(group = "Distributed Models") +public class SparseDistributedVectorStorageTest extends GridCommonAbstractTest { + /** Number of nodes in grid */ + private static final int NODE_COUNT = 3; + /** Cache name. */ + private static final String CACHE_NAME = "test-cache"; + /** */ + private static final String UNEXPECTED_ATTRIBUTE_VALUE = "Unexpected attribute value."; + /** Grid instance. */ + private Ignite ignite; + + /** + * Default constructor. + */ + public SparseDistributedVectorStorageTest() { + super(false); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + for (int i = 1; i <= NODE_COUNT; i++) + startGrid(i); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * {@inheritDoc} + */ + @Override protected void beforeTest() throws Exception { + ignite = grid(NODE_COUNT); + + ignite.configuration().setPeerClassLoadingEnabled(true); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + ignite.destroyCache(CACHE_NAME); + } + + /** */ + public void testCacheCreation() throws Exception { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + final int size = MathTestConstants.STORAGE_SIZE; + + SparseDistributedVectorStorage storage = new SparseDistributedVectorStorage(size, StorageConstants.RANDOM_ACCESS_MODE); + + assertNotNull("SparseDistributedMatrixStorage cache is null.", storage.cache()); + } + + /** */ + public void testSetGet() throws Exception { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + final int size = MathTestConstants.STORAGE_SIZE; + + SparseDistributedVectorStorage storage = new SparseDistributedVectorStorage(size, StorageConstants.RANDOM_ACCESS_MODE); + + for (int i = 0; i < size; i++) { + double v = Math.random(); + storage.set(i, v); + + assert Double.compare(v, storage.get(i)) == 0; + assert Double.compare(v, storage.get(i)) == 0; + } + } + + /** */ + public void testAttributes() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + final int size = MathTestConstants.STORAGE_SIZE; + + SparseDistributedVectorStorage storage = new SparseDistributedVectorStorage(size, StorageConstants.RANDOM_ACCESS_MODE); + + assertEquals(UNEXPECTED_ATTRIBUTE_VALUE, storage.size(), size); + + assertFalse(UNEXPECTED_ATTRIBUTE_VALUE, storage.isArrayBased()); + assertFalse(UNEXPECTED_ATTRIBUTE_VALUE, storage.isDense()); + assertTrue(UNEXPECTED_ATTRIBUTE_VALUE, storage.isDistributed()); + + assertEquals(UNEXPECTED_ATTRIBUTE_VALUE, storage.isRandomAccess(), !storage.isSequentialAccess()); + assertTrue(UNEXPECTED_ATTRIBUTE_VALUE, storage.isRandomAccess()); + + } + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVectorTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVectorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVectorTest.java new file mode 100644 index 0000000..4ac2845 --- /dev/null +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseBlockDistributedVectorTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.vector; + +import org.apache.ignite.Ignite; +import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.impls.MathTestConstants; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.junits.common.GridCommonTest; + +import java.io.*; + +import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL; + +/** + * Tests for {@link SparseDistributedVector}. + */ +@GridCommonTest(group = "Distributed Models") +public class SparseBlockDistributedVectorTest extends GridCommonAbstractTest { + /** Number of nodes in grid */ + private static final int NODE_COUNT = 3; + /** Precision. */ + private static final double PRECISION = 0.0; + /** Grid instance. */ + private Ignite ignite; + /** Vector size */ + private final int size = MathTestConstants.STORAGE_SIZE; + /** Vector for tests */ + private SparseBlockDistributedVector sparseBlockDistributedVector; + + /** + * Default constructor. + */ + public SparseBlockDistributedVectorTest() { + super(false); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + for (int i = 1; i <= NODE_COUNT; i++) + startGrid(i); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * {@inheritDoc} + */ + @Override protected void beforeTest() throws Exception { + ignite = grid(NODE_COUNT); + + ignite.configuration().setPeerClassLoadingEnabled(true); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + if (sparseBlockDistributedVector != null) { + sparseBlockDistributedVector.destroy(); + sparseBlockDistributedVector = null; + } + } + + /** */ + public void testGetSet() throws Exception { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseBlockDistributedVector = new SparseBlockDistributedVector(size); + + for (int i = 0; i < size; i++) { + double v = Math.random(); + sparseBlockDistributedVector.set(i, v); + assertEquals("Unexpected value for vector element[" + i + "]", v, sparseBlockDistributedVector.get(i), PRECISION); + } + } + + /** */ + public void testExternalize() throws IOException, ClassNotFoundException { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseBlockDistributedVector = new SparseBlockDistributedVector(size); + + sparseBlockDistributedVector.set(1, 1.0); + + ByteArrayOutputStream byteArrOutputStream = new ByteArrayOutputStream(); + ObjectOutputStream objOutputStream = new ObjectOutputStream(byteArrOutputStream); + + objOutputStream.writeObject(sparseBlockDistributedVector); + + ByteArrayInputStream byteArrInputStream = new ByteArrayInputStream(byteArrOutputStream.toByteArray()); + ObjectInputStream objInputStream = new ObjectInputStream(byteArrInputStream); + + SparseBlockDistributedVector objRestored = (SparseBlockDistributedVector)objInputStream.readObject(); + + assertTrue(MathTestConstants.VAL_NOT_EQUALS, sparseBlockDistributedVector.equals(objRestored)); + assertEquals(MathTestConstants.VAL_NOT_EQUALS, objRestored.get(1), 1.0, PRECISION); + } + + /** Test simple math. */ + public void testMath() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseBlockDistributedVector = new SparseBlockDistributedVector(size); + initVector(sparseBlockDistributedVector); + + sparseBlockDistributedVector.assign(2.0); + for (int i = 0; i < sparseBlockDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 2.0, sparseBlockDistributedVector.get(i), PRECISION); + + sparseBlockDistributedVector.plus(3.0); + for (int i = 0; i < sparseBlockDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 5.0, sparseBlockDistributedVector.get(i), PRECISION); + + sparseBlockDistributedVector.times(2.0); + for (int i = 0; i < sparseBlockDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 10.0, sparseBlockDistributedVector.get(i), PRECISION); + + sparseBlockDistributedVector.divide(10.0); + for (int i = 0; i < sparseBlockDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 1.0, sparseBlockDistributedVector.get(i), PRECISION); + } + + + /** */ + public void testMap() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseBlockDistributedVector = new SparseBlockDistributedVector(size); + initVector(sparseBlockDistributedVector); + + sparseBlockDistributedVector.map(i -> 100.0); + for (int i = 0; i < sparseBlockDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 100.0, sparseBlockDistributedVector.get(i), PRECISION); + } + + /** */ + public void testCopy() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseBlockDistributedVector = new SparseBlockDistributedVector(size); + + Vector cp = sparseBlockDistributedVector.copy(); + assertNotNull(cp); + for (int i = 0; i < size; i++) + assertEquals(UNEXPECTED_VAL, cp.get(i), sparseBlockDistributedVector.get(i), PRECISION); + } + + /** */ + public void testLike() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseBlockDistributedVector = new SparseBlockDistributedVector(size); + + assertNotNull(sparseBlockDistributedVector.like(1)); + } + + + /** */ + private void initVector(Vector v) { + for (int i = 0; i < v.size(); i++) + v.set(i, 1.0); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b0a86018/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVectorTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVectorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVectorTest.java new file mode 100644 index 0000000..416e254 --- /dev/null +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/SparseDistributedVectorTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.vector; + +import org.apache.ignite.Ignite; + +import org.apache.ignite.internal.util.IgniteUtils; + +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.Vector; + +import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; +import org.apache.ignite.ml.math.impls.MathTestConstants; + +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.junits.common.GridCommonTest; +import org.junit.Ignore; + +import java.io.*; + + +import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL; + +/** + * Tests for {@link SparseDistributedVector}. + */ +@GridCommonTest(group = "Distributed Models") +public class SparseDistributedVectorTest extends GridCommonAbstractTest { + /** Number of nodes in grid */ + private static final int NODE_COUNT = 3; + /** Precision. */ + private static final double PRECISION = 0.0; + /** Grid instance. */ + private Ignite ignite; + /** Vector size */ + private final int size = MathTestConstants.STORAGE_SIZE; + /** Vector for tests */ + private SparseDistributedVector sparseDistributedVector; + + /** + * Default constructor. + */ + public SparseDistributedVectorTest() { + super(false); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + for (int i = 1; i <= NODE_COUNT; i++) + startGrid(i); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * {@inheritDoc} + */ + @Override protected void beforeTest() throws Exception { + ignite = grid(NODE_COUNT); + + ignite.configuration().setPeerClassLoadingEnabled(true); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + if (sparseDistributedVector != null) { + sparseDistributedVector.destroy(); + sparseDistributedVector = null; + } + } + + /** */ + public void testGetSet() throws Exception { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseDistributedVector = new SparseDistributedVector(size, StorageConstants.RANDOM_ACCESS_MODE); + + for (int i = 0; i < size; i++) { + double v = Math.random(); + sparseDistributedVector.set(i, v); + assertEquals("Unexpected value for vector element[" + i + "]", v, sparseDistributedVector.get(i), PRECISION); + } + } + + /** */ + public void testExternalize() throws IOException, ClassNotFoundException { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseDistributedVector = new SparseDistributedVector(size, StorageConstants.RANDOM_ACCESS_MODE); + + sparseDistributedVector.set(1, 1.0); + + ByteArrayOutputStream byteArrOutputStream = new ByteArrayOutputStream(); + ObjectOutputStream objOutputStream = new ObjectOutputStream(byteArrOutputStream); + + objOutputStream.writeObject(sparseDistributedVector); + + ByteArrayInputStream byteArrInputStream = new ByteArrayInputStream(byteArrOutputStream.toByteArray()); + ObjectInputStream objInputStream = new ObjectInputStream(byteArrInputStream); + + SparseDistributedVector objRestored = (SparseDistributedVector)objInputStream.readObject(); + + assertTrue(MathTestConstants.VAL_NOT_EQUALS, sparseDistributedVector.equals(objRestored)); + assertEquals(MathTestConstants.VAL_NOT_EQUALS, objRestored.get(1), 1.0, PRECISION); + } + + /** Test simple math. */ + public void testMath() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseDistributedVector = new SparseDistributedVector(size, StorageConstants.RANDOM_ACCESS_MODE); + initVector(sparseDistributedVector); + + sparseDistributedVector.assign(2.0); + for (int i = 0; i < sparseDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 2.0, sparseDistributedVector.get(i), PRECISION); + + sparseDistributedVector.plus(3.0); + for (int i = 0; i < sparseDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 5.0, sparseDistributedVector.get(i), PRECISION); + + sparseDistributedVector.times(2.0); + for (int i = 0; i < sparseDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 10.0, sparseDistributedVector.get(i), PRECISION); + + sparseDistributedVector.divide(10.0); + for (int i = 0; i < sparseDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 1.0, sparseDistributedVector.get(i), PRECISION); + + // assertEquals(UNEXPECTED_VAL, sparseDistributedVector.rowSize() * sparseDistributedVector.columnSize(), sparseDistributedVector.sum(), PRECISION); + } + + + /** */ + public void testMap() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseDistributedVector = new SparseDistributedVector(size, StorageConstants.RANDOM_ACCESS_MODE); + initVector(sparseDistributedVector); + + sparseDistributedVector.map(i -> 100.0); + for (int i = 0; i < sparseDistributedVector.size(); i++) + assertEquals(UNEXPECTED_VAL, 100.0, sparseDistributedVector.get(i), PRECISION); + } + + /** */ + public void testCopy() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseDistributedVector = new SparseDistributedVector(size, StorageConstants.RANDOM_ACCESS_MODE); + + Vector copy = sparseDistributedVector.copy(); + assertNotNull(copy); + for (int i = 0; i < size; i++) + assertEquals(UNEXPECTED_VAL, copy.get(i), sparseDistributedVector.get(i), PRECISION); + } + + + /** */ + public void testLike() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + sparseDistributedVector = new SparseDistributedVector(size, StorageConstants.RANDOM_ACCESS_MODE); + + assertNotNull(sparseDistributedVector.like(1)); + } + + + /** */ + private void initVector(Vector v) { + for (int i = 0; i < v.size(); i++) + v.set(i, 1.0); + } +}