Author: srowen
Date: Mon Jun 27 16:09:10 2011
New Revision: 1140224
URL: http://svn.apache.org/viewvc?rev=1140224&view=rev
Log:
MAHOUT-744 iterate over SparseMatrix rows sparse-ly in iterate()
Modified:
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java
mahout/trunk/math/src/main/java/org/apache/mahout/math/SparseMatrix.java
Modified:
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java?rev=1140224&r1=1140223&r2=1140224&view=diff
==============================================================================
---
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
(original)
+++
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
Mon Jun 27 16:09:10 2011
@@ -17,6 +17,8 @@
package org.apache.mahout.math.hadoop;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -329,28 +331,18 @@ public final class TestDistributedRowMat
}
private static DistributedRowMatrix saveToFs(final Matrix m, Path
baseTmpDirPath) throws IOException {
- Configuration conf = new Configuration();
+ Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
ClusteringTestUtils.writePointsToFile(new Iterable<VectorWritable>() {
@Override
public Iterator<VectorWritable> iterator() {
- final Iterator<MatrixSlice> it = m.iterator();
- return new Iterator<VectorWritable>() {
- @Override
- public boolean hasNext() {
- return it.hasNext();
- }
- @Override
- public VectorWritable next() {
- MatrixSlice slice = it.next();
- return new VectorWritable(slice.vector());
- }
+ return Iterators.transform(m.iterator(), new
Function<MatrixSlice,VectorWritable>() {
@Override
- public void remove() {
- it.remove();
+ public VectorWritable apply(MatrixSlice input) {
+ return new VectorWritable(input.vector());
}
- };
+ });
}
}, true, new Path(baseTmpDirPath, "distMatrix/part-00000"), fs, conf);
Modified:
mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java?rev=1140224&r1=1140223&r2=1140224&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java
(original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java
Mon Jun 27 16:09:10 2011
@@ -17,6 +17,7 @@
package org.apache.mahout.math;
+import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Maps;
import org.apache.mahout.math.function.DoubleDoubleFunction;
import org.apache.mahout.math.function.Functions;
@@ -26,7 +27,6 @@ import org.apache.mahout.math.function.V
import java.util.Iterator;
import java.util.Map;
-import java.util.NoSuchElementException;
/** A few universal implementations of convenience functions */
public abstract class AbstractMatrix implements Matrix {
@@ -59,27 +59,16 @@ public abstract class AbstractMatrix imp
@Override
public Iterator<MatrixSlice> iterateAll() {
- return new Iterator<MatrixSlice>() {
+ return new AbstractIterator<MatrixSlice>() {
private int slice;
-
- @Override
- public boolean hasNext() {
- return slice < numSlices();
- }
-
@Override
- public MatrixSlice next() {
+ protected MatrixSlice computeNext() {
if (slice >= numSlices()) {
- throw new NoSuchElementException();
+ return endOfData();
}
int i = slice++;
return new MatrixSlice(slice(i), i);
}
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("remove() not supported for
Matrix iterator");
- }
};
}
@@ -642,25 +631,15 @@ public abstract class AbstractMatrix imp
@Override
public Iterator<Element> iterator() {
- return new Iterator<Element>() {
+ return new AbstractIterator<Element>() {
private int i;
@Override
- public boolean hasNext() {
- return i < size();
- }
-
- @Override
- public Element next() {
+ protected Element computeNext() {
if (i >= size()) {
- throw new NoSuchElementException();
+ return endOfData();
}
return getElement(i++);
}
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("Element removal not
supported");
- }
};
}
Modified:
mahout/trunk/math/src/main/java/org/apache/mahout/math/SparseMatrix.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/SparseMatrix.java?rev=1140224&r1=1140223&r2=1140224&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/SparseMatrix.java
(original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/SparseMatrix.java
Mon Jun 27 16:09:10 2011
@@ -17,12 +17,16 @@
package org.apache.mahout.math;
+import com.google.common.collect.AbstractIterator;
+import org.apache.mahout.math.list.IntArrayList;
import org.apache.mahout.math.map.OpenIntObjectHashMap;
+import java.util.Iterator;
import java.util.Map;
/** Doubly sparse matrix. Implemented as a Map of RandomAccessSparseVector
rows */
public class SparseMatrix extends AbstractMatrix {
+
private OpenIntObjectHashMap<Vector> rows;
public SparseMatrix() {
@@ -70,6 +74,25 @@ public class SparseMatrix extends Abstra
clone.rows = rows.clone();
return clone;
}
+
+ @Override
+ public Iterator<MatrixSlice> iterator() {
+ final IntArrayList keys = new IntArrayList(rows.size());
+ rows.keys(keys);
+ return new AbstractIterator<MatrixSlice>() {
+ private int slice;
+ @Override
+ protected MatrixSlice computeNext() {
+ if (slice >= rows.size()) {
+ return endOfData();
+ }
+ int i = keys.get(slice);
+ Vector row = rows.get(i);
+ slice++;
+ return new MatrixSlice(row, i);
+ }
+ };
+ }
@Override
public double getQuick(int row, int column) {
