Author: edwardyoon
Date: Tue Sep 2 01:59:52 2008
New Revision: 691160
URL: http://svn.apache.org/viewvc?rev=691160&view=rev
Log:
Multiplication
Added:
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/java/org/apache/hama/Constants.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java
incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java
incubator/hama/trunk/src/test/org/apache/hama/TestVector.java
Modified: incubator/hama/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Sep 2 01:59:52 2008
@@ -4,6 +4,7 @@
NEW FEATURES
+ HAMA-13: Matrix multiplication (edwardyoon)
HAMA-48: Add getColumn(int column) method which returns column vector
(edwardyoon)
HAMA-49: Add iterator() method to vector (edwardyoon)
HAMA-43: Color Hama Logo (Morakot via chanwit)
Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Tue Sep 2
01:59:52 2008
@@ -23,6 +23,7 @@
* Some constants used in the Hama
*/
public class Constants {
+
/**
* Meta-columnFamily to store the matrix-info
*/
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue Sep 2
01:59:52 2008
@@ -28,12 +28,14 @@
import org.apache.hadoop.hbase.client.Scanner;
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hama.algebra.AdditionMap;
import org.apache.hama.algebra.AdditionReduce;
+import org.apache.hama.algebra.MultiplicationMap;
+import org.apache.hama.algebra.MultiplicationReduce;
import org.apache.hama.mapred.DenseMap;
import org.apache.hama.mapred.MatrixReduce;
import org.apache.hama.util.Numeric;
@@ -136,13 +138,12 @@
jobConf.setJobName("addition MR job");
DenseMap.initJob(this.getName(), B.getName(), AdditionMap.class,
- ImmutableBytesWritable.class, DenseVector.class, jobConf);
+ IntWritable.class, DenseVector.class, jobConf);
MatrixReduce.initJob(C.getName(), AdditionReduce.class, jobConf);
try {
JobClient.runJob(jobConf);
} catch (IOException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
}
@@ -173,8 +174,23 @@
}
public Matrix mult(Matrix B) {
- // TODO Auto-generated method stub
- return null;
+ String output = RandomVariable.randMatrixName();
+ Matrix C = new DenseMatrix(config, output);
+
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("multiplication MR job");
+
+ DenseMap.initJob(this.getName(), B.getName(), MultiplicationMap.class,
+ IntWritable.class, DenseVector.class, jobConf);
+ MatrixReduce.initJob(C.getName(), MultiplicationReduce.class, jobConf);
+
+ try {
+ JobClient.runJob(jobConf);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return C;
}
public Matrix multAdd(double alpha, Matrix B, Matrix C) {
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java Tue Sep 2
01:59:52 2008
@@ -74,14 +74,21 @@
}
public Vector add(Vector v2) {
- HbaseMapWritable<byte[], Cell> trunk = new HbaseMapWritable<byte[],
Cell>();
+ if (this.size() == 0) {
+ DenseVector trunk = (DenseVector) v2;
+ this.row = trunk.row;
+ this.cells = trunk.cells;
+ return this;
+ }
+
for (int i = 0; i < this.size(); i++) {
double value = (this.get(i) + v2.get(i));
- Cell cValue = new Cell(String.valueOf(value),
System.currentTimeMillis());
- trunk.put(Numeric.getColumnIndex(i), cValue);
+
+ Cell cValue = new Cell(Numeric.doubleToBytes(value), now());
+ this.cells.put(Numeric.getColumnIndex(i), cValue);
}
- return new DenseVector(row, trunk);
+ return this;
}
public double dot(Vector v) {
@@ -99,18 +106,22 @@
Set<byte[]> keySet = cells.keySet();
Iterator<byte[]> it = keySet.iterator();
+ int i = 0;
while (it.hasNext()) {
byte[] key = it.next();
- double oValue = Numeric.bytesToDouble(get(key).getValue());
+ double oValue = Numeric.bytesToDouble(this.get(key).getValue());
double nValue = oValue * alpha;
- Cell cValue = new Cell(String.valueOf(nValue),
System.currentTimeMillis());
- cells.put(key, cValue);
+
+ Cell cValue = new Cell(Numeric.doubleToBytes(nValue), now());
+ this.cells.put(Numeric.getColumnIndex(i), cValue);
+ i++;
}
return this;
}
public double get(int index) {
+
return Numeric.bytesToDouble(cells.get(Numeric.getColumnIndex(index))
.getValue());
}
@@ -127,7 +138,7 @@
}
public void set(int index, double value) {
- Cell cValue = new Cell(String.valueOf(value), System.currentTimeMillis());
+ Cell cValue = new Cell(Numeric.doubleToBytes(value), now());
cells.put(Numeric.getColumnIndex(index), cValue);
}
Added:
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java?rev=691160&view=auto
==============================================================================
---
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
(added)
+++
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
Tue Sep 2 01:59:52 2008
@@ -0,0 +1,55 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseVector;
+import org.apache.hama.Vector;
+import org.apache.hama.mapred.DenseMap;
+import org.apache.hama.util.Numeric;
+import org.apache.log4j.Logger;
+
+public class MultiplicationMap extends DenseMap<IntWritable, DenseVector> {
+ static final Logger LOG = Logger.getLogger(MultiplicationMap.class);
+
+ @Override
+ public void map(IntWritable key, DenseVector value,
+ OutputCollector<IntWritable, DenseVector> output, Reporter reporter)
+ throws IOException {
+
+ Iterator<Cell> it = value.iterator();
+ int i = 0;
+ while (it.hasNext()) {
+ Cell c = it.next();
+
+ Vector v = MATRIX_B.getRow(i);
+
+ double alpha = Numeric.bytesToDouble(c.getValue());
+ output.collect(key, (DenseVector) v.scale(alpha));
+ i++;
+ }
+ }
+}
Added:
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java?rev=691160&view=auto
==============================================================================
---
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
(added)
+++
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
Tue Sep 2 01:59:52 2008
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseVector;
+import org.apache.hama.mapred.MatrixReduce;
+import org.apache.hama.util.Numeric;
+import org.apache.log4j.Logger;
+
+public class MultiplicationReduce extends
+ MatrixReduce<IntWritable, DenseVector> {
+ static final Logger LOG = Logger.getLogger(MultiplicationReduce.class);
+
+ @Override
+ public void reduce(IntWritable key, Iterator<DenseVector> values,
+ OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+ throws IOException {
+
+ BatchUpdate b = new BatchUpdate(Numeric.intToBytes(key.get()));
+ DenseVector sum;
+ Map<Integer, Double> buffer = new HashMap<Integer, Double>();
+
+ // Summation vectors
+ while (values.hasNext()) {
+ sum = values.next();
+ for (int i = 0; i < sum.size(); i++) {
+ if (buffer.containsKey(i)) {
+ buffer.put(i, sum.get(i) + buffer.get(i));
+ } else {
+ buffer.put(i, sum.get(i));
+ }
+ }
+ }
+
+ for (Map.Entry<Integer, Double> f : buffer.entrySet()) {
+ byte[] value = Numeric.doubleToBytes(f.getValue());
+ b.put(Numeric.getColumnIndex(f.getKey()), value);
+ }
+
+ output.collect(key, b);
+ }
+
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
(original)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java Tue
Sep 2 01:59:52 2008
@@ -206,4 +206,8 @@
return cell;
}
}
+
+ public long now() {
+ return System.currentTimeMillis();
+ }
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java Tue Sep 2
01:59:52 2008
@@ -26,6 +26,7 @@
* Provides a number format conversion
*/
public class Numeric {
+
/**
* Bytes to integer conversion
*
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java Tue Sep 2
01:59:52 2008
@@ -31,16 +31,14 @@
public class TestMatrix extends HamaTestCase {
/**
- * Random matrix creation test
- *
- * @throws IOException
+ * Matrix Test
*
* @throws IOException
*/
- public void testRandomMatrix() throws IOException {
+ public void testGetMatrixSize() throws IOException {
Matrix rand = DenseMatrix.random(conf, SIZE, SIZE);
assertTrue(rand.getRows() == SIZE);
-
+
getColumnTest(rand);
}
@@ -59,4 +57,55 @@
x++;
}
}
+
+ /**
+ * Test matrices addition
+ */
+ public void testMatrixAdd() {
+ Matrix rand1 = DenseMatrix.random(conf, SIZE, SIZE);
+ Matrix rand2 = DenseMatrix.random(conf, SIZE, SIZE);
+
+ Matrix result = rand1.add(rand2);
+
+ for (int i = 0; i < SIZE; i++) {
+ for (int j = 0; j < SIZE; j++) {
+ assertEquals(result.get(i, j), rand1.get(i, j) + rand2.get(i, j));
+ }
+ }
+ }
+
+ /**
+ * Test matrices multiplication
+ */
+ public void testMatrixMult() {
+ Matrix m1 = DenseMatrix.random(conf, SIZE, SIZE);
+ Matrix m2 = DenseMatrix.random(conf, SIZE, SIZE);
+
+ Matrix result = m1.mult(m2);
+
+ verifyMultResult(SIZE, m1, m2, result);
+ }
+
+ /**
+ * Verifying multiplication result
+ *
+ * @param size
+ * @param m1
+ * @param m2
+ * @param result
+ */
+ private void verifyMultResult(int size, Matrix m1, Matrix m2, Matrix result)
{
+ double[][] C = new double[SIZE][SIZE];
+
+ for (int i = 0; i < SIZE; i++)
+ for (int j = 0; j < SIZE; j++)
+ for (int k = 0; k < SIZE; k++)
+ C[i][k] += m1.get(i, j) * m2.get(j, k);
+
+ for (int i = 0; i < 2; i++) {
+ for (int j = 0; j < 2; j++) {
+ assertEquals(result.get(i, j), C[i][j]);
+ }
+ }
+ }
}
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestVector.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestVector.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestVector.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestVector.java Tue Sep 2
01:59:52 2008
@@ -34,7 +34,8 @@
/**
* Test vector
- * @throws IOException
+ *
+ * @throws IOException
*/
public void testGetVector() throws IOException {
Matrix m1 = new DenseMatrix(conf, m);
@@ -79,6 +80,11 @@
assertEquals(norm2, ((DenseVector) v1).getNorm2());
}
+ /**
+ * Test scaling
+ *
+ * @param v2
+ */
private void scalingTest(Vector v2) {
v2.scale(0.5);
@@ -87,12 +93,18 @@
}
}
+ /**
+ * Test get/set methods
+ */
public void testGetSet() {
Vector v1 = new DenseVector();
v1.set(0, 0.2);
assertEquals(v1.get(0), 0.2);
}
+ /**
+ * Test iterator
+ */
public void testIterator() {
Vector v1 = new DenseVector();
v1.set(0, 0.2);
