Author: edwardyoon
Date: Mon Sep 8 04:39:59 2008
New Revision: 693057
URL: http://svn.apache.org/viewvc?rev=693057&view=rev
Log:
NullPointerException on distributed cluster
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionMap.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/DenseMap.java
incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java
Modified: incubator/hama/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=693057&r1=693056&r2=693057&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Sep 8 04:39:59 2008
@@ -48,5 +48,6 @@
BUG FIXES
+ HAMA-53: NullPointerException on distributed cluster (edwardyoon)
HAMA-26: hama-formatter.xml should be removed (edwardyoon)
HAMA-25: Vector.get() returns double (edwardyoon)
Modified:
incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java?rev=693057&r1=693056&r2=693057&view=diff
==============================================================================
---
incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
(original)
+++
incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
Mon Sep 8 04:39:59 2008
@@ -1,70 +1,74 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.examples;
-
-import java.io.IOException;
-
-import org.apache.hama.DenseMatrix;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.Matrix;
-
-public class MatrixAddition {
-
- public static void main(String[] args) throws IOException {
- if (args.length < 2) {
- System.out.println("addition <row_m> <column_n>");
- System.exit(-1);
- }
-
- int row = Integer.parseInt(args[0]);
- int column = Integer.parseInt(args[1]);
-
- HamaConfiguration conf = new HamaConfiguration();
-
- Matrix a = DenseMatrix.random(conf, row, column);
- Matrix b = DenseMatrix.random(conf, row, column);
-
- Matrix c = a.add(b);
-
- System.out.println("\nMatrix A");
- System.out.println("----------------------");
- for(int i = 0; i < row; i++) {
- for(int j = 0; j < row; j++) {
- System.out.println(a.get(i, j));
- }
- }
-
- System.out.println("\nMatrix B");
- System.out.println("----------------------");
- for(int i = 0; i < row; i++) {
- for(int j = 0; j < row; j++) {
- System.out.println(b.get(i, j));
- }
- }
-
- System.out.println("\nC = A + B");
- System.out.println("----------------------");
- for(int i = 0; i < row; i++) {
- for(int j = 0; j < row; j++) {
- System.out.println(c.get(i, j));
- }
- }
- }
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import java.io.IOException;
+
+import org.apache.hama.DenseMatrix;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.Matrix;
+
+public class MatrixAddition {
+
+ public static void main(String[] args) throws IOException {
+ if (args.length < 2) {
+ System.out.println("addition <row_m> <column_n>");
+ System.exit(-1);
+ }
+
+ int row = Integer.parseInt(args[0]);
+ int column = Integer.parseInt(args[1]);
+
+ HamaConfiguration conf = new HamaConfiguration();
+
+ conf.set("fs.default.name", "hdfs://udanax.org:54310");
+ conf.set("hbase.master", "udanax.org:60000");
+
+ Matrix a = DenseMatrix.random(conf, row, column);
+ Matrix b = DenseMatrix.random(conf, row, column);
+
+ System.out.println("\nMatrix A");
+ System.out.println("----------------------");
+ for(int i = 0; i < row; i++) {
+ for(int j = 0; j < row; j++) {
+ System.out.println(a.get(i, j));
+ }
+ }
+
+ System.out.println("\nMatrix B");
+ System.out.println("----------------------");
+ for(int i = 0; i < row; i++) {
+ for(int j = 0; j < row; j++) {
+ System.out.println(b.get(i, j));
+ }
+ }
+ System.out.println();
+
+ Matrix c = a.add(b);
+
+ System.out.println("\nC = A + B");
+ System.out.println("----------------------");
+ for(int i = 0; i < row; i++) {
+ for(int j = 0; j < row; j++) {
+ System.out.println(c.get(i, j));
+ }
+ }
+ }
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=693057&r1=693056&r2=693057&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Mon Sep 8
04:39:59 2008
@@ -1,208 +1,207 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Scanner;
-import org.apache.hadoop.hbase.io.RowResult;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hama.algebra.AdditionMap;
-import org.apache.hama.algebra.AdditionReduce;
-import org.apache.hama.algebra.MultiplicationMap;
-import org.apache.hama.algebra.MultiplicationReduce;
-import org.apache.hama.io.VectorEntry;
-import org.apache.hama.io.VectorMapWritable;
-import org.apache.hama.mapred.DenseMap;
-import org.apache.hama.mapred.MatrixReduce;
-import org.apache.hama.util.Numeric;
-import org.apache.hama.util.RandomVariable;
-
-public class DenseMatrix extends AbstractMatrix implements Matrix {
-
- /**
- * Construct
- *
- * @param conf configuration object
- */
- public DenseMatrix(HamaConfiguration conf) {
- setConfiguration(conf);
- }
-
- /**
- * Construct an matrix
- *
- * @param conf configuration object
- * @param matrixName the name of the matrix
- */
- public DenseMatrix(HamaConfiguration conf, String matrixName) {
- try {
- setConfiguration(conf);
- this.matrixName = matrixName;
-
- if (!admin.tableExists(matrixName)) {
- tableDesc = new HTableDescriptor(matrixName);
- tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
- create();
- }
-
- table = new HTable(config, matrixName);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- /**
- * Construct an m-by-n constant matrix.
- *
- * @param conf configuration object
- * @param m the number of rows.
- * @param n the number of columns.
- * @param s fill the matrix with this scalar value.
- */
- public DenseMatrix(HamaConfiguration conf, int m, int n, double s) {
- try {
- setConfiguration(conf);
- matrixName = RandomVariable.randMatrixName();
-
- if (!admin.tableExists(matrixName)) {
- tableDesc = new HTableDescriptor(matrixName);
- tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
- create();
- }
-
- table = new HTable(config, matrixName);
-
- for (int i = 0; i < m; i++) {
- for (int j = 0; j < n; j++) {
- set(i, j, s);
- }
- }
-
- setDimension(m, n);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- /**
- * Generate matrix with random elements
- *
- * @param conf configuration object
- * @param m the number of rows.
- * @param n the number of columns.
- * @return an m-by-n matrix with uniformly distributed random elements.
- * @throws IOException
- */
- public static Matrix random(HamaConfiguration conf, int m, int n)
- throws IOException {
- String name = RandomVariable.randMatrixName();
- Matrix rand = new DenseMatrix(conf, name);
- for (int i = 0; i < m; i++) {
- for (int j = 0; j < n; j++) {
- rand.set(i, j, RandomVariable.rand());
- }
- }
-
- rand.setDimension(m, n);
- LOG.info("Create the " + m + " * " + n + " random matrix : " + name);
- return rand;
- }
-
- public Matrix add(Matrix B) throws IOException {
- String output = RandomVariable.randMatrixName();
- Matrix C = new DenseMatrix(config, output);
-
- JobConf jobConf = new JobConf(config);
- jobConf.setJobName("addition MR job");
-
- DenseMap.initJob(this.getName(), B.getName(), AdditionMap.class,
- IntWritable.class, DenseVector.class, jobConf);
- MatrixReduce.initJob(C.getName(), AdditionReduce.class, jobConf);
-
- JobClient.runJob(jobConf);
- return C;
- }
-
- public Matrix add(double alpha, Matrix B) throws IOException {
- // TODO Auto-generated method stub
- return null;
- }
-
- public DenseVector getRow(int row) throws IOException {
- return new DenseVector(row, table.getRow(String.valueOf(row)));
- }
-
- public Vector getColumn(int column) throws IOException {
- byte[] columnKey = Numeric.getColumnIndex(column);
- byte[][] c = { columnKey };
- Scanner scan = table.getScanner(c, HConstants.EMPTY_START_ROW);
-
- VectorMapWritable<Integer, VectorEntry> trunk = new
VectorMapWritable<Integer, VectorEntry>();
-
- for (RowResult row : scan) {
- trunk.put(Numeric.bytesToInt(row.getRow()), new VectorEntry(row
- .get(columnKey)));
- }
-
- return new DenseVector(column, trunk);
- }
-
- public Matrix mult(Matrix B) throws IOException {
- String output = RandomVariable.randMatrixName();
- Matrix C = new DenseMatrix(config, output);
-
- JobConf jobConf = new JobConf(config);
- jobConf.setJobName("multiplication MR job");
-
- DenseMap.initJob(this.getName(), B.getName(), MultiplicationMap.class,
- IntWritable.class, DenseVector.class, jobConf);
- MatrixReduce.initJob(C.getName(), MultiplicationReduce.class, jobConf);
-
- JobClient.runJob(jobConf);
- return C;
- }
-
- public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException {
- // TODO Auto-generated method stub
- return null;
- }
-
- public double norm(Norm type) throws IOException {
- // TODO Auto-generated method stub
- return 0;
- }
-
- public Matrix set(double alpha, Matrix B) throws IOException {
- // TODO Auto-generated method stub
- return null;
- }
-
- public Matrix set(Matrix B) throws IOException {
- // TODO Auto-generated method stub
- return null;
- }
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Scanner;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hama.algebra.AdditionMap;
+import org.apache.hama.algebra.AdditionReduce;
+import org.apache.hama.algebra.MultiplicationMap;
+import org.apache.hama.algebra.MultiplicationReduce;
+import org.apache.hama.io.VectorEntry;
+import org.apache.hama.io.VectorMapWritable;
+import org.apache.hama.mapred.MatrixReduce;
+import org.apache.hama.util.Numeric;
+import org.apache.hama.util.RandomVariable;
+
+public class DenseMatrix extends AbstractMatrix implements Matrix {
+
+ /**
+ * Construct
+ *
+ * @param conf configuration object
+ */
+ public DenseMatrix(HamaConfiguration conf) {
+ setConfiguration(conf);
+ }
+
+ /**
+ * Construct an matrix
+ *
+ * @param conf configuration object
+ * @param matrixName the name of the matrix
+ */
+ public DenseMatrix(HamaConfiguration conf, String matrixName) {
+ try {
+ setConfiguration(conf);
+ this.matrixName = matrixName;
+
+ if (!admin.tableExists(matrixName)) {
+ tableDesc = new HTableDescriptor(matrixName);
+ tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
+ create();
+ }
+
+ table = new HTable(config, matrixName);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Construct an m-by-n constant matrix.
+ *
+ * @param conf configuration object
+ * @param m the number of rows.
+ * @param n the number of columns.
+ * @param s fill the matrix with this scalar value.
+ */
+ public DenseMatrix(HamaConfiguration conf, int m, int n, double s) {
+ try {
+ setConfiguration(conf);
+ matrixName = RandomVariable.randMatrixName();
+
+ if (!admin.tableExists(matrixName)) {
+ tableDesc = new HTableDescriptor(matrixName);
+ tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
+ create();
+ }
+
+ table = new HTable(config, matrixName);
+
+ for (int i = 0; i < m; i++) {
+ for (int j = 0; j < n; j++) {
+ set(i, j, s);
+ }
+ }
+
+ setDimension(m, n);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Generate matrix with random elements
+ *
+ * @param conf configuration object
+ * @param m the number of rows.
+ * @param n the number of columns.
+ * @return an m-by-n matrix with uniformly distributed random elements.
+ * @throws IOException
+ */
+ public static Matrix random(HamaConfiguration conf, int m, int n)
+ throws IOException {
+ String name = RandomVariable.randMatrixName();
+ Matrix rand = new DenseMatrix(conf, name);
+ for (int i = 0; i < m; i++) {
+ for (int j = 0; j < n; j++) {
+ rand.set(i, j, RandomVariable.rand());
+ }
+ }
+
+ rand.setDimension(m, n);
+ LOG.info("Create the " + m + " * " + n + " random matrix : " + name);
+ return rand;
+ }
+
+ public Matrix add(Matrix B) throws IOException {
+ String output = RandomVariable.randMatrixName();
+ Matrix C = new DenseMatrix(config, output);
+
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("addition MR job");
+
+ AdditionMap.initJob(this.getName(), B.getName(), AdditionMap.class,
+ IntWritable.class, DenseVector.class, jobConf);
+ MatrixReduce.initJob(C.getName(), AdditionReduce.class, jobConf);
+
+ JobClient.runJob(jobConf);
+ return C;
+ }
+
+ public Matrix add(double alpha, Matrix B) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ public DenseVector getRow(int row) throws IOException {
+ return new DenseVector(row, table.getRow(String.valueOf(row)));
+ }
+
+ public Vector getColumn(int column) throws IOException {
+ byte[] columnKey = Numeric.getColumnIndex(column);
+ byte[][] c = { columnKey };
+ Scanner scan = table.getScanner(c, HConstants.EMPTY_START_ROW);
+
+ VectorMapWritable<Integer, VectorEntry> trunk = new
VectorMapWritable<Integer, VectorEntry>();
+
+ for (RowResult row : scan) {
+ trunk.put(Numeric.bytesToInt(row.getRow()), new VectorEntry(row
+ .get(columnKey)));
+ }
+
+ return new DenseVector(column, trunk);
+ }
+
+ public Matrix mult(Matrix B) throws IOException {
+ String output = RandomVariable.randMatrixName();
+ Matrix C = new DenseMatrix(config, output);
+
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("multiplication MR job");
+
+ MultiplicationMap.initJob(this.getName(), B.getName(),
MultiplicationMap.class,
+ IntWritable.class, DenseVector.class, jobConf);
+ MatrixReduce.initJob(C.getName(), MultiplicationReduce.class, jobConf);
+
+ JobClient.runJob(jobConf);
+ return C;
+ }
+
+ public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ public double norm(Norm type) throws IOException {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ public Matrix set(double alpha, Matrix B) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ public Matrix set(Matrix B) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionMap.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionMap.java?rev=693057&r1=693056&r2=693057&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionMap.java
(original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionMap.java Mon
Sep 8 04:39:59 2008
@@ -1,43 +1,69 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.algebra;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hama.DenseVector;
-import org.apache.hama.Vector;
-import org.apache.hama.mapred.DenseMap;
-
-public class AdditionMap extends DenseMap<IntWritable, DenseVector> {
-
- @Override
- public void map(IntWritable key, DenseVector value,
- OutputCollector<IntWritable, DenseVector> output,
- Reporter reporter) throws IOException {
-
- Vector v1 = MATRIX_B.getRow(key.get());
- output.collect(key, (DenseVector) v1.add(value));
-
- }
-
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseMatrix;
+import org.apache.hama.DenseVector;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.Matrix;
+import org.apache.hama.Vector;
+import org.apache.hama.mapred.DenseMap;
+import org.apache.log4j.Logger;
+
+public class AdditionMap extends DenseMap<IntWritable, DenseVector> {
+ static final Logger LOG = Logger.getLogger(AdditionMap.class);
+ protected Matrix matrix_b;
+ public static final String MATRIX_B = "hama.addition.matrix.b";
+
+ public void configure(JobConf job) {
+ matrix_b = new DenseMatrix(new HamaConfiguration(), job.get(MATRIX_B, ""));
+ }
+
+ public static void initJob(String matrix_a, String matrix_b,
+ Class<AdditionMap> map,
+ Class<IntWritable> outputKeyClass,
+ Class<DenseVector> outputValueClass,
+ JobConf jobConf) {
+
+ jobConf.setMapOutputValueClass(outputValueClass);
+ jobConf.setMapOutputKeyClass(outputKeyClass);
+ jobConf.setMapperClass(map);
+ jobConf.set(MATRIX_B, matrix_b);
+
+ initJob(matrix_a, map, jobConf);
+ }
+
+ @Override
+ public void map(IntWritable key, DenseVector value,
+ OutputCollector<IntWritable, DenseVector> output, Reporter reporter)
+ throws IOException {
+
+ Vector v1 = matrix_b.getRow(key.get());
+ output.collect(key, (DenseVector) v1.add(value));
+
+ }
+
+}
Modified:
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java?rev=693057&r1=693056&r2=693057&view=diff
==============================================================================
---
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
(original)
+++
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
Mon Sep 8 04:39:59 2008
@@ -23,9 +23,13 @@
import java.util.Iterator;
import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseMatrix;
import org.apache.hama.DenseVector;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.Matrix;
import org.apache.hama.Vector;
import org.apache.hama.io.VectorEntry;
import org.apache.hama.mapred.DenseMap;
@@ -33,7 +37,27 @@
public class MultiplicationMap extends DenseMap<IntWritable, DenseVector> {
static final Logger LOG = Logger.getLogger(MultiplicationMap.class);
-
+ protected Matrix matrix_b;
+ public static final String MATRIX_B = "hama.multiplication.matrix.b";
+
+ public void configure(JobConf job) {
+ matrix_b = new DenseMatrix(new HamaConfiguration(), job.get(MATRIX_B, ""));
+ }
+
+ public static void initJob(String matrix_a, String matrix_b,
+ Class<MultiplicationMap> map,
+ Class<IntWritable> outputKeyClass,
+ Class<DenseVector> outputValueClass,
+ JobConf jobConf) {
+
+ jobConf.setMapOutputValueClass(outputValueClass);
+ jobConf.setMapOutputKeyClass(outputKeyClass);
+ jobConf.setMapperClass(map);
+ jobConf.set(MATRIX_B, matrix_b);
+
+ initJob(matrix_a, map, jobConf);
+ }
+
@Override
public void map(IntWritable key, DenseVector value,
OutputCollector<IntWritable, DenseVector> output, Reporter reporter)
@@ -42,7 +66,7 @@
Iterator<VectorEntry> it = value.iterator();
int i = 0;
while (it.hasNext()) {
- Vector v = MATRIX_B.getRow(i);
+ Vector v = matrix_b.getRow(i);
output.collect(key, (DenseVector) v.scale(it.next().getValue()));
i++;
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/DenseMap.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/DenseMap.java?rev=693057&r1=693056&r2=693057&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/DenseMap.java
(original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/DenseMap.java Mon Sep
8 04:39:59 2008
@@ -1,62 +1,56 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.mapred;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hama.Constants;
-import org.apache.hama.DenseMatrix;
-import org.apache.hama.DenseVector;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.Matrix;
-
[EMAIL PROTECTED]("unchecked")
-public abstract class DenseMap<K extends WritableComparable, V extends
Writable>
- extends MapReduceBase implements
- Mapper<IntWritable, DenseVector, K, V> {
- public static Matrix MATRIX_B;
-
- public static void initJob(String matrixA, String matrixB,
- Class<? extends DenseMap> mapper,
- Class<? extends WritableComparable> outputKeyClass,
- Class<? extends Writable> outputValueClass, JobConf job) {
-
- job.setInputFormat(MatrixInputFormat.class);
- job.setMapOutputValueClass(outputValueClass);
- job.setMapOutputKeyClass(outputKeyClass);
- job.setMapperClass(mapper);
- FileInputFormat.addInputPaths(job, matrixA);
-
- MATRIX_B = new DenseMatrix(new HamaConfiguration(), matrixB);
- job.set(MatrixInputFormat.COLUMN_LIST, Constants.COLUMN);
- }
-
- public abstract void map(IntWritable key, DenseVector value,
- OutputCollector<K, V> output, Reporter reporter) throws IOException;
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.Constants;
+import org.apache.hama.DenseVector;
+import org.apache.hama.Matrix;
+
[EMAIL PROTECTED]("unchecked")
+public abstract class DenseMap<K extends WritableComparable, V extends
Writable>
+ extends MapReduceBase implements
+ Mapper<IntWritable, DenseVector, K, V> {
+ public static Matrix MATRIX_B;
+
+ public static void initJob(String matrixA,
+ Class<? extends DenseMap> mapper,
+ JobConf job) {
+
+ job.setInputFormat(MatrixInputFormat.class);
+ job.setMapperClass(mapper);
+ FileInputFormat.addInputPaths(job, matrixA);
+
+ job.set(MatrixInputFormat.COLUMN_LIST, Constants.COLUMN);
+ }
+
+ public abstract void map(IntWritable key, DenseVector value,
+ OutputCollector<K, V> output, Reporter reporter) throws IOException;
+}
Modified:
incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java?rev=693057&r1=693056&r2=693057&view=diff
==============================================================================
---
incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java
(original)
+++
incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java
Mon Sep 8 04:39:59 2008
@@ -1,79 +1,79 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.mapred;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hama.DenseMatrix;
-import org.apache.hama.DenseVector;
-import org.apache.hama.HCluster;
-import org.apache.hama.Matrix;
-import org.apache.hama.algebra.AdditionMap;
-import org.apache.hama.algebra.AdditionReduce;
-import org.apache.log4j.Logger;
-
-/**
- * Test Matrix Map/Reduce job
- */
-public class TestMatrixMapReduce extends HCluster {
- static final Logger LOG = Logger.getLogger(TestMatrixMapReduce.class);
- private String A = "matrixA";
- private String B = "matrixB";
- private String output = "output";
-
- /** constructor */
- public TestMatrixMapReduce() {
- super();
- }
-
- public void testMatrixMapReduce() throws IOException {
- Matrix matrixA = new DenseMatrix(conf, A);
- matrixA.set(0, 0, 1);
- matrixA.set(0, 1, 0);
-
- Matrix matrixB = new DenseMatrix(conf, B);
- matrixB.set(0, 0, 1);
- matrixB.set(0, 1, 1);
-
- miniMRJob();
- }
-
- public void miniMRJob() throws IOException {
- Matrix c = new DenseMatrix(conf, output);
-
- JobConf jobConf = new JobConf(conf, TestMatrixMapReduce.class);
- jobConf.setJobName("test MR job");
-
- DenseMap.initJob(A, B, AdditionMap.class, IntWritable.class,
- DenseVector.class, jobConf);
- MatrixReduce.initJob(output, AdditionReduce.class, jobConf);
-
- jobConf.setNumMapTasks(1);
- jobConf.setNumReduceTasks(1);
-
- JobClient.runJob(jobConf);
-
- assertEquals(c.get(0, 0), 2.0);
- assertEquals(c.get(0, 1), 1.0);
- }
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hama.DenseMatrix;
+import org.apache.hama.DenseVector;
+import org.apache.hama.HCluster;
+import org.apache.hama.Matrix;
+import org.apache.hama.algebra.AdditionMap;
+import org.apache.hama.algebra.AdditionReduce;
+import org.apache.log4j.Logger;
+
+/**
+ * Test Matrix Map/Reduce job
+ */
+public class TestMatrixMapReduce extends HCluster {
+ static final Logger LOG = Logger.getLogger(TestMatrixMapReduce.class);
+ private String A = "matrixA";
+ private String B = "matrixB";
+ private String output = "output";
+
+ /** constructor */
+ public TestMatrixMapReduce() {
+ super();
+ }
+
+ public void testMatrixMapReduce() throws IOException {
+ Matrix matrixA = new DenseMatrix(conf, A);
+ matrixA.set(0, 0, 1);
+ matrixA.set(0, 1, 0);
+
+ Matrix matrixB = new DenseMatrix(conf, B);
+ matrixB.set(0, 0, 1);
+ matrixB.set(0, 1, 1);
+
+ miniMRJob();
+ }
+
+ public void miniMRJob() throws IOException {
+ Matrix c = new DenseMatrix(conf, output);
+
+ JobConf jobConf = new JobConf(conf, TestMatrixMapReduce.class);
+ jobConf.setJobName("test MR job");
+
+ AdditionMap.initJob(A, B, AdditionMap.class, IntWritable.class,
+ DenseVector.class, jobConf);
+ MatrixReduce.initJob(output, AdditionReduce.class, jobConf);
+
+ jobConf.setNumMapTasks(1);
+ jobConf.setNumReduceTasks(1);
+
+ JobClient.runJob(jobConf);
+
+ assertEquals(c.get(0, 0), 2.0);
+ assertEquals(c.get(0, 1), 1.0);
+ }
+}
