Author: edwardyoon
Date: Thu Sep 18 02:15:15 2008
New Revision: 696609
URL: http://svn.apache.org/viewvc?rev=696609&view=rev
Log:
Load/Save methods
Added:
incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/Constants.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/Matrix.java
incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
Modified: incubator/hama/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Thu Sep 18 02:15:15 2008
@@ -4,6 +4,7 @@
NEW FEATURES
+ HAMA-61: Load / save matrices from HTable (edwardyoon)
HAMA-51: Add get/setRowAttribute() method (edwardyoon)
HAMA-13: Matrix multiplication (edwardyoon)
HAMA-48: Add getColumn(int column) method which returns column vector
(edwardyoon)
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Thu Sep
18 02:15:15 2008
@@ -51,7 +51,8 @@
protected HTable table;
/** Matrix attribute description */
protected HTableDescriptor tableDesc;
-
+ public HamaAdmin hAdmin;
+
/**
* Sets the job configuration
*
@@ -64,13 +65,17 @@
} catch (MasterNotRunningException e) {
LOG.error(e, e);
}
+
+ hAdmin = new HamaAdmin(conf, admin);
}
/**
* Create matrix space
*/
protected void create() throws IOException {
+ this.tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
+
LOG.info("Initializing the matrix storage.");
this.admin.createTable(this.tableDesc);
}
@@ -161,4 +166,12 @@
public String getName() {
return (matrixName != null) ? matrixName : null;
}
+
+ public void clear() throws IOException {
+ admin.deleteTable(matrixName);
+ }
+
+ public boolean save(String path) throws IOException {
+ return hAdmin.put(this.matrixName, path);
+ }
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Thu Sep 18
02:15:15 2008
@@ -63,4 +63,14 @@
* Temporary random matrices name prefix
*/
public final static String RANDOM = "rand";
+
+ /**
+ * Admin table name
+ */
+ public final static String ADMINTABLE = "admin.table";
+
+ /**
+ * Matrix path columnFamily
+ */
+ public static final String PATHCOLUMN = "path:";
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Thu Sep 18
02:15:15 2008
@@ -64,7 +64,6 @@
if (!admin.tableExists(matrixName)) {
tableDesc = new HTableDescriptor(matrixName);
- tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
create();
}
@@ -215,4 +214,9 @@
public void setColumn(int column, Vector vector) throws IOException {
// TODO Auto-generated method stub
}
+
+ public void load(String path) throws IOException {
+ matrixName = hAdmin.get(path);
+ table = new HTable(matrixName);
+ }
}
Added: incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java?rev=696609&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java Thu Sep 18
02:15:15 2008
@@ -0,0 +1,88 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Admin tool
+ */
+public class HamaAdmin {
+ public HamaConfiguration conf;
+ public HBaseAdmin admin;
+ public HTable table;
+
+ public HamaAdmin(HamaConfiguration conf) {
+ this.conf = conf;
+ initialJob();
+ }
+
+ public HamaAdmin(HamaConfiguration conf, HBaseAdmin admin) {
+ this.conf = conf;
+ this.admin = admin;
+ initialJob();
+ }
+
+ public void initialJob() {
+ try {
+ if (!admin.tableExists(Constants.ADMINTABLE)) {
+ HTableDescriptor tableDesc = new
HTableDescriptor(Constants.ADMINTABLE);
+ tableDesc.addFamily(new HColumnDescriptor(Constants.PATHCOLUMN));
+ admin.createTable(tableDesc);
+ }
+
+ table = new HTable(conf, Constants.ADMINTABLE);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public boolean put(String matrixName, String path) {
+ boolean result = false;
+
+ BatchUpdate update = new BatchUpdate(path);
+ update.put(Constants.PATHCOLUMN, Bytes.toBytes(matrixName));
+ try {
+ table.commit(update);
+ result = true;
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return result;
+ }
+
+ public String get(String path) {
+ try {
+ byte[] result = table.get(path, Constants.PATHCOLUMN).getValue();
+ return Bytes.toString(result);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/Matrix.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Matrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Matrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Matrix.java Thu Sep 18
02:15:15 2008
@@ -32,7 +32,7 @@
* @param i ith row of the matrix
* @param j jth column of the matrix
* @return the value of entry
- * @throws IOException
+ * @throws IOException
*/
public double get(int i, int j) throws IOException;
@@ -58,7 +58,7 @@
* Get the number of row of the matrix from the meta-data column
*
* @return a number of rows of the matrix
- * @throws IOException
+ * @throws IOException
*/
public int getRows() throws IOException;
@@ -66,7 +66,7 @@
* Get the number of column of the matrix from the meta-data column
*
* @return a number of columns of the matrix
- * @throws IOException
+ * @throws IOException
*/
public int getColumns() throws IOException;
@@ -76,23 +76,30 @@
* @throws IOException
*/
public String getRowAttribute(int i) throws IOException;
-
+
/**
- * Sets the attribute of the row
+ * Gets the attribute of the column
*
- * @param i
- * @param name
* @throws IOException
*/
- public void setRowAttribute(int i, String name) throws IOException;
-
+ public String getColumnAttribute(int j) throws IOException;
+
/**
- * Gets the attribute of the column
+ * Return the matrix name
*
+ * @return the name of the matrix
+ */
+ public String getName();
+
+ /**
+ * Sets the attribute of the row
+ *
+ * @param i
+ * @param name
* @throws IOException
*/
- public String getColumnAttribute(int j) throws IOException;
-
+ public void setRowAttribute(int i, String name) throws IOException;
+
/**
* Sets the attribute of the column
*
@@ -101,14 +108,14 @@
* @throws IOException
*/
public void setColumnAttribute(int j, String name) throws IOException;
-
+
/**
* Sets the double value of (i, j)
*
* @param i ith row of the matrix
* @param j jth column of the matrix
* @param value the value of entry
- * @throws IOException
+ * @throws IOException
*/
public void set(int i, int j, double value) throws IOException;
@@ -118,7 +125,7 @@
* @param alpha
* @param B
* @return A
- * @throws IOException
+ * @throws IOException
*/
public Matrix set(double alpha, Matrix B) throws IOException;
@@ -127,7 +134,7 @@
*
* @param B
* @return A
- * @throws IOException
+ * @throws IOException
*/
public Matrix set(Matrix B) throws IOException;
@@ -139,7 +146,7 @@
* @throws IOException
*/
public void setRow(int row, Vector vector) throws IOException;
-
+
/**
* Set the column of a matrix to a given vector
*
@@ -148,7 +155,7 @@
* @throws IOException
*/
public void setColumn(int column, Vector vector) throws IOException;
-
+
/**
* Sets the dimension of matrix
*
@@ -164,7 +171,7 @@
* @param i
* @param j
* @param value
- * @throws IOException
+ * @throws IOException
*/
public void add(int i, int j, double value) throws IOException;
@@ -173,7 +180,7 @@
*
* @param B
* @return A
- * @throws IOException
+ * @throws IOException
*/
public Matrix add(Matrix B) throws IOException;
@@ -192,7 +199,7 @@
*
* @param B
* @return C
- * @throws IOException
+ * @throws IOException
*/
public Matrix mult(Matrix B) throws IOException;
@@ -203,7 +210,7 @@
* @param B
* @param C
* @return C
- * @throws IOException
+ * @throws IOException
*/
public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException;
@@ -212,7 +219,7 @@
*
* @param type
* @return norm of the matrix
- * @throws IOException
+ * @throws IOException
*/
public double norm(Norm type) throws IOException;
@@ -223,11 +230,28 @@
/** Largest entry in absolute value */
Infinity
}
-
+
/**
- * Return the matrix name
+ * Save to a table or file
*
- * @return the name of the matrix
+ * @param path
+ * @return true if saved
+ * @throws IOException
*/
- public String getName();
+ public boolean save(String path) throws IOException;
+
+ /**
+ * Load the matrix from a table or file
+ *
+ * @param path
+ * @throws IOException
+ */
+ public void load(String path) throws IOException;
+
+ /**
+ * Clear current matrix.
+ *
+ * @throws Exception
+ */
+ public void clear() throws IOException;
}
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
(original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Thu Sep
18 02:15:15 2008
@@ -50,12 +50,21 @@
}
protected void tearDown() {
- LOG.info("tearDown()");
+ try {
+ clearTest();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
};
return setup;
}
+ public static void clearTest() throws IOException {
+ m1.clear();
+ m2.clear();
+ }
+
/**
* Column vector test.
*
@@ -76,12 +85,12 @@
m1.setRowAttribute(0, "row1");
assertEquals(m1.getRowAttribute(0), "row1");
assertEquals(m1.getRowAttribute(1), null);
-
+
m1.setColumnAttribute(0, "column1");
assertEquals(m1.getColumnAttribute(0), "column1");
assertEquals(m1.getColumnAttribute(1), null);
}
-
+
/**
* Test matrices addition
*
@@ -133,6 +142,19 @@
}
}
+ public void testLoadSave() throws IOException {
+ m1.save("udanax");
+ HCluster hCluster = new HCluster();
+ DenseMatrix loadTest = new DenseMatrix(hCluster.conf);
+ loadTest.load("udanax");
+
+ for (int i = 0; i < loadTest.getRows(); i++) {
+ for (int j = 0; j < loadTest.getColumns(); j++) {
+ assertEquals(m1.get(i, j), loadTest.get(i, j));
+ }
+ }
+ }
+
/**
* Verifying multiplication result
*
