Author: edwardyoon
Date: Mon Oct 20 18:04:30 2008
New Revision: 706484
URL: http://svn.apache.org/viewvc?rev=706484&view=rev
Log:
clear the matrices in hbase after quiting the shell
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/java/org/apache/hama/Constants.java
incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java
incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java
incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java
incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
Modified: incubator/hama/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Oct 20 18:04:30 2008
@@ -28,6 +28,7 @@
IMPROVEMENTS
+ HAMA-77: clear the matrices in hbase after quiting the shell (samuel via
edwardyoon)
HAMA-28: Implement Vector.add() method (edwardyoon)
HAMA-78: Separate Interface and Implementation for HamaAdmin (edwardyoon)
HAMA-76: Remove load() since it duplicated with constructor (edwardyoon)
Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Mon Oct 20
18:04:30 2008
@@ -1,76 +1,81 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-/**
- * Some constants used in the Hama
- */
-public class Constants {
-
- /**
- * Meta-columnFamily to store the matrix-info
- */
- public final static String METADATA = "metadata";
-
- /**
- * Column index & attributes
- */
- public final static String CINDEX = "cIndex";
-
- /**
- * The attribute column family
- */
- public final static String ATTRIBUTE = "attribute:";
-
- /**
- * The number of the matrix rows
- */
- public final static String METADATA_ROWS = "attribute:rows";
-
- /**
- * The number of the matrix columns
- */
- public final static String METADATA_COLUMNS = "attribute:columns";
-
- /**
- * The type of the matrix
- */
- public final static String METADATA_TYPE = "attribute:type";
-
- /**
- * Default columnFamily name
- */
- public final static String COLUMN = "column:";
-
- /**
- * Temporary random matrices name prefix
- */
- public final static String RANDOM = "rand";
-
- /**
- * Admin table name
- */
- public final static String ADMINTABLE = "admin.table";
-
- /**
- * Matrix path columnFamily
- */
- public static final String PATHCOLUMN = "path:";
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+/**
+ * Some constants used in the Hama
+ */
+public class Constants {
+
+ /**
+ * Meta-columnFamily to store the matrix-info
+ */
+ public final static String METADATA = "metadata";
+
+ /**
+ * Column index & attributes
+ */
+ public final static String CINDEX = "cIndex";
+
+ /**
+ * The attribute column family
+ */
+ public final static String ATTRIBUTE = "attribute:";
+
+ /**
+ * The number of the matrix rows
+ */
+ public final static String METADATA_ROWS = "attribute:rows";
+
+ /**
+ * The number of the matrix columns
+ */
+ public final static String METADATA_COLUMNS = "attribute:columns";
+
+ /**
+ * The type of the matrix
+ */
+ public final static String METADATA_TYPE = "attribute:type";
+
+ /**
+ * Default columnFamily name
+ */
+ public final static String COLUMN = "column:";
+
+ /**
+ * Temporary random matrices name prefix
+ */
+ public final static String RANDOM = "rand";
+
+ /**
+ * Admin table name
+ */
+ public final static String ADMINTABLE = "admin.table";
+
+ /**
+ * Matrix path columnFamily
+ */
+ public static final String PATHCOLUMN = "path:";
+
+ /**
+ * Temporary Aliase name prefix in Hama Shell
+ */
+ public static final String RANDOMALIASE = "_";
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java
(original)
+++ incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java Mon Oct
20 18:04:30 2008
@@ -85,9 +85,11 @@
}
public static void main(String[] args) {
+ HamaShellEnv shellEnv = new HamaShellEnv();
HamaShell shell = new HamaShell(new BufferedReader(new InputStreamReader(
- System.in)), new HamaShellEnv(), new HamaConfiguration());
+ System.in)), shellEnv, new HamaConfiguration());
shell.run();
+ shellEnv.clearAliases();
}
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java
(original)
+++ incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java Mon
Oct 20 18:04:30 2008
@@ -17,8 +17,13 @@
*/
package org.apache.hama.shell;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+
+import org.apache.hama.Matrix;
+import org.apache.hama.util.RandomVariable;
+import org.apache.log4j.Logger;
/**
* HamaShellEnv.
*
@@ -32,7 +37,8 @@
// so that we can use these information to rearrange the expressions
// and make the hama shell executed more efficiently and parallel.
public class HamaShellEnv {
-
+
+ static final Logger LOG = Logger.getLogger(HamaShellEnv.class);
Map<String, Object> aliases = new HashMap<String, Object>();
public static final int DEFAULT_MAP_NUM = 2;
@@ -57,4 +63,34 @@
return aliases;
}
+ /**
+ * Get the random aliase name from hama shell.
+ * make sure the random aliase name doesn't exist in hama shell before.
+ * @return random aliase name
+ */
+ public String getRandomAliaseName() {
+ String rName = RandomVariable.randAliaseName();
+ while(aliases.containsKey(rName)) {
+ rName = RandomVariable.randAliaseName();
+ }
+ return rName;
+ }
+
+ /**
+ * Clear all the aliases in the hama shell.
+ */
+ public void clearAliases() {
+ for(Object obj : aliases.values()) {
+ if(obj == null)
+ continue;
+ if(obj instanceof Matrix) {
+ try {
+ ((Matrix)obj).close();
+ } catch (IOException e) {
+ LOG.info("Matrix close : " + e.getMessage());
+ }
+ }
+ }
+ }
+
}
Modified:
incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
---
incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java
(original)
+++
incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java
Mon Oct 20 18:04:30 2008
@@ -22,6 +22,7 @@
import java.util.List;
import org.apache.hama.Matrix;
+import org.apache.hama.shell.HamaShellEnv;
public class AlgebraOperator {
@@ -32,6 +33,9 @@
Object operand;
List<Entry> otherOperands;
+ // HamaShellEnv to keep all the temporary matrices generated
+ // during the execution of hama expression in shell
+ HamaShellEnv env;
/**
*
* An Entry Class to store all the Operators and other Operands.
@@ -55,9 +59,10 @@
}
}
- public AlgebraOperator(Object operand) {
+ public AlgebraOperator(Object operand, HamaShellEnv env) {
this.operand = operand;
this.otherOperands = null;
+ this.env = env;
}
public void addOpAndOperand(String op, AlgebraOperator otherOperand) {
@@ -183,7 +188,10 @@
} else
throw new AlgebraOpException(
"Uncognized Operator. It should be '+' '-' '*'.");
-
+
+ // for "m" will be replace by a new matrix in next loop.
+ // so we need to record it.
+ env.setAliase(env.getRandomAliaseName(), m);
}
return m;
@@ -209,10 +217,21 @@
}
// Now we can do multiplication.
+ // All the matrices in the multiplication has been recorded as
+ // Aliase or Temp Aliase in Hama Shell Env. so we just need to
+ // keep all the temp matrix in Hama Shell Env during the matrix
+ // multiplication.
Matrix matrix = matrices.get(0);
+ if(matrices.size() == 1)
+ return matrix;
+
for (int i = 1; i < matrices.size(); i++) {
Matrix m = matrices.get(i);
matrix = matrix.mult(m);
+ // keep the temp matrix in Hama shell env for cleaning
+ // TODO: the last matrix may be keeped by an Aliase an a TempAliase.
+ // we need some method to tell them from each other.
+ env.setAliase(env.getRandomAliaseName(), matrix);
}
return matrix;
Modified:
incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
---
incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
(original)
+++
incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
Mon Oct 20 18:04:30 2008
@@ -221,7 +221,7 @@
/** An Additive Operation. */
AlgebraOperator AdditiveOp() : {AlgebraOperator op, op1, op2; Token t;}
{
- ( op1=MultiplicativeOp() { op = new AlgebraOperator(op1); } ( ( t=<ADDOP>
op2=MultiplicativeOp() ){ op.addOpAndOperand(t.image, op2); } )* )
+ ( op1=MultiplicativeOp() { op = new AlgebraOperator(op1, shellEnv); } ( (
t=<ADDOP> op2=MultiplicativeOp() ){ op.addOpAndOperand(t.image, op2); } )* )
{
return op;
}
@@ -230,7 +230,7 @@
/** A Multiplicative Operation. */
AlgebraOperator MultiplicativeOp() : {AlgebraOperator op, op1, op2; Token t;}
{
- ( op1=UnaryOp() { op = new AlgebraOperator(op1); } ( ( t=<MULOP>
op2=UnaryOp() ){op.addOpAndOperand(t.image, op2);} )* )
+ ( op1=UnaryOp() { op = new AlgebraOperator(op1, shellEnv); } ( ( t=<MULOP>
op2=UnaryOp() ){op.addOpAndOperand(t.image, op2);} )* )
{
return op;
}
@@ -242,7 +242,7 @@
(
( "(" op=AdditiveOp() ")" )
|
- b = AliaseName() { op = new AlgebraOperator(b); }
+ b = AliaseName() { op = new AlgebraOperator(b, shellEnv); }
)
{
return op;
Modified: incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
URL:
http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
(original)
+++ incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java Mon
Oct 20 18:04:30 2008
@@ -51,18 +51,39 @@
}
/**
- * Generate a random name.
+ * Generate a random matrix name.
*
- * @return random name
+ * @return random matrix name
*/
public static String randMatrixName() {
- String rName = Constants.RANDOM;
- for (int i = 1; i <= 5; i++) {
+ return randString(Constants.RANDOM, 5);
+ }
+
+ /**
+ * Generate a random aliase name.
+ *
+ * @return random aliase name
+ */
+ public static String randAliaseName() {
+ return randString(Constants.RANDOMALIASE, 5);
+ }
+
+ /**
+ * Generate a random string using the specified prefix and a fixed length.
+ * @param prefix
+ * the specified string prefix.
+ * @param length
+ * the length of the string to be appended.
+ * @return random string.
+ */
+ public static String randString(String prefix, int length) {
+ StringBuilder result = new StringBuilder(prefix);
+ for (int i = 0; i < length; i++) {
char ch = (char) ((Math.random() * 26) + 97);
- rName += ch;
+ result.append(ch);
}
-
- return rName;
+
+ return result.toString();
}
/**
