Author: edwardyoon
Date: Fri Sep 27 03:34:49 2013
New Revision: 1526783
URL: http://svn.apache.org/r1526783
Log:
Minor fix
Modified:
hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
Modified:
hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
URL:
http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java?rev=1526783&r1=1526782&r2=1526783&view=diff
==============================================================================
---
hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
(original)
+++
hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
Fri Sep 27 03:34:49 2013
@@ -22,7 +22,6 @@ import java.io.BufferedWriter;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.URI;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -36,10 +35,9 @@ import org.apache.hama.ml.math.FunctionF
/**
*
- *
*/
public class NeuralNetwork {
-
+
public static void main(String[] args) throws Exception {
if (args.length < 3) {
printUsage();
@@ -51,24 +49,27 @@ public class NeuralNetwork {
printUsage();
return;
}
-
+
String modelPath = args[1];
String featureDataPath = args[2];
String resultDataPath = args[3];
-
+
SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(modelPath);
-
+
// process data in streaming approach
- FileSystem fs = FileSystem.get(new URI(featureDataPath), new
Configuration());
- BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new
Path(featureDataPath))));
+ FileSystem fs = FileSystem.get(new URI(featureDataPath),
+ new Configuration());
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ fs.open(new Path(featureDataPath))));
Path outputPath = new Path(resultDataPath);
if (fs.exists(outputPath)) {
fs.delete(outputPath, true);
}
- BufferedWriter bw = new BufferedWriter(new
OutputStreamWriter(fs.create(outputPath)));
-
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+ fs.create(outputPath)));
+
String line = null;
-
+
while ((line = br.readLine()) != null) {
if (line.trim().length() == 0) {
continue;
@@ -86,18 +87,16 @@ public class NeuralNetwork {
sb.append(arrResult[i]);
if (i != arrResult.length - 1) {
sb.append(",");
- }
- else {
+ } else {
sb.append("\n");
}
}
bw.write(sb.toString());
}
-
+
br.close();
bw.close();
- }
- else if (mode.equals("train")) {
+ } else if (mode.equals("train")) {
if (args.length < 5) {
printUsage();
return;
@@ -105,22 +104,23 @@ public class NeuralNetwork {
String trainingDataPath = args[1];
String trainedModelPath = args[2];
-
+
int featureDimension = Integer.parseInt(args[3]);
int labelDimension = Integer.parseInt(args[4]);
-
+
int iteration = 1000;
double learningRate = 0.4;
double momemtumWeight = 0.2;
double regularizationWeight = 0.01;
-
+
// parse parameters
if (args.length >= 6) {
try {
iteration = Integer.parseInt(args[5]);
System.out.printf("Iteration: %d\n", iteration);
} catch (NumberFormatException e) {
- System.err.println("MAX_ITERATION format invalid. It should be a
positive number.");
+ System.err
+ .println("MAX_ITERATION format invalid. It should be a positive
number.");
return;
}
}
@@ -129,7 +129,8 @@ public class NeuralNetwork {
learningRate = Double.parseDouble(args[6]);
System.out.printf("Learning rate: %f\n", learningRate);
} catch (NumberFormatException e) {
- System.err.println("LEARNING_RATE format invalid. It should be a
positive double in range (0, 1.0)");
+ System.err
+ .println("LEARNING_RATE format invalid. It should be a positive
double in range (0, 1.0)");
return;
}
}
@@ -138,21 +139,24 @@ public class NeuralNetwork {
momemtumWeight = Double.parseDouble(args[7]);
System.out.printf("Momemtum weight: %f\n", momemtumWeight);
} catch (NumberFormatException e) {
- System.err.println("MOMEMTUM_WEIGHT format invalid. It should be a
positive double in range (0, 1.0)");
+ System.err
+ .println("MOMEMTUM_WEIGHT format invalid. It should be a
positive double in range (0, 1.0)");
return;
}
}
if (args.length >= 9) {
try {
regularizationWeight = Double.parseDouble(args[8]);
- System.out.printf("Regularization weight: %f\n",
regularizationWeight);
+ System.out
+ .printf("Regularization weight: %f\n", regularizationWeight);
} catch (NumberFormatException e) {
- System.err.println("REGULARIZATION_WEIGHT format invalid. It should
be a positive double in range (0, 1.0)");
+ System.err
+ .println("REGULARIZATION_WEIGHT format invalid. It should be a
positive double in range (0, 1.0)");
return;
}
}
-
- // train the model
+
+ // train the model
SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork();
ann.setLearningRate(learningRate);
ann.setMomemtumWeight(momemtumWeight);
@@ -161,7 +165,8 @@ public class NeuralNetwork {
FunctionFactory.createDoubleFunction("Sigmoid"));
ann.addLayer(featureDimension, false,
FunctionFactory.createDoubleFunction("Sigmoid"));
- ann.addLayer(labelDimension, true,
FunctionFactory.createDoubleFunction("Sigmoid"));
+ ann.addLayer(labelDimension, true,
+ FunctionFactory.createDoubleFunction("Sigmoid"));
ann.setCostFunction(FunctionFactory
.createDoubleDoubleFunction("CrossEntropy"));
ann.setModelPath(trainedModelPath);
@@ -173,25 +178,38 @@ public class NeuralNetwork {
trainingParameters.put("convergence.check.interval", "1000");
ann.train(new Path(trainingDataPath), trainingParameters);
}
-
+
}
-
+
private static void printUsage() {
- System.out.println("USAGE: <MODE> <INPUT_PATH> <OUTPUT_PATH>
<MODEL_PATH>|<FEATURE_DIMENSION> <LABEL_DIMENSION> [<MAX_ITERATION>
<LEARNING_RATE> <MOMEMTUM_WEIGHT> <REGULARIZATION_WEIGHT>]");
- System.out.println("\tMODE\t- train: train the model with given training
data.");
- System.out.println("\t\t- evaluate: obtain the result by feeding the
features to the neural network.");
- System.out.println("\tINPUT_PATH\tin 'train' mode, it is the path of the
training data; in 'evaluate' mode, it is the path of the to be evaluated data
that lacks the label.");
- System.out.println("\tOUTPUT_PATH\tin 'train' mode, it is where the
trained model is stored; in 'evaluate' mode, it is where the labeled data is
stored.");
+ System.out
+ .println("USAGE: <MODE> <INPUT_PATH> <OUTPUT_PATH>
<MODEL_PATH>|<FEATURE_DIMENSION> <LABEL_DIMENSION> [<MAX_ITERATION>
<LEARNING_RATE> <MOMEMTUM_WEIGHT> <REGULARIZATION_WEIGHT>]");
+ System.out
+ .println("\tMODE\t- train: train the model with given training data.");
+ System.out
+ .println("\t\t- evaluate: obtain the result by feeding the features to
the neural network.");
+ System.out
+ .println("\tINPUT_PATH\tin 'train' mode, it is the path of the
training data; in 'evaluate' mode, it is the path of the to be evaluated data
that lacks the label.");
+ System.out
+ .println("\tOUTPUT_PATH\tin 'train' mode, it is where the trained
model is stored; in 'evaluate' mode, it is where the labeled data is stored.");
System.out.println("\n\tConditional Parameters:");
- System.out.println("\tMODEL_PATH\tonly required in 'evaluate' mode. It
specifies where to load the trained neural network model.");
- System.out.println("\tMAX_ITERATION\tonly used in 'train' mode. It
specifies how many iterations for the neural network to run. Default is 0.01.");
- System.out.println("\tLEARNING_RATE\tonly used to 'train' mode. It
specifies the degree of aggregation for learning, usually in range (0, 1.0).
Default is 0.1.");
- System.out.println("\tMOMEMTUM_WEIGHT\tonly used to 'train' mode. It
specifies the weight of momemtum. Default is 0.");
- System.out.println("\tREGULARIZATION_WEIGHT\tonly required in 'train'
model. It specifies the weight of reqularization.");
+ System.out
+ .println("\tMODEL_PATH\tonly required in 'evaluate' mode. It specifies
where to load the trained neural network model.");
+ System.out
+ .println("\tMAX_ITERATION\tonly used in 'train' mode. It specifies how
many iterations for the neural network to run. Default is 0.01.");
+ System.out
+ .println("\tLEARNING_RATE\tonly used to 'train' mode. It specifies the
degree of aggregation for learning, usually in range (0, 1.0). Default is
0.1.");
+ System.out
+ .println("\tMOMEMTUM_WEIGHT\tonly used to 'train' mode. It specifies
the weight of momemtum. Default is 0.");
+ System.out
+ .println("\tREGULARIZATION_WEIGHT\tonly required in 'train' model. It
specifies the weight of reqularization.");
System.out.println("\nExample:");
- System.out.println("Train a neural network with default
setting:\n\tneuralnets train hdfs://localhost:30002/training_data
hdfs://localhost:30002/model 8 1");
- System.out.println("Train a neural network by specify learning rate as
0.1, momemtum rate as 0.2, and regularization weight as
0.01:\n\tneuralnets.train hdfs://localhost:30002/training_data
hdfs://localhost:30002/model 0.1 0.2 0.01");
- System.out.println("Label the data with trained model:\n\tneuralnets
evaluate hdfs://localhost:30002/unlabeled_data hdfs://localhost:30002/result
hdfs://localhost:30002/model");
+ System.out
+ .println("Train a neural network with default setting:\n\tneuralnets
train hdfs://localhost:30002/training_data hdfs://localhost:30002/model 8 1");
+ System.out
+ .println("Train a neural network by specify learning rate as 0.1,
momemtum rate as 0.2, and regularization weight as 0.01:\n\tneuralnets.train
hdfs://localhost:30002/training_data hdfs://localhost:30002/model 0.1 0.2
0.01");
+ System.out
+ .println("Label the data with trained model:\n\tneuralnets evaluate
hdfs://localhost:30002/unlabeled_data hdfs://localhost:30002/result
hdfs://localhost:30002/model");
}
}
Modified:
hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
URL:
http://svn.apache.org/viewvc/hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java?rev=1526783&r1=1526782&r2=1526783&view=diff
==============================================================================
---
hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
(original)
+++
hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
Fri Sep 27 03:34:49 2013
@@ -23,34 +23,42 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import junit.framework.TestCase;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.hama.HamaConfiguration;
import org.apache.hama.ml.math.DenseDoubleVector;
import org.apache.hama.ml.writable.VectorWritable;
-import org.junit.Test;
/**
* Test the functionality of NeuralNetwork Example.
*
*/
-public class NeuralNetworkTest {
+public class NeuralNetworkTest extends TestCase {
+ private Configuration conf = new HamaConfiguration();
+ private FileSystem fs;
+ private String MODEL_PATH = "/tmp/neuralnets.model";
+ private String RESULT_PATH = "/tmp/neuralnets.txt";
+ private String SEQTRAIN_DATA = "/tmp/test-neuralnets.data";
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ fs = FileSystem.get(conf);
+ }
- @Test
- public void testNeuralnetsLabeling() {
- this.testNeuralNetworkTraining();
+ public void testNeuralnetsLabeling() throws IOException {
+ this.neuralNetworkTraining();
String dataPath = "src/test/resources/neuralnets_classification_test.txt";
- String modelPath = "/tmp/neuralnets.model";
- String resultPath = "/tmp/neuralnets.txt";
String mode = "label";
- Configuration conf = new Configuration();
try {
- FileSystem fs = FileSystem.get(conf);
NeuralNetwork
- .main(new String[] { mode, modelPath, dataPath, resultPath });
+ .main(new String[] { mode, MODEL_PATH, dataPath, RESULT_PATH });
// compare results with ground-truth
BufferedReader groundTruthReader = new BufferedReader(new FileReader(
@@ -61,8 +69,9 @@ public class NeuralNetworkTest {
groundTruthList.add(Double.parseDouble(line));
}
groundTruthReader.close();
-
- BufferedReader resultReader = new BufferedReader(new
FileReader(resultPath));
+
+ BufferedReader resultReader = new BufferedReader(new FileReader(
+ RESULT_PATH));
List<Double> resultList = new ArrayList<Double>();
while ((line = resultReader.readLine()) != null) {
resultList.add(Double.parseDouble(line));
@@ -78,22 +87,23 @@ public class NeuralNetworkTest {
}
}
System.out.printf("Precision: %f\n", correct / total);
- fs.delete(new Path(resultPath), true);
- fs.delete(new Path(modelPath), true);
} catch (Exception e) {
e.printStackTrace();
+ } finally {
+ fs.delete(new Path(RESULT_PATH), true);
+ fs.delete(new Path(MODEL_PATH), true);
+ fs.delete(new Path(SEQTRAIN_DATA), true);
}
}
-
- private void testNeuralNetworkTraining() {
+
+ private void neuralNetworkTraining() {
String mode = "train";
String strTrainingDataPath =
"src/test/resources/neuralnets_classification_training.txt";
- String strSequenceTrainingDataPath = "/tmp/test-neuralnets.data";
int featureDimension = 8;
int labelDimension = 1;
- Path sequenceTrainingDataPath = new Path(strSequenceTrainingDataPath);
+ Path sequenceTrainingDataPath = new Path(SEQTRAIN_DATA);
Configuration conf = new Configuration();
FileSystem fs;
try {
@@ -119,13 +129,9 @@ public class NeuralNetworkTest {
e1.printStackTrace();
}
- String modelPath = "/tmp/neuralnets.model";
try {
- NeuralNetwork.main(new String[] { mode, strSequenceTrainingDataPath,
- modelPath, "" + featureDimension, "" + labelDimension });
- fs = FileSystem.get(conf);
- fs.delete(new Path(strSequenceTrainingDataPath), true);
- fs.delete(new Path(modelPath), true);
+ NeuralNetwork.main(new String[] { mode, SEQTRAIN_DATA,
+ MODEL_PATH, "" + featureDimension, "" + labelDimension });
} catch (Exception e) {
e.printStackTrace();
}
