Author: srowen
Date: Fri Apr 15 09:48:07 2011
New Revision: 1092645
URL: http://svn.apache.org/viewvc?rev=1092645&view=rev
Log:
Fix intermitten failure with MahoutEvaluatorTest and MahoutEvaluator by
rationalizing use of input/output dirs
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutEvaluator.java
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutFitnessEvaluator.java
mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutEvaluator.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutEvaluator.java?rev=1092645&r1=1092644&r2=1092645&view=diff
==============================================================================
---
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutEvaluator.java
(original)
+++
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutEvaluator.java
Fri Apr 15 09:48:07 2011
@@ -41,7 +41,9 @@ import org.uncommons.watchmaker.framewor
* storing the population into an input file, and loading the fitness from job
outputs.
*/
public final class MahoutEvaluator {
- private MahoutEvaluator() { }
+
+ private MahoutEvaluator() {
+ }
/**
* Uses Mahout to evaluate every candidate from the input population using
the given evaluator.
@@ -54,38 +56,29 @@ public final class MahoutEvaluator {
* <code>List<Double></code> that contains the evaluated
fitness for each candidate from the
* input population, sorted in the same order as the candidates.
*/
- public static void evaluate(FitnessEvaluator<?> evaluator, Iterable<?>
population, Collection<Double> evaluations)
- throws IOException, ClassNotFoundException, InterruptedException {
+ public static void evaluate(FitnessEvaluator<?> evaluator,
+ Iterable<?> population,
+ Collection<Double> evaluations,
+ Path input,
+ Path output) throws IOException,
ClassNotFoundException, InterruptedException {
+
Job job = new Job();
job.setJarByClass(MahoutEvaluator.class);
+
Configuration conf = job.getConfiguration();
+
FileSystem fs = FileSystem.get(conf);
- Path inpath = prepareInput(fs, population);
- Path outpath = new Path("output");
+ HadoopUtil.delete(conf, input);
+ HadoopUtil.delete(conf, output);
- configureJob(job, conf, evaluator, inpath, outpath);
+ storePopulation(fs, new Path(input, "population"), population);
+
+ configureJob(job, conf, evaluator, input, output);
job.waitForCompletion(true);
- OutputUtils.importEvaluations(fs, conf, outpath, evaluations);
- }
-
- /**
- * Create the input directory and stores the population in it.
- *
- * @param fs
- * <code>FileSystem</code> to use
- * @param population
- * population to store
- * @return input <code>Path</code>
- */
- private static Path prepareInput(FileSystem fs, Iterable<?> population)
throws IOException {
- Path inpath = new Path(fs.getWorkingDirectory(), "input");
- Configuration conf = fs.getConf();
- HadoopUtil.delete(conf, inpath);
- storePopulation(fs, new Path(inpath, "population"), population);
- return inpath;
+ OutputUtils.importEvaluations(fs, conf, output, evaluations);
}
-
+
/**
* Configure the job
*
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutFitnessEvaluator.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutFitnessEvaluator.java?rev=1092645&r1=1092644&r2=1092645&view=diff
==============================================================================
---
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutFitnessEvaluator.java
(original)
+++
mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutFitnessEvaluator.java
Fri Apr 15 09:48:07 2011
@@ -20,6 +20,7 @@ package org.apache.mahout.ga.watchmaker;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.fs.Path;
import org.uncommons.watchmaker.framework.FitnessEvaluator;
/** Watchmaker compatible Fitness Evaluator that delegates the evaluation of
the whole population to Mahout. */
@@ -34,7 +35,7 @@ public class MahoutFitnessEvaluator<T> e
@Override
protected void evaluate(List<? extends T> population, List<Double>
evaluations) {
try {
- MahoutEvaluator.evaluate(evaluator, population, evaluations);
+ MahoutEvaluator.evaluate(evaluator, population, evaluations, new
Path("input"), new Path("output"));
} catch (IOException e) {
throw new IllegalStateException("Exception while evaluating the
population", e);
} catch (ClassNotFoundException e) {
Modified:
mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java
URL:
http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java?rev=1092645&r1=1092644&r2=1092645&view=diff
==============================================================================
---
mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java
(original)
+++
mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java
Fri Apr 15 09:48:07 2011
@@ -32,16 +32,15 @@ import org.uncommons.watchmaker.framewor
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Collection;
public final class MahoutEvaluatorTest extends MahoutTestCase {
+ private static final int POPULATION_SIZE = 100;
+
@Test
public void testEvaluate() throws Exception {
// candidate population
- int populationSize = 100;
- List<DummyCandidate> population = DummyCandidate
- .generatePopulation(populationSize);
+ List<DummyCandidate> population =
DummyCandidate.generatePopulation(POPULATION_SIZE);
// fitness evaluator
DummyEvaluator.clearEvaluations();
@@ -49,10 +48,13 @@ public final class MahoutEvaluatorTest e
// run MahoutEvaluator
List<Double> results = new ArrayList<Double>();
- MahoutEvaluator.evaluate(evaluator, population, results);
+ Path input = getTestTempDirPath("input");
+ Path output = getTestTempDirPath("output");
+
+ MahoutEvaluator.evaluate(evaluator, population, results, input, output);
// check results
- assertEquals("Number of evaluations", populationSize, results.size());
+ assertEquals("Number of evaluations", POPULATION_SIZE, results.size());
for (int index = 0; index < population.size(); index++) {
DummyCandidate candidate = population.get(index);
assertEquals("Evaluation of the candidate " + index,
@@ -62,40 +64,31 @@ public final class MahoutEvaluatorTest e
@Test
public void testStoreLoadPopulation() throws Exception {
- int populationSize = 100;
-
- List<DummyCandidate> population = DummyCandidate
- .generatePopulation(populationSize);
-
- storeLoadPopulation(population);
- }
+ List<DummyCandidate> population =
DummyCandidate.generatePopulation(POPULATION_SIZE);
- private static void storeLoadPopulation(List<DummyCandidate> population)
- throws IOException {
- Path f = new Path("build/test.txt");
- FileSystem fs = FileSystem.get(f.toUri(), new Configuration());
+ Path tempPath = getTestTempFilePath("test.txt");
+ FileSystem fs = tempPath.getFileSystem(new Configuration());
// store the population
- MahoutEvaluator.storePopulation(fs, f, population);
+ MahoutEvaluator.storePopulation(fs, tempPath, population);
// load the population
- List<DummyCandidate> inpop = new ArrayList<DummyCandidate>();
- loadPopulation(fs, f, inpop);
+ List<DummyCandidate> inpop = loadPopulation(fs, tempPath);
// check that the file contains the correct population
assertEquals("Population size", population.size(), inpop.size());
for (int index = 0; index < population.size(); index++) {
- assertEquals("Bad candidate " + index, population.get(index), inpop
- .get(index));
+ assertEquals("Bad candidate " + index, population.get(index),
inpop.get(index));
}
}
- private static void loadPopulation(FileSystem fs, Path f,
- Collection<DummyCandidate> population)
throws IOException {
+ private static List<DummyCandidate> loadPopulation(FileSystem fs, Path f)
throws IOException {
+ List<DummyCandidate> population = new ArrayList<DummyCandidate>();
FSDataInputStream in = fs.open(f);
for (String line : new FileLineIterable(in)) {
population.add(StringUtils.<DummyCandidate>fromString(line));
}
+ return population;
}
}
