Author: srowen
Date: Wed Nov 12 04:43:18 2008
New Revision: 713356
URL: http://svn.apache.org/viewvc?rev=713356&view=rev
Log:
Ensure readers are closed in BayesFileFormatter and tighten up close() logic a
little elsewhere
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/OutputUtils.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/WeightedDistanceMeasure.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java?rev=713356&r1=713355&r2=713356&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
Wed Nov 12 04:43:18 2008
@@ -26,7 +26,6 @@
import org.apache.commons.cli2.OptionException;
import org.apache.commons.cli2.commandline.Parser;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -42,6 +41,7 @@
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
+import java.io.Closeable;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
@@ -72,10 +72,12 @@
Charset charset, File outputFile) throws IOException {
Writer writer = new OutputStreamWriter(new FileOutputStream(outputFile),
charset);
- inputDir.listFiles(new FileProcessor(label, analyzer, charset, writer));
- // listFiles() is called here as a way to recursively visit files, actually
- writer.close();
-
+ try {
+ inputDir.listFiles(new FileProcessor(label, analyzer, charset, writer));
+ // listFiles() is called here as a way to recursively visit files,
actually
+ } finally {
+ quietClose(writer);
+ }
}
/**
@@ -96,9 +98,11 @@
} else {
Writer writer = new OutputStreamWriter(new FileOutputStream(new File(
outDir, input.getName())), charset);
- writeFile(label, analyzer, new InputStreamReader(new FileInputStream(
- input), charset), writer);
- writer.close();
+ try {
+ writeFile(label, analyzer, input, charset, writer);
+ } finally {
+ quietClose(writer);
+ }
}
}
@@ -151,27 +155,26 @@
public boolean accept(File file) {
if (file.isFile()) {
+ Writer theWriter = null;
try {
- Writer theWriter;
if (writer == null) {
theWriter = new OutputStreamWriter(new FileOutputStream(new File(
outputDir, file.getName())), charset);
} else {
theWriter = writer;
}
- writeFile(label, analyzer, new InputStreamReader(new FileInputStream(
- file), charset), theWriter);
- if (writer == null) {
- theWriter.close();// we are writing a single file per input file
- } else {
+ writeFile(label, analyzer, file, charset, theWriter);
+ if (writer != null) {
// just write a new line
theWriter.write(LINE_SEP);
-
}
-
} catch (IOException e) {
// TODO: report failed files instead of throwing exception
throw new RuntimeException(e);
+ } finally {
+ if (writer == null) {
+ quietClose(theWriter);
+ }
}
} else {
file.listFiles(this);
@@ -185,31 +188,39 @@
*
* @param label The label
* @param analyzer The analyzer to use
- * @param reader The reader to pass to the Analyzer
+ * @param inFile the file to read and whose contents are passed to the
analyzer
+ * @param charset character encoding to assume when reading the input file
* @param writer The Writer, is not closed by this method
* @throws java.io.IOException if there was a problem w/ the reader
*/
- public static void writeFile(String label, Analyzer analyzer, Reader reader,
- Writer writer) throws IOException {
- TokenStream ts = analyzer.tokenStream(label, reader);
- writer.write(label);
- writer.write('\t'); // edit: Inorder to match Hadoop standard
- // TextInputFormat
- Token token = new Token();
- //CharArraySet seen = new CharArraySet(256, false);
- //long numTokens = 0;
- while ((token = ts.next(token)) != null) {
- char[] termBuffer = token.termBuffer();
- int termLen = token.termLength();
-
- writer.write(termBuffer, 0, termLen);
- writer.write(' ');
- //char[] tmp = new char[termLen];
- //System.arraycopy(termBuffer, 0, tmp, 0, termLen);
- //seen.add(tmp);// do this b/c CharArraySet doesn't allow offsets
+ private static void writeFile(String label, Analyzer analyzer, File inFile,
+ Charset charset, Writer writer) throws IOException {
+ Reader reader = new InputStreamReader(new FileInputStream(inFile),
charset);
+ try {
+ TokenStream ts = analyzer.tokenStream(label, reader);
+ writer.write(label);
+ writer.write('\t'); // edit: Inorder to match Hadoop standard
+ // TextInputFormat
+ Token token = new Token();
+ while ((token = ts.next(token)) != null) {
+ char[] termBuffer = token.termBuffer();
+ int termLen = token.termLength();
+ writer.write(termBuffer, 0, termLen);
+ writer.write(' ');
+ }
+ } finally {
+ quietClose(reader);
}
- ///numTokens++;
+ }
+ private static void quietClose(Closeable closeable) {
+ if (closeable != null) {
+ try {
+ closeable.close();
+ } catch (IOException ioe) {
+ // continue
+ }
+ }
}
/**
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java?rev=713356&r1=713355&r2=713356&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java
Wed Nov 12 04:43:18 2008
@@ -192,26 +192,28 @@
Text.class, DoubleWritable.class);
MapFile.Writer.setIndexInterval(conf, 3);
- FileStatus[] outputFiles = fs.globStatus(pathPattern);
- for (FileStatus fileStatus : outputFiles) {
- Path path = fileStatus.getPath();
- log.info("{}", path);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- // the key is either _label_ or label,feature
- while (reader.next(key, value)) {
- String keyStr = key.toString();
- if (!keyStr.startsWith("_") && !keyStr.startsWith(",") &&
!keyStr.startsWith("*")) {
- int idx = keyStr.indexOf(',');
- if (idx != -1) {
- //Map<String,Double> data = new HashMap<String,Double>();
- //data.put(keyStr.substring(0, idx), value.get());
- writer.append(new Text(key.toString()), value);
+ try {
+ FileStatus[] outputFiles = fs.globStatus(pathPattern);
+ for (FileStatus fileStatus : outputFiles) {
+ Path path = fileStatus.getPath();
+ log.info("{}", path);
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
+ // the key is either _label_ or label,feature
+ while (reader.next(key, value)) {
+ String keyStr = key.toString();
+ if (!keyStr.startsWith("_") && !keyStr.startsWith(",") &&
!keyStr.startsWith("*")) {
+ int idx = keyStr.indexOf(',');
+ if (idx != -1) {
+ //Map<String,Double> data = new HashMap<String,Double>();
+ //data.put(keyStr.substring(0, idx), value.get());
+ writer.append(new Text(key.toString()), value);
+ }
}
}
}
+ } finally {
+ writer.close();
}
- writer.close();
- // return model;
}
public Map<String,Double> readLabelSums(FileSystem fs, Path pathPattern,
Configuration conf) throws IOException {
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/OutputUtils.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/OutputUtils.java?rev=713356&r1=713355&r2=713356&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/OutputUtils.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/OutputUtils.java
Wed Nov 12 04:43:18 2008
@@ -104,12 +104,13 @@
LongWritable key = new LongWritable();
DoubleWritable value = new DoubleWritable();
Reader reader = new Reader(fs, output, conf);
-
- while (reader.next(key, value)) {
- evaluations.add(value.get());
+ try {
+ while (reader.next(key, value)) {
+ evaluations.add(value.get());
+ }
+ } finally {
+ reader.close();
}
-
- reader.close();
}
}
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/WeightedDistanceMeasure.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/WeightedDistanceMeasure.java?rev=713356&r1=713355&r2=713356&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/WeightedDistanceMeasure.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/WeightedDistanceMeasure.java
Wed Nov 12 04:43:18 2008
@@ -67,8 +67,11 @@
throw new FileNotFoundException(weightsFile.get().toString());
}
DataInputStream in = fs.open(weightsFile.get());
- weights.readFields(in);
- in.close();
+ try {
+ weights.readFields(in);
+ } finally {
+ in.close();
+ }
this.weights = weights;
}
} catch (IOException e) {
