Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/ChunkedWriter.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/ChunkedWriter.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/text/ChunkedWriter.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/ChunkedWriter.java Sat Jun 4 08:42:57 2011 @@ -16,6 +16,7 @@ */ package org.apache.mahout.text; +import com.google.common.io.Closeables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -53,7 +54,7 @@ public final class ChunkedWriter impleme public void write(String key, String value) throws IOException { if (currentChunkSize > maxChunkSizeInBytes) { - writer.close(); + Closeables.closeQuietly(writer); writer = new SequenceFile.Writer(fs, conf, getPath(currentChunkID++), Text.class, Text.class); currentChunkSize = 0; } @@ -66,7 +67,7 @@ public final class ChunkedWriter impleme @Override public void close() throws IOException { - writer.close(); + Closeables.closeQuietly(writer); } }
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromDirectory.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromDirectory.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromDirectory.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromDirectory.java Sat Jun 4 08:42:57 2011 @@ -23,6 +23,7 @@ import java.lang.reflect.InvocationTarge import java.util.HashMap; import java.util.Map; +import com.google.common.io.Closeables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -61,20 +62,24 @@ public class SequenceFilesFromDirectory NoSuchMethodException, ClassNotFoundException { FileSystem fs = FileSystem.get(conf); ChunkedWriter writer = new ChunkedWriter(conf, Integer.parseInt(options.get(CHUNK_SIZE_OPTION[0])), output); - - SequenceFilesFromDirectoryFilter pathFilter; - String fileFilterClassName = options.get(FILE_FILTER_CLASS_OPTION[0]); - if (PrefixAdditionFilter.class.getName().equals(fileFilterClassName)) { - pathFilter = new PrefixAdditionFilter(conf, keyPrefix, options, writer); - } else { - Class<? extends SequenceFilesFromDirectoryFilter> pathFilterClass = Class.forName(fileFilterClassName).asSubclass(SequenceFilesFromDirectoryFilter.class); - Constructor<? extends SequenceFilesFromDirectoryFilter> constructor = - pathFilterClass.getConstructor(Configuration.class, String.class, Map.class, ChunkedWriter.class); - pathFilter = constructor.newInstance(conf, keyPrefix, options, writer); + try { + SequenceFilesFromDirectoryFilter pathFilter; + + String fileFilterClassName = options.get(FILE_FILTER_CLASS_OPTION[0]); + if (PrefixAdditionFilter.class.getName().equals(fileFilterClassName)) { + pathFilter = new PrefixAdditionFilter(conf, keyPrefix, options, writer); + } else { + Class<? extends SequenceFilesFromDirectoryFilter> pathFilterClass = + Class.forName(fileFilterClassName).asSubclass(SequenceFilesFromDirectoryFilter.class); + Constructor<? extends SequenceFilesFromDirectoryFilter> constructor = + pathFilterClass.getConstructor(Configuration.class, String.class, Map.class, ChunkedWriter.class); + pathFilter = constructor.newInstance(conf, keyPrefix, options, writer); + } + fs.listStatus(input, pathFilter); + } finally { + Closeables.closeQuietly(writer); } - fs.listStatus(input, pathFilter); - writer.close(); } public static void main(String[] args) throws Exception { Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromMailArchives.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromMailArchives.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromMailArchives.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromMailArchives.java Sat Jun 4 08:42:57 2011 @@ -25,6 +25,7 @@ import java.nio.charset.Charset; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.io.Closeables; import org.apache.commons.cli2.CommandLine; import org.apache.commons.cli2.Group; import org.apache.commons.cli2.Option; @@ -63,11 +64,13 @@ public final class SequenceFilesFromMail int chunkSizeInMB, Charset charset) throws IOException { ChunkedWriter writer = createNewChunkedWriter(chunkSizeInMB, outputDir); - PrefixAdditionFilter filter = new PrefixAdditionFilter(prefix, writer, charset); - parentDir.listFiles(filter); - writer.close(); - - log.info("Parsed "+filter.getMessageCount()+" messages from "+parentDir.getAbsolutePath()); + try { + PrefixAdditionFilter filter = new PrefixAdditionFilter(prefix, writer, charset); + parentDir.listFiles(filter); + log.info("Parsed "+filter.getMessageCount()+" messages from "+parentDir.getAbsolutePath()); + } finally { + Closeables.closeQuietly(writer); + } } public static class ChunkedWriter implements Closeable { @@ -97,7 +100,7 @@ public final class SequenceFilesFromMail public void write(String key, String value) throws IOException { if (currentChunkSize > maxChunkSizeInBytes) { - writer.close(); + Closeables.closeQuietly(writer); log.info("Chunk size ("+currentChunkSize+") reached MAX; creating new chunk "+(currentChunkID+1)); writer = SequenceFile.createWriter(fs, conf, getPath(currentChunkID++), Text.class, Text.class, SequenceFile.CompressionType.BLOCK); currentChunkSize = 0; @@ -111,7 +114,7 @@ public final class SequenceFilesFromMail @Override public void close() throws IOException { - writer.close(); + Closeables.closeQuietly(writer); } } Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java Sat Jun 4 08:42:57 2011 @@ -22,6 +22,7 @@ import java.io.OutputStreamWriter; import java.io.Writer; import com.google.common.base.Charsets; +import com.google.common.io.Closeables; import com.google.common.io.Files; import org.apache.commons.cli2.CommandLine; import org.apache.commons.cli2.Group; @@ -119,7 +120,7 @@ public final class SequenceFileDumper { writer.append("Count: ").append(String.valueOf(count)).append('\n'); } } finally { - writer.close(); + Closeables.closeQuietly(writer); } } Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java Sat Jun 4 08:42:57 2011 @@ -32,6 +32,7 @@ import java.util.Map; import java.util.TreeMap; import com.google.common.base.Charsets; +import com.google.common.io.Closeables; import com.google.common.io.Files; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -181,7 +182,7 @@ public final class ClusterDumper extends } } finally { if (shouldClose) { - writer.close(); + Closeables.closeQuietly(writer); } } } Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/RowIdJob.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/RowIdJob.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/RowIdJob.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/RowIdJob.java Sat Jun 4 08:42:57 2011 @@ -17,6 +17,7 @@ package org.apache.mahout.utils.vectors; +import com.google.common.io.Closeables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -53,23 +54,26 @@ public class RowIdJob extends AbstractJo matrixPath, IntWritable.class, VectorWritable.class); - IntWritable docId = new IntWritable(); - int i = 0; - int numCols = 0; - for (Pair<Text,VectorWritable> record : + try { + IntWritable docId = new IntWritable(); + int i = 0; + int numCols = 0; + for (Pair<Text,VectorWritable> record : new SequenceFileDirIterable<Text,VectorWritable>(inputPath, PathType.LIST, null, null, true, conf)) { - VectorWritable value = record.getSecond(); - docId.set(i); - indexWriter.append(docId, record.getFirst()); - matrixWriter.append(docId, value); - i++; - numCols = value.get().size(); + VectorWritable value = record.getSecond(); + docId.set(i); + indexWriter.append(docId, record.getFirst()); + matrixWriter.append(docId, value); + i++; + numCols = value.get().size(); + } + + log.info("Wrote out matrix with {} rows and {} columns to {}", new Object[] { i, numCols, matrixPath }); + return 0; + } finally { + Closeables.closeQuietly(indexWriter); + Closeables.closeQuietly(matrixWriter); } - - matrixWriter.close(); - indexWriter.close(); - log.info("Wrote out matrix with {} rows and {} columns to {}", new Object[] { i, numCols, matrixPath }); - return 0; } public static void main(String[] args) throws Exception { Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java Sat Jun 4 08:42:57 2011 @@ -18,6 +18,7 @@ package org.apache.mahout.utils.vectors; import com.google.common.base.Charsets; +import com.google.common.io.Closeables; import com.google.common.io.Files; import org.apache.commons.cli2.CommandLine; import org.apache.commons.cli2.Group; @@ -180,7 +181,7 @@ public final class VectorDumper { } } } finally { - writer.close(); + Closeables.closeQuietly(writer); } } Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java Sat Jun 4 08:42:57 2011 @@ -24,6 +24,7 @@ import java.io.Writer; import java.util.Map; import com.google.common.base.Charsets; +import com.google.common.io.Closeables; import com.google.common.io.Files; import org.apache.commons.cli2.CommandLine; import org.apache.commons.cli2.Group; @@ -137,7 +138,7 @@ public final class Driver { writer.write('\n'); } } finally { - writer.close(); + Closeables.closeQuietly(writer); } } @@ -159,7 +160,7 @@ public final class Driver { long numDocs = vectorWriter.write(iteratable, maxDocs); log.info("Wrote: {} vectors", numDocs); } finally { - vectorWriter.close(); + Closeables.closeQuietly(vectorWriter); } } Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/DelimitedTermInfoWriter.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/DelimitedTermInfoWriter.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/DelimitedTermInfoWriter.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/DelimitedTermInfoWriter.java Sat Jun 4 08:42:57 2011 @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.Writer; import java.util.Iterator; +import com.google.common.io.Closeables; import org.apache.mahout.utils.vectors.TermEntry; import org.apache.mahout.utils.vectors.TermInfo; @@ -42,22 +43,23 @@ public class DelimitedTermInfoWriter imp public void write(TermInfo ti) throws IOException { Iterator<TermEntry> entIter = ti.getAllEntries(); - - writer.write(String.valueOf(ti.totalTerms(field))); - writer.write('\n'); - writer.write("#term" + delimiter + "doc freq" + delimiter + "idx"); - writer.write('\n'); - while (entIter.hasNext()) { - TermEntry entry = entIter.next(); - writer.write(entry.getTerm()); - writer.write(delimiter); - writer.write(String.valueOf(entry.getDocFreq())); - writer.write(delimiter); - writer.write(String.valueOf(entry.getTermIdx())); + try { + writer.write(String.valueOf(ti.totalTerms(field))); + writer.write('\n'); + writer.write("#term" + delimiter + "doc freq" + delimiter + "idx"); writer.write('\n'); + while (entIter.hasNext()) { + TermEntry entry = entIter.next(); + writer.write(entry.getTerm()); + writer.write(delimiter); + writer.write(String.valueOf(entry.getDocFreq())); + writer.write(delimiter); + writer.write(String.valueOf(entry.getTermIdx())); + writer.write('\n'); + } + } finally { + Closeables.closeQuietly(writer); } - writer.flush(); - writer.close(); } /** Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/SequenceFileVectorWriter.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/SequenceFileVectorWriter.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/SequenceFileVectorWriter.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/SequenceFileVectorWriter.java Sat Jun 4 08:42:57 2011 @@ -19,6 +19,7 @@ package org.apache.mahout.utils.vectors. import java.io.IOException; +import com.google.common.io.Closeables; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.mahout.math.Vector; @@ -65,9 +66,7 @@ public class SequenceFileVectorWriter im @Override public void close() throws IOException { - if (writer != null) { - writer.close(); - } + Closeables.closeQuietly(writer); } public SequenceFile.Writer getWriter() { Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/TextualVectorWriter.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/TextualVectorWriter.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/TextualVectorWriter.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/io/TextualVectorWriter.java Sat Jun 4 08:42:57 2011 @@ -20,6 +20,7 @@ package org.apache.mahout.utils.vectors. import java.io.IOException; import java.io.Writer; +import com.google.common.io.Closeables; import org.apache.mahout.math.Vector; /** @@ -64,7 +65,6 @@ public class TextualVectorWriter impleme @Override public void close() throws IOException { - writer.flush(); - writer.close(); + Closeables.closeQuietly(writer); } } Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java Sat Jun 4 08:42:57 2011 @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import com.google.common.io.Closeables; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermEnum; @@ -40,24 +41,27 @@ public class CachedTermInfo implements T public CachedTermInfo(IndexReader reader, String field, int minDf, int maxDfPercent) throws IOException { this.field = field; TermEnum te = reader.terms(new Term(field, "")); - int numDocs = reader.numDocs(); - double percent = numDocs * maxDfPercent / 100.0; - //Should we use a linked hash map so that we know terms are in order? - termEntries = new LinkedHashMap<String, TermEntry>(); - int count = 0; - do { - Term term = te.term(); - if (term == null || !term.field().equals(field)) { - break; - } - int df = te.docFreq(); - if (df < minDf || df > percent) { - continue; - } - TermEntry entry = new TermEntry(term.text(), count++, df); - termEntries.put(entry.getTerm(), entry); - } while (te.next()); - te.close(); + try { + int numDocs = reader.numDocs(); + double percent = numDocs * maxDfPercent / 100.0; + //Should we use a linked hash map so that we know terms are in order? + termEntries = new LinkedHashMap<String, TermEntry>(); + int count = 0; + do { + Term term = te.term(); + if (term == null || !term.field().equals(field)) { + break; + } + int df = te.docFreq(); + if (df < minDf || df > percent) { + continue; + } + TermEntry entry = new TermEntry(term.text(), count++, df); + termEntries.put(entry.getTerm(), entry); + } while (te.next()); + } finally { + Closeables.closeQuietly(te); + } } @Override Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java Sat Jun 4 08:42:57 2011 @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import com.google.common.base.Charsets; +import com.google.common.io.Closeables; import com.google.common.io.Files; import org.apache.commons.cli2.CommandLine; import org.apache.commons.cli2.Group; @@ -134,7 +135,7 @@ public class ClusterLabels { } } } finally { - writer.close(); + Closeables.closeQuietly(writer); } } @@ -180,32 +181,36 @@ public class ClusterLabels { */ TermEnum te = reader.terms(new Term(contentField, "")); - int count = 0; - Map<String, TermEntry> termEntryMap = new LinkedHashMap<String, TermEntry>(); - do { - Term term = te.term(); - if (term == null || !term.field().equals(contentField)) { - break; - } - OpenBitSet termBitset = new OpenBitSet(reader.maxDoc()); - // Generate bitset for the term - TermDocs termDocs = reader.termDocs(term); + try { + int count = 0; - while (termDocs.next()) { - termBitset.set(termDocs.doc()); - } + do { + Term term = te.term(); + if (term == null || !term.field().equals(contentField)) { + break; + } + OpenBitSet termBitset = new OpenBitSet(reader.maxDoc()); - // AND the term's bitset with cluster doc bitset to get the term's in-cluster frequency. - // This modifies the termBitset, but that's fine as we are not using it anywhere else. - termBitset.and(clusterDocBitset); - int inclusterDF = (int) termBitset.cardinality(); + // Generate bitset for the term + TermDocs termDocs = reader.termDocs(term); - TermEntry entry = new TermEntry(term.text(), count++, inclusterDF); - termEntryMap.put(entry.getTerm(), entry); - } while (te.next()); - te.close(); + while (termDocs.next()) { + termBitset.set(termDocs.doc()); + } + + // AND the term's bitset with cluster doc bitset to get the term's in-cluster frequency. + // This modifies the termBitset, but that's fine as we are not using it anywhere else. + termBitset.and(clusterDocBitset); + int inclusterDF = (int) termBitset.cardinality(); + + TermEntry entry = new TermEntry(term.text(), count++, inclusterDF); + termEntryMap.put(entry.getTerm(), entry); + } while (te.next()); + } finally { + Closeables.closeQuietly(te); + } List<TermInfoClusterInOut> clusteredTermInfo = new LinkedList<TermInfoClusterInOut>(); @@ -223,7 +228,7 @@ public class ClusterLabels { Collections.sort(clusteredTermInfo); // Cleanup - reader.close(); + Closeables.closeQuietly(reader); termEntryMap.clear(); return clusteredTermInfo.subList(0, Math.min(clusteredTermInfo.size(), maxLabels)); Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java (original) +++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java Sat Jun 4 08:42:57 2011 @@ -23,6 +23,7 @@ import java.io.Writer; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.google.common.io.Closeables; import com.google.common.io.Files; import org.apache.commons.cli2.CommandLine; import org.apache.commons.cli2.Group; @@ -104,18 +105,22 @@ public final class Driver { log.info("Output File: {}", outFile); VectorWriter vectorWriter = getSeqFileWriter(outFile); - - long numDocs = vectorWriter.write(iterable, maxDocs); - vectorWriter.close(); - log.info("Wrote: {} vectors", numDocs); + try { + long numDocs = vectorWriter.write(iterable, maxDocs); + log.info("Wrote: {} vectors", numDocs); + } finally { + Closeables.closeQuietly(vectorWriter); + } File dictOutFile = new File(dictOut); log.info("Dictionary Output file: {}", dictOutFile); Writer writer = Files.newWriter(dictOutFile, Charsets.UTF_8); DelimitedTermInfoWriter tiWriter = new DelimitedTermInfoWriter(writer, delimiter, field); - tiWriter.write(termInfo); - tiWriter.close(); - writer.close(); + try { + tiWriter.write(termInfo); + } finally { + Closeables.closeQuietly(tiWriter); + } } public static void main(String[] args) throws IOException { Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java (original) +++ mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java Sat Jun 4 08:42:57 2011 @@ -17,6 +17,7 @@ package org.apache.mahout.clustering; +import com.google.common.io.Closeables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -112,16 +113,19 @@ public final class TestClusterDumper ext new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED); - for (int i = 0; i < docs2.length; i++) { - Document doc = new Document(); - Fieldable id = new Field("id", "doc_" + i, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add(id); - // Store both position and offset information - Fieldable text = new Field("content", docs2[i], Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES); - doc.add(text); - writer.addDocument(doc); + try { + for (int i = 0; i < docs2.length; i++) { + Document doc = new Document(); + Fieldable id = new Field("id", "doc_" + i, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); + doc.add(id); + // Store both position and offset information + Fieldable text = new Field("content", docs2[i], Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES); + doc.add(text); + writer.addDocument(doc); + } + } finally { + Closeables.closeQuietly(writer); } - writer.close(); IndexReader reader = IndexReader.open(directory, true); Weight weight = new TFIDF(); TermInfo termInfo = new CachedTermInfo(reader, "content", 1, 100); @@ -316,7 +320,7 @@ public final class TestClusterDumper ext writer.append(key, value); } } finally { - writer.close(); + Closeables.closeQuietly(writer); } // now run the Canopy job to prime kMeans canopies CanopyDriver.run(conf, svdData, output, measure, 8, 4, false, true); Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/dirichlet/TestL1ModelClustering.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/dirichlet/TestL1ModelClustering.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/dirichlet/TestL1ModelClustering.java (original) +++ mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/dirichlet/TestL1ModelClustering.java Sat Jun 4 08:42:57 2011 @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; +import com.google.common.io.Closeables; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -104,16 +105,19 @@ public final class TestL1ModelClustering new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED); - for (int i = 0; i < docs2.length; i++) { - Document doc = new Document(); - Fieldable id = new Field("id", "doc_" + i, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add(id); - // Store both position and offset information - Fieldable text = new Field("content", docs2[i], Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES); - doc.add(text); - writer.addDocument(doc); + try { + for (int i = 0; i < docs2.length; i++) { + Document doc = new Document(); + Fieldable id = new Field("id", "doc_" + i, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); + doc.add(id); + // Store both position and offset information + Fieldable text = new Field("content", docs2[i], Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES); + doc.add(text); + writer.addDocument(doc); + } + } finally { + Closeables.closeQuietly(writer); } - writer.close(); IndexReader reader = IndexReader.open(directory, true); Weight weight = new TFIDF(); TermInfo termInfo = new CachedTermInfo(reader, "content", 1, 100); Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/TestSequenceFilesFromDirectory.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/TestSequenceFilesFromDirectory.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/test/java/org/apache/mahout/text/TestSequenceFilesFromDirectory.java (original) +++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/TestSequenceFilesFromDirectory.java Sat Jun 4 08:42:57 2011 @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Map; import com.google.common.base.Charsets; +import com.google.common.io.Closeables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -110,19 +111,25 @@ public final class TestSequenceFilesFrom private static void createFilesFromArrays(Configuration conf, Path inputDir, String[][] data) throws IOException { FileSystem fs = FileSystem.get(conf); for (String[] aData : data) { - OutputStreamWriter osw = new OutputStreamWriter(fs.create(new Path(inputDir, aData[0])), Charsets.UTF_8); - osw.write(aData[1]); - osw.close(); + OutputStreamWriter writer = new OutputStreamWriter(fs.create(new Path(inputDir, aData[0])), Charsets.UTF_8); + try { + writer.write(aData[1]); + } finally { + Closeables.closeQuietly(writer); + } } } private static void createTsvFilesFromArrays(Configuration conf, Path inputDir, String[][] data) throws IOException { FileSystem fs = FileSystem.get(conf); - OutputStreamWriter osw = new OutputStreamWriter(fs.create(new Path(inputDir, "inputTsvFile"))); - for (String[] aData : data) { - osw.write(aData[0] + '\t' + aData[1] + '\n'); + OutputStreamWriter writer = new OutputStreamWriter(fs.create(new Path(inputDir, "inputTsvFile"))); + try { + for (String[] aData : data) { + writer.write(aData[0] + '\t' + aData[1] + '\n'); + } + } finally { + Closeables.closeQuietly(writer); } - osw.close(); } private static void checkChunkFiles(Configuration conf, @@ -149,14 +156,17 @@ public final class TestSequenceFilesFrom // read a chunk to check content SequenceFileIterator<Text,Text> iterator = new SequenceFileIterator<Text,Text>(fstats[0].getPath(), true, conf); - for (String[] datum : data) { - assertTrue(iterator.hasNext()); - Pair<Text,Text> record = iterator.next(); - String retrievedData = fileToData.get(record.getFirst().toString().trim()); - assertNotNull(retrievedData); - assertEquals(retrievedData, record.getSecond().toString().trim()); + try { + for (String[] datum : data) { + assertTrue(iterator.hasNext()); + Pair<Text,Text> record = iterator.next(); + String retrievedData = fileToData.get(record.getFirst().toString().trim()); + assertNotNull(retrievedData); + assertEquals(retrievedData, record.getSecond().toString().trim()); + } + } finally { + Closeables.closeQuietly(iterator); } - iterator.close(); } /** Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/csv/CSVVectorIteratorTest.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/csv/CSVVectorIteratorTest.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/csv/CSVVectorIteratorTest.java (original) +++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/csv/CSVVectorIteratorTest.java Sat Jun 4 08:42:57 2011 @@ -17,6 +17,7 @@ package org.apache.mahout.utils.vectors.csv; +import com.google.common.io.Closeables; import org.apache.mahout.math.Vector; import org.apache.mahout.utils.MahoutTestCase; import org.apache.mahout.utils.vectors.RandomVectorIterable; @@ -35,16 +36,19 @@ public class CSVVectorIteratorTest exten public void testCount() throws Exception { StringWriter sWriter = new StringWriter(); - TextualVectorWriter jwvw = new TextualVectorWriter(sWriter) { + TextualVectorWriter writer = new TextualVectorWriter(sWriter) { @Override public void write(Vector vector) throws IOException { String vecStr = VectorHelper.vectorToCSVString(vector, false); getWriter().write(vecStr); } }; - Iterable<Vector> iter = new RandomVectorIterable(50); - jwvw.write(iter); - jwvw.close(); + try { + Iterable<Vector> iter = new RandomVectorIterable(50); + writer.write(iter); + } finally { + Closeables.closeQuietly(writer); + } Iterator<Vector> csvIter = new CSVVectorIterator(new StringReader(sWriter.getBuffer().toString())); int count = 0; while (csvIter.hasNext()) { Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/io/VectorWriterTest.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/io/VectorWriterTest.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/io/VectorWriterTest.java (original) +++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/io/VectorWriterTest.java Sat Jun 4 08:42:57 2011 @@ -21,6 +21,7 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; +import com.google.common.io.Closeables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -43,9 +44,11 @@ public final class VectorWriterTest exte FileSystem fs = FileSystem.get(conf); SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, LongWritable.class, VectorWritable.class); SequenceFileVectorWriter writer = new SequenceFileVectorWriter(seqWriter); - Iterable<Vector> iter = new RandomVectorIterable(50); - writer.write(iter); - writer.close(); + try { + writer.write(new RandomVectorIterable(50)); + } finally { + Closeables.closeQuietly(writer); + } long count = HadoopUtil.countRecords(path, conf); assertEquals(50, count); @@ -55,11 +58,14 @@ public final class VectorWriterTest exte public void testTextOutputSize() throws Exception { StringWriter strWriter = new StringWriter(); VectorWriter writer = new TextualVectorWriter(strWriter); - Collection<Vector> vectors = new ArrayList<Vector>(); - vectors.add(new DenseVector(new double[]{0.3, 1.5, 4.5})); - vectors.add(new DenseVector(new double[]{1.3, 1.5, 3.5})); - writer.write(vectors); - writer.close(); + try { + Collection<Vector> vectors = new ArrayList<Vector>(); + vectors.add(new DenseVector(new double[]{0.3, 1.5, 4.5})); + vectors.add(new DenseVector(new double[]{1.3, 1.5, 3.5})); + writer.write(vectors); + } finally { + Closeables.closeQuietly(writer); + } String buffer = strWriter.toString(); assertNotNull(buffer); assertTrue(buffer.length() > 0); Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java?rev=1131350&r1=1131349&r2=1131350&view=diff ============================================================================== --- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java (original) +++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java Sat Jun 4 08:42:57 2011 @@ -17,6 +17,7 @@ package org.apache.mahout.utils.vectors.lucene; +import com.google.common.io.Closeables; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -172,16 +173,20 @@ public final class LuceneIterableTest ex new StandardAnalyzer(Version.LUCENE_30), createNew, IndexWriter.MaxFieldLength.UNLIMITED); - for (int i = 0; i < LuceneIterableTest.DOCS.length; i++) { - Document doc = new Document(); - Fieldable id = new Field("id", "doc_" + (i + startingId), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add(id); - //Store both position and offset information - Fieldable text = new Field("content", DOCS[i], Field.Store.NO, Field.Index.ANALYZED, termVector); - doc.add(text); - writer.addDocument(doc); + try { + for (int i = 0; i < LuceneIterableTest.DOCS.length; i++) { + Document doc = new Document(); + Fieldable id = new Field("id", "doc_" + (i + startingId), Field.Store.YES, + Field.Index.NOT_ANALYZED_NO_NORMS); + doc.add(id); + //Store both position and offset information + Fieldable text = new Field("content", DOCS[i], Field.Store.NO, Field.Index.ANALYZED, termVector); + doc.add(text); + writer.addDocument(doc); + } + } finally { + Closeables.closeQuietly(writer); } - writer.close(); return directory; } }
