Author: srowen
Date: Sat Jan 17 07:24:43 2009
New Revision: 735304
URL: http://svn.apache.org/viewvc?rev=735304&view=rev
Log:
Support compressed input files
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java?rev=735304&r1=735303&r2=735304&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java
Sat Jan 17 07:24:43 2009
@@ -26,6 +26,8 @@
* Iterable representing the lines of a text file. It can produce an {...@link
Iterator} over those lines.
* This assumes the text file's lines are delimited in a manner
* consistent with how {...@link java.io.BufferedReader} defines lines.
+ *
+ * This class will uncompress files that end in .zip or .gz accordingly, too.
*/
public final class FileLineIterable implements Iterable<String> {
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java?rev=735304&r1=735303&r2=735304&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java
Sat Jan 17 07:24:43 2009
@@ -27,12 +27,16 @@
import java.io.InputStreamReader;
import java.util.Iterator;
import java.util.NoSuchElementException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipInputStream;
import java.nio.charset.Charset;
/**
* Iterates over the lines of a text file. This assumes the text file's lines
* are delimited in a manner consistent with how {...@link BufferedReader}
* defines lines.
+ *
+ * This class will uncompress files that end in .zip or .gz accordingly, too.
*/
public final class FileLineIterator implements Iterator<String>, Closeable {
@@ -56,7 +60,7 @@
* @throws IOException if the file cannot be read
*/
public FileLineIterator(File file, Charset encoding, boolean skipFirstLine)
throws IOException {
- InputStream is = new FileInputStream(file);
+ InputStream is = getFileInputStream(file);
reader = new BufferedReader(new InputStreamReader(is, encoding));
if (skipFirstLine) {
reader.readLine();
@@ -64,6 +68,18 @@
nextLine = reader.readLine();
}
+ private static InputStream getFileInputStream(File file) throws IOException {
+ InputStream is = new FileInputStream(file);
+ String name = file.getName();
+ if (name.endsWith(".gz")) {
+ return new GZIPInputStream(is);
+ } else if (name.endsWith(".zip")) {
+ return new ZipInputStream(is);
+ } else {
+ return is;
+ }
+ }
+
@Override
public boolean hasNext() {
return nextLine != null;
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java?rev=735304&r1=735303&r2=735304&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
Sat Jan 17 07:24:43 2009
@@ -72,7 +72,8 @@
private final ReentrantLock reloadLock;
/**
- * @param dataFile file containing preferences data
+ * @param dataFile file containing preferences data. If file is compressed
(and name ends in .gz
+ * or .zip accordingly) it will be decompressed as it is read)
* @throws FileNotFoundException if dataFile does not exist
*/
public FileDataModel(File dataFile) throws FileNotFoundException {
