Author: cutting
Date: Thu Feb 16 15:31:52 2006
New Revision: 378396
URL: http://svn.apache.org/viewcvs?rev=378396&view=rev
Log:
Fix for NUTCH-211: add close method to search classes. Contributed by Stefan.
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/FetchedSegments.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitContent.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitInlinks.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/LinkDbInlinks.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Searcher.java
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java Thu
Feb 16 15:31:52 2006
@@ -32,7 +32,7 @@
import java.util.logging.Logger;
/** . */
-public class LinkDbReader {
+public class LinkDbReader implements Closeable {
public static final Logger LOG =
LogFormatter.getLogger(LinkDbReader.class.getName());
private static final Partitioner PARTITIONER = new HashPartitioner();
@@ -66,6 +66,14 @@
return (Inlinks)MapFileOutputFormat.getEntry
(readers, PARTITIONER, url, new Inlinks());
+ }
+
+ public void close() throws IOException {
+ if (readers != null) {
+ for (int i = 0; i < readers.length; i++) {
+ readers[i].close();
+ }
+ }
}
public static void processDumpJob(String linkdb, String output,
Configuration config) throws IOException {
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/FetchedSegments.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/FetchedSegments.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/FetchedSegments.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/FetchedSegments.java
Thu Feb 16 15:31:52 2006
@@ -20,6 +20,7 @@
import java.io.File;
import java.util.HashMap;
+import java.util.Iterator;
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
@@ -34,7 +35,7 @@
* fetched segments. */
public class FetchedSegments implements HitSummarizer, HitContent {
- private static class Segment {
+ private static class Segment implements Closeable {
private static final Partitioner PARTITIONER = new HashPartitioner();
private FileSystem fs;
@@ -93,6 +94,19 @@
return MapFileOutputFormat.getEntry(readers, PARTITIONER, url, entry);
}
+ public void close() throws IOException {
+ if (content != null) { closeReaders(content); }
+ if (parseText != null) { closeReaders(parseText); }
+ if (parseData != null) { closeReaders(parseData); }
+ if (crawl != null) { closeReaders(crawl); }
+ }
+
+ private void closeReaders(MapFile.Reader[] readers) throws IOException {
+ for (int i = 0; i < readers.length; i++) {
+ readers[i].close();
+ }
+ }
+
}
private HashMap segments = new HashMap();
@@ -206,5 +220,11 @@
return new UTF8(details.getValue("url"));
}
-
+ public void close() throws IOException {
+ Iterator iterator = segments.values().iterator();
+ while (iterator.hasNext()) {
+ ((Segment) iterator.next()).close();
+ }
+ }
+
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitContent.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitContent.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitContent.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitContent.java Thu
Feb 16 15:31:52 2006
@@ -18,11 +18,12 @@
import java.io.IOException;
+import org.apache.hadoop.io.Closeable;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.parse.ParseText;
/** Service that returns the content of a hit. */
-public interface HitContent {
+public interface HitContent extends Closeable {
/** Returns the content of a hit document. */
byte[] getContent(HitDetails details) throws IOException;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitInlinks.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitInlinks.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitInlinks.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/HitInlinks.java Thu
Feb 16 15:31:52 2006
@@ -18,10 +18,11 @@
import java.io.IOException;
+import org.apache.hadoop.io.Closeable;
import org.apache.nutch.crawl.Inlinks;
/** Service that returns information about incoming links to a hit. */
-public interface HitInlinks {
+public interface HitInlinks extends Closeable {
/** Returns the anchors of a hit document. */
String[] getAnchors(HitDetails details) throws IOException;
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
Thu Feb 16 15:31:52 2006
@@ -165,5 +165,10 @@
}
return new Hits(topDocs.totalHits, hits);
}
+
+ public void close() throws IOException {
+ if (luceneSearcher != null) { luceneSearcher.close(); }
+ if (reader != null) { reader.close(); }
+ }
}
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/LinkDbInlinks.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/LinkDbInlinks.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/LinkDbInlinks.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/LinkDbInlinks.java
Thu Feb 16 15:31:52 2006
@@ -30,4 +30,9 @@
public Inlinks getInlinks(HitDetails details) throws IOException {
return linkdb.getInlinks(new UTF8(details.getValue("url")));
}
+
+ public void close() throws IOException {
+ if (linkdb != null) { linkdb.close(); }
+ }
+
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java Thu
Feb 16 15:31:52 2006
@@ -22,6 +22,7 @@
import javax.servlet.ServletContext;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.Closeable;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.parse.*;
@@ -35,7 +36,7 @@
*/
public class NutchBean
implements Searcher, HitDetailer, HitSummarizer, HitContent, HitInlinks,
- DistributedSearch.Protocol {
+ DistributedSearch.Protocol, Closeable {
public static final Logger LOG =
LogFormatter.getLogger("org.apache.nutch.searcher.NutchBean");
@@ -352,6 +353,13 @@
return content.getFetchDate(hit);
}
+ public void close() throws IOException {
+ if (content != null) { content.close(); }
+ if (searcher != null) { searcher.close(); }
+ if (linkDb != null) { linkDb.close(); }
+ if (fs != null) { fs.close(); }
+ }
+
/** For debugging. */
public static void main(String[] args) throws Exception {
String usage = "NutchBean query";
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Searcher.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Searcher.java?rev=378396&r1=378395&r2=378396&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Searcher.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Searcher.java Thu Feb
16 15:31:52 2006
@@ -18,8 +18,10 @@
import java.io.IOException;
+import org.apache.hadoop.io.Closeable;
+
/** Service that searches. */
-public interface Searcher {
+public interface Searcher extends Closeable {
/** Return the top-scoring hits for a query. */
Hits search(Query query, int numHits,
String dedupField, String sortField, boolean reverse)
