Author: kubes
Date: Sun Nov 4 07:38:35 2007
New Revision: 591791
URL: http://svn.apache.org/viewvc?rev=591791&view=rev
Log:
NUTCH-552 - Upgrade Nutch to Hadoop 0.15.x.
Added:
lucene/nutch/trunk/lib/hadoop-0.15.0-core.jar (with props)
Removed:
lucene/nutch/trunk/lib/hadoop-0.12.3-core.jar
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/FsDirectory.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Sun Nov 4 07:38:35 2007
@@ -154,6 +154,8 @@
52. NUTCH-501 - Implement a different caching mechanism for objects cached in
configuration. (dogacan)
+53. NUTCH-552 - Upgrade Nutch to Hadoop 0.15.x. (kubes)
+
Release 0.9 - 2007-04-02
1. Changed log4j confiquration to log to stdout on commandline
Added: lucene/nutch/trunk/lib/hadoop-0.15.0-core.jar
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/hadoop-0.15.0-core.jar?rev=591791&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/lib/hadoop-0.15.0-core.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java Sun Nov 4
07:38:35 2007
@@ -172,7 +172,7 @@
CrawlDb.install(mergeJob, crawlDb);
// clean up
- FileSystem fs = new JobClient(getConf()).getFs();
+ FileSystem fs = FileSystem.get(getConf());
fs.delete(tempDir);
if (LOG.isInfoEnabled()) { LOG.info("Injector: done"); }
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java
Sun Nov 4 07:38:35 2007
@@ -174,7 +174,7 @@
this.index = index;
}
- public boolean next(Writable key, Writable value)
+ public boolean next(WritableComparable key, Writable value)
throws IOException {
// skip empty indexes
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/FsDirectory.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/FsDirectory.java?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/FsDirectory.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/FsDirectory.java Sun
Nov 4 07:38:35 2007
@@ -118,19 +118,9 @@
public Lock makeLock(final String name) {
return new Lock() {
public boolean obtain() {
- try {
- fs.lock(new Path(directory, name), false);
- return true;
- } catch (IOException e) {
- return false;
- }
+ return true;
}
public void release() {
- try {
- fs.release(new Path(directory, name));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
}
public boolean isLocked() {
throw new UnsupportedOperationException();
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/IndexSearcher.java
Sun Nov 4 07:38:35 2007
@@ -17,7 +17,9 @@
package org.apache.nutch.searcher;
+import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Enumeration;
@@ -81,7 +83,8 @@
private Directory getDirectory(Path file) throws IOException {
if ("local".equals(this.fs.getName())) {
- return FSDirectory.getDirectory(file.toString(), false);
+ URI fileUri = file.toUri();
+ return FSDirectory.getDirectory(new File(fileUri).toString(), false);
} else {
return new FsDirectory(this.fs, file, false, this.conf);
}
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java?rev=591791&r1=591790&r2=591791&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java Sun
Nov 4 07:38:35 2007
@@ -129,11 +129,8 @@
public RecordReader getRecordReader(InputSplit split,
JobConf job, Reporter reporter) {
- try{
- reporter.setStatus(split.toString());
- } catch (IOException e) {
- throw new RuntimeException("Cannot set status for reported:", e);
- }
+ reporter.setStatus(split.toString());
+
// find part name
SegmentPart segmentPart;
final String spString;
@@ -148,7 +145,7 @@
return new SequenceFileRecordReader(job, (FileSplit)split) {
@Override
- public synchronized boolean next(Writable key, Writable value)
throws IOException {
+ public synchronized boolean next(WritableComparable key, Writable
value) throws IOException {
LOG.debug("Running OIF.next()");
MetaWrapper wrapper = (MetaWrapper) value;
