Author: markus
Date: Fri Nov 11 11:55:21 2011
New Revision: 1200830
URL: http://svn.apache.org/viewvc?rev=1200830&view=rev
Log:
NUTCH-1180 UpdateDB to backup previous CrawlDB
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1200830&r1=1200829&r2=1200830&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Fri Nov 11 11:55:21 2011
@@ -1,5 +1,7 @@
Nutch Change Log
+* NUTCH-1180 UpdateDB to backup previous CrawlDB (markus)
+
* NUTCH-1173 DomainStats doesn't count db_not_modified (markus)
* NUTCH-1155 Host/domain limit in generator is generate.max.count+1 (markus)
Modified: nutch/trunk/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1200830&r1=1200829&r2=1200830&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Fri Nov 11 11:55:21 2011
@@ -404,6 +404,15 @@
</property>
<property>
+ <name>db.preserve.backup</name>
+ <value>true</value>
+ <description>If true, updatedb will keep a backup of the previous CrawlDB
+ version in the old directory. In case of disaster, one can rename old to
+ current and restore the CrawlDB to its previous state.
+ </description>
+</property>
+
+<property>
<name>db.update.purge.404</name>
<value>false</value>
<description>If true, updatedb will add purge records with status DB_GONE
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java?rev=1200830&r1=1200829&r2=1200830&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java Fri Nov 11
11:55:21 2011
@@ -146,6 +146,8 @@ public class CrawlDb extends Configured
}
public static void install(JobConf job, Path crawlDb) throws IOException {
+ boolean preserveBackup = job.getBoolean("db.preserve.backup", true);
+
Path newCrawlDb = FileOutputFormat.getOutputPath(job);
FileSystem fs = new JobClient(job).getFs();
Path old = new Path(crawlDb, "old");
@@ -156,7 +158,7 @@ public class CrawlDb extends Configured
}
fs.mkdirs(crawlDb);
fs.rename(newCrawlDb, current);
- if (fs.exists(old)) fs.delete(old, true);
+ if (!preserveBackup && fs.exists(old)) fs.delete(old, true);
Path lock = new Path(crawlDb, LOCK_NAME);
LockUtil.removeLockFile(fs, lock);
}