Author: markus
Date: Fri Nov 11 11:55:21 2011
New Revision: 1200830

URL: http://svn.apache.org/viewvc?rev=1200830&view=rev
Log:
NUTCH-1180 UpdateDB to backup previous CrawlDB

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/conf/nutch-default.xml
    nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1200830&r1=1200829&r2=1200830&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Fri Nov 11 11:55:21 2011
@@ -1,5 +1,7 @@
 Nutch Change Log
 
+* NUTCH-1180 UpdateDB to backup previous CrawlDB (markus)
+
 * NUTCH-1173 DomainStats doesn't count db_not_modified (markus)
 
 * NUTCH-1155 Host/domain limit in generator is generate.max.count+1 (markus)

Modified: nutch/trunk/conf/nutch-default.xml
URL: 
http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1200830&r1=1200829&r2=1200830&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Fri Nov 11 11:55:21 2011
@@ -404,6 +404,15 @@
 </property>
 
 <property>
+  <name>db.preserve.backup</name>
+  <value>true</value>
+  <description>If true, updatedb will keep a backup of the previous CrawlDB
+  version in the old directory. In case of disaster, one can rename old to 
+  current and restore the CrawlDB to its previous state.
+  </description>
+</property>
+
+<property>
   <name>db.update.purge.404</name>
   <value>false</value>
   <description>If true, updatedb will add purge records with status DB_GONE

Modified: nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java?rev=1200830&r1=1200829&r2=1200830&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java Fri Nov 11 
11:55:21 2011
@@ -146,6 +146,8 @@ public class CrawlDb extends Configured 
   }
 
   public static void install(JobConf job, Path crawlDb) throws IOException {
+    boolean preserveBackup = job.getBoolean("db.preserve.backup", true);
+
     Path newCrawlDb = FileOutputFormat.getOutputPath(job);
     FileSystem fs = new JobClient(job).getFs();
     Path old = new Path(crawlDb, "old");
@@ -156,7 +158,7 @@ public class CrawlDb extends Configured 
     }
     fs.mkdirs(crawlDb);
     fs.rename(newCrawlDb, current);
-    if (fs.exists(old)) fs.delete(old, true);
+    if (!preserveBackup && fs.exists(old)) fs.delete(old, true);
     Path lock = new Path(crawlDb, LOCK_NAME);
     LockUtil.removeLockFile(fs, lock);
   }


Reply via email to