Author: jnioche
Date: Fri Sep 20 08:03:24 2013
New Revision: 1524931
URL: http://svn.apache.org/r1524931
Log:
NUTCH-1641 Log timings for main jobs
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java
nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserJob.java
Modified: nutch/branches/2.x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1524931&r1=1524930&r2=1524931&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Fri Sep 20 08:03:24 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Current Development
+* NUTCH-1641 Log timings for main jobs (jnioche)
+
* NUTCH-1556 enabling updatedb to accept batchId (kaveh minooie,Feng)
* NUTCH-1619 Writes Dmoz Description and Title information to db with snippet
argument ( Yasin Kılınç via feng)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java?rev=1524931&r1=1524930&r2=1524931&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdaterJob.java Fri
Sep 20 08:03:24 2013
@@ -16,6 +16,7 @@
******************************************************************************/
package org.apache.nutch.crawl;
+import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
@@ -34,6 +35,7 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.NutchTool;
+import org.apache.nutch.util.TimingUtil;
import org.apache.nutch.util.ToolUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,7 +108,11 @@ public class DbUpdaterJob extends NutchT
}
private int updateTable(String crawlId,String batchId) throws Exception {
- LOG.info("DbUpdaterJob: starting");
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ long start = System.currentTimeMillis();
+ LOG.info("DbUpdaterJob: starting at " + sdf.format(start));
+
if (batchId.equals(Nutch.ALL_BATCH_ID_STR)) {
LOG.info("DbUpdaterJob: updatinging all");
} else {
@@ -114,7 +120,9 @@ public class DbUpdaterJob extends NutchT
}
run(ToolUtil.toArgMap(Nutch.ARG_CRAWL, crawlId,
Nutch.ARG_BATCH, batchId));
- LOG.info("DbUpdaterJob: done");
+
+ long finish = System.currentTimeMillis();
+ LOG.info("DbUpdaterJob: finished at " + sdf.format(finish) + ", time
elapsed: " + TimingUtil.elapsedTime(start, finish));
return 0;
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java?rev=1524931&r1=1524930&r2=1524931&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java Fri
Sep 20 08:03:24 2013
@@ -17,6 +17,7 @@
package org.apache.nutch.fetcher;
import java.io.IOException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -44,6 +45,7 @@ import org.apache.nutch.util.NutchConfig
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.NutchTool;
import org.apache.nutch.util.TableUtil;
+import org.apache.nutch.util.TimingUtil;
import org.apache.nutch.util.ToolUtil;
import org.apache.gora.mapreduce.GoraMapper;
@@ -208,7 +210,10 @@ public class FetcherJob extends NutchToo
*/
public int fetch(String batchId, int threads, boolean shouldResume, int
numTasks)
throws Exception {
- LOG.info("FetcherJob: starting");
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ long start = System.currentTimeMillis();
+ LOG.info("FetcherJob: starting at " + sdf.format(start));
if (batchId.equals(Nutch.ALL_BATCH_ID_STR)) {
LOG.info("FetcherJob: fetching all");
@@ -221,7 +226,10 @@ public class FetcherJob extends NutchToo
Nutch.ARG_THREADS, threads,
Nutch.ARG_RESUME, shouldResume,
Nutch.ARG_NUMTASKS, numTasks));
- LOG.info("FetcherJob: done");
+
+ long finish = System.currentTimeMillis();
+ LOG.info("FetcherJob: finished at " + sdf.format(finish) + ", time
elapsed: " + TimingUtil.elapsedTime(start, finish));
+
return 0;
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserJob.java?rev=1524931&r1=1524930&r2=1524931&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserJob.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserJob.java Fri Sep
20 08:03:24 2013
@@ -18,6 +18,7 @@ package org.apache.nutch.parse;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
@@ -44,6 +45,7 @@ import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.NutchTool;
import org.apache.nutch.util.StringUtil;
import org.apache.nutch.util.TableUtil;
+import org.apache.nutch.util.TimingUtil;
import org.apache.nutch.util.ToolUtil;
import org.apache.gora.mapreduce.GoraMapper;
@@ -256,13 +258,19 @@ public class ParserJob extends NutchTool
}
public int parse(String batchId, boolean shouldResume, boolean force) throws
Exception {
- LOG.info("ParserJob: starting");
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ long start = System.currentTimeMillis();
+ LOG.info("ParserJob: starting at " + sdf.format(start));
run(ToolUtil.toArgMap(
Nutch.ARG_BATCH, batchId,
Nutch.ARG_RESUME, shouldResume,
Nutch.ARG_FORCE, force));
LOG.info("ParserJob: success");
+
+ long finish = System.currentTimeMillis();
+ LOG.info("ParserJob: finished at " + sdf.format(finish) + ", time elapsed:
" + TimingUtil.elapsedTime(start, finish));
return 0;
}
