Update of /cvsroot/nutch/nutch/src/java/net/nutch/fetcher
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv23218/src/java/net/nutch/fetcher
Modified Files:
Fetcher.java
Log Message:
Add a method and a class to help with collecting progress status through API
from a running Fetcher.
Index: Fetcher.java
===================================================================
RCS file: /cvsroot/nutch/nutch/src/java/net/nutch/fetcher/Fetcher.java,v
retrieving revision 1.41
retrieving revision 1.42
diff -C2 -d -r1.41 -r1.42
*** Fetcher.java 20 Aug 2004 20:36:10 -0000 1.41
--- Fetcher.java 22 Oct 2004 23:47:57 -0000 1.42
***************
*** 36,39 ****
--- 36,40 ----
private ArrayFile.Writer parseDataWriter;
+ private String name; // name of the segment
private long start; // start time of fetcher run
private long bytes; // total bytes fetched
***************
*** 201,204 ****
--- 202,206 ----
parseDataWriter = new ArrayFile.Writer
(nfs, new File(directory, ParseData.DIR_NAME).toString(), ParseData.class);
+ name = new File(directory).getName();
}
***************
*** 252,256 ****
if (noMoreFetcherThread) {
if (LOG.isLoggable(Level.FINE))
! LOG.fine("number of avtive threads: "+n);
if (pages == pages0 && errors == errors0 && bytes == bytes0)
break;
--- 254,258 ----
if (noMoreFetcherThread) {
if (LOG.isLoggable(Level.FINE))
! LOG.fine("number of active threads: "+n);
if (pages == pages0 && errors == errors0 && bytes == bytes0)
break;
***************
*** 267,283 ****
}
/** Display the status of the fetcher run. */
public synchronized void status() {
! long ms = System.currentTimeMillis() - start;
! LOG.info("status: "
! + pages + " pages, "
! + errors + " errors, "
! + bytes + " bytes, "
! + ms + " ms");
LOG.info("status: "
! + (((float)pages)/(ms/1000.0f))+" pages/s, "
! + (((float)bytes*8/1024)/(ms/1000.0f))+" kb/s, "
! + (((float)bytes)/pages) + " bytes/page");
}
--- 269,325 ----
}
+
+ public static class FetcherStatus {
+ private String name;
+ private long startTime, curTime;
+ private int pageCount, errorCount;
+ private long byteCount;
+
+ /**
+ * FetcherStatus encapsulates a snapshot of the Fetcher progress status.
+ * @param name short name of the segment being processed
+ * @param start the time in millisec. this fetcher was started
+ * @param pages number of pages fetched
+ * @param errors number of fetching errors
+ * @param bytes number of bytes fetched
+ */
+ public FetcherStatus(String name, long start, int pages, int errors, long bytes)
{
+ this.name = name;
+ this.startTime = start;
+ this.curTime = System.currentTimeMillis();
+ this.pageCount = pages;
+ this.errorCount = errors;
+ this.byteCount = bytes;
+ }
+
+ public String getName() {return name;}
+ public long getStartTime() {return startTime;}
+ public long getCurTime() {return curTime;}
+ public long getElapsedTime() {return curTime - startTime;}
+ public int getPageCount() {return pageCount;}
+ public int getErrorCount() {return errorCount;}
+ public long getByteCount() {return byteCount;}
+
+ public String toString() {
+ return "status: segment " + name + ", "
+ + pageCount + " pages, "
+ + errorCount + " errors, "
+ + byteCount + " bytes, "
+ + (curTime - startTime) + " ms";
+ }
+ }
+
+ public synchronized FetcherStatus getStatus() {
+ return new FetcherStatus(name, start, pages, errors, bytes);
+ }
/** Display the status of the fetcher run. */
public synchronized void status() {
! FetcherStatus status = getStatus();
! LOG.info(status.toString());
LOG.info("status: "
! + (((float)status.getPageCount())/(status.getElapsedTime()/1000.0f))+"
pages/s, "
! +
(((float)status.getByteCount()*8/1024)/(status.getElapsedTime()/1000.0f))+" kb/s, "
! + (((float)status.getByteCount())/status.getPageCount()) + "
bytes/page");
}
-------------------------------------------------------
This SF.net email is sponsored by: IT Product Guide on ITManagersJournal
Use IT products in your business? Tell us what you think of them. Give us
Your Opinions, Get Free ThinkGeek Gift Certificates! Click to find out more
http://productguide.itmanagersjournal.com/guidepromo.tmpl
_______________________________________________
Nutch-cvs mailing list
[EMAIL PROTECTED]
https://lists.sourceforge.net/lists/listinfo/nutch-cvs
