This is an automated email from the ASF dual-hosted git repository.
snagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 77ec28f NUTCH-2768 FetcherThread: unnecessary usage of class casts
new 1cdbb93 Merge pull request #499 from
sebastian-nagel/NUTCH-2768-fetcher-thread-unnecessary-class-cast
77ec28f is described below
commit 77ec28f49fb9b7ed05355dba74f2d3bb928b2581
Author: Sebastian Nagel <[email protected]>
AuthorDate: Fri Feb 21 13:12:59 2020 +0100
NUTCH-2768 FetcherThread: unnecessary usage of class casts
---
.../org/apache/nutch/fetcher/FetcherThread.java | 45 +++++++++++-----------
1 file changed, 22 insertions(+), 23 deletions(-)
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java
b/src/java/org/apache/nutch/fetcher/FetcherThread.java
index e3cf411..5d5a20b 100644
--- a/src/java/org/apache/nutch/fetcher/FetcherThread.java
+++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java
@@ -113,11 +113,11 @@ public class FetcherThread extends Thread {
private AtomicInteger activeThreads;
- private Object fetchQueues;
+ private FetchItemQueues fetchQueues;
private QueueFeeder feeder;
- private Object spinWaiting;
+ private AtomicInteger spinWaiting;
private AtomicLong lastRequestStart;
@@ -253,17 +253,17 @@ public class FetcherThread extends Thread {
return;
}
- fit = ((FetchItemQueues) fetchQueues).getFetchItem();
+ fit = fetchQueues.getFetchItem();
if (fit == null) {
- if (feeder.isAlive() || ((FetchItemQueues)
fetchQueues).getTotalSize() > 0) {
+ if (feeder.isAlive() || fetchQueues.getTotalSize() > 0) {
LOG.debug("{} spin-waiting ...", getName());
// spin-wait.
- ((AtomicInteger) spinWaiting).incrementAndGet();
+ spinWaiting.incrementAndGet();
try {
Thread.sleep(500);
} catch (Exception e) {
}
- ((AtomicInteger) spinWaiting).decrementAndGet();
+ spinWaiting.decrementAndGet();
continue;
} else {
// all done, finish this thread
@@ -297,8 +297,7 @@ public class FetcherThread extends Thread {
if (LOG.isInfoEnabled()) {
LOG.info("{} {} fetching {} (queue crawl delay={}ms)", getName(),
Thread.currentThread().getId(), fit.url,
- ((FetchItemQueues) fetchQueues)
- .getFetchItemQueue(fit.queueID).crawlDelay);
+ fetchQueues.getFetchItemQueue(fit.queueID).crawlDelay);
}
if (LOG.isDebugEnabled()) {
LOG.debug("redirectCount={}", redirectCount);
@@ -313,7 +312,7 @@ public class FetcherThread extends Thread {
}
if (!rules.isAllowed(fit.url.toString())) {
// unblock
- ((FetchItemQueues) fetchQueues).finishFetchItem(fit, true);
+ fetchQueues.finishFetchItem(fit, true);
LOG.info("Denied by robots.txt: {}", fit.url);
output(fit.url, fit.datum, null,
ProtocolStatus.STATUS_ROBOTS_DENIED,
@@ -324,7 +323,7 @@ public class FetcherThread extends Thread {
if (rules.getCrawlDelay() > 0) {
if (rules.getCrawlDelay() > maxCrawlDelay && maxCrawlDelay >= 0)
{
// unblock
- ((FetchItemQueues) fetchQueues).finishFetchItem(fit, true);
+ fetchQueues.finishFetchItem(fit, true);
LOG.info("Crawl-Delay for {} too long ({}), skipping", fit.url,
rules.getCrawlDelay());
output(fit.url, fit.datum, null,
@@ -334,8 +333,7 @@ public class FetcherThread extends Thread {
"robots_denied_maxcrawldelay").increment(1);
continue;
} else {
- FetchItemQueue fiq = ((FetchItemQueues) fetchQueues)
- .getFetchItemQueue(fit.queueID);
+ FetchItemQueue fiq =
fetchQueues.getFetchItemQueue(fit.queueID);
fiq.crawlDelay = rules.getCrawlDelay();
if (LOG.isDebugEnabled()) {
LOG.debug("Crawl delay for queue: " + fit.queueID
@@ -350,7 +348,7 @@ public class FetcherThread extends Thread {
Content content = output.getContent();
ParseStatus pstatus = null;
// unblock queue
- ((FetchItemQueues) fetchQueues).finishFetchItem(fit);
+ fetchQueues.finishFetchItem(fit);
// used for FetchNode
if (fetchNode != null) {
@@ -371,7 +369,7 @@ public class FetcherThread extends Thread {
case ProtocolStatus.WOULDBLOCK:
// retry ?
- ((FetchItemQueues) fetchQueues).addFetchItem(fit);
+ fetchQueues.addFetchItem(fit);
break;
case ProtocolStatus.SUCCESS: // got a page
@@ -416,8 +414,8 @@ public class FetcherThread extends Thread {
case ProtocolStatus.EXCEPTION:
logError(fit.url, status.getMessage());
- int killedURLs = ((FetchItemQueues)
fetchQueues).checkExceptionThreshold(fit
- .getQueueID());
+ int killedURLs = fetchQueues
+ .checkExceptionThreshold(fit.getQueueID());
if (killedURLs != 0)
context.getCounter("FetcherStatus",
"AboveExceptionThresholdInQueue").increment(killedURLs);
@@ -451,7 +449,7 @@ public class FetcherThread extends Thread {
}
if (redirecting && redirectCount > maxRedirect) {
- ((FetchItemQueues) fetchQueues).finishFetchItem(fit);
+ fetchQueues.finishFetchItem(fit);
if (LOG.isInfoEnabled()) {
LOG.info("{} {} - redirect count exceeded {} ({})", getName(),
Thread.currentThread().getId(), fit.url,
@@ -471,7 +469,7 @@ public class FetcherThread extends Thread {
} catch (Throwable t) { // unexpected exception
// unblock
- ((FetchItemQueues) fetchQueues).finishFetchItem(fit);
+ fetchQueues.finishFetchItem(fit);
String message;
if (LOG.isDebugEnabled()) {
message = StringUtils.stringifyException(t);
@@ -491,8 +489,9 @@ public class FetcherThread extends Thread {
LOG.error("fetcher caught:", e);
}
} finally {
- if (fit != null)
- ((FetchItemQueues) fetchQueues).finishFetchItem(fit);
+ if (fit != null) {
+ fetchQueues.finishFetchItem(fit);
+ }
activeThreads.decrementAndGet(); // count threads
LOG.info("{} {} -finishing thread {}, activeThreads={}", getName(),
Thread.currentThread().getId(), getName(), activeThreads);
@@ -589,7 +588,7 @@ public class FetcherThread extends Thread {
CrawlDatum newDatum = createRedirDatum(redirUrl, fit,
CrawlDatum.STATUS_DB_UNFETCHED);
fit = FetchItem.create(redirUrl, newDatum, queueMode);
if (fit != null) {
- FetchItemQueue fiq = ((FetchItemQueues)
fetchQueues).getFetchItemQueue(fit.queueID);
+ FetchItemQueue fiq = fetchQueues.getFetchItemQueue(fit.queueID);
fiq.addInProgressFetchItem(fit);
} else {
// stop redirecting
@@ -768,7 +767,7 @@ public class FetcherThread extends Thread {
// Only process depth N outlinks
if (maxOutlinkDepth > 0 && outlinkDepth < maxOutlinkDepth) {
FetchItem ft = FetchItem.create(url, null, queueMode);
- FetchItemQueue queue = ((FetchItemQueues)
fetchQueues).getFetchItemQueue(ft.queueID);
+ FetchItemQueue queue = fetchQueues.getFetchItemQueue(ft.queueID);
queue.alreadyFetched.add(url.toString().hashCode());
context.getCounter("FetcherOutlinks",
"outlinks_detected").increment(
@@ -806,7 +805,7 @@ public class FetcherThread extends Thread {
context.getCounter("FetcherOutlinks",
"outlinks_following").increment(1);
- ((FetchItemQueues) fetchQueues).addFetchItem(fit);
+ fetchQueues.addFetchItem(fit);
outlinkCounter++;
}
