Author: kwright
Date: Fri Apr 26 19:36:03 2013
New Revision: 1476354
URL: http://svn.apache.org/r1476354
Log:
Another fix for throttling (CONNECTORS-679). Use one synchronizer instead of
two. This may make some things slightly slower, but I believe our current use
allowed unsynchronized variables to be modified.
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1476354&r1=1476353&r2=1476354&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
(original)
+++
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
Fri Apr 26 19:36:03 2013
@@ -978,9 +978,6 @@ public class ThrottledFetcher
/** Total actual bytes read in this series; this includes fetches in
progress */
protected long totalBytesRead = -1L;
- /** This object is used to gate access while the first chunk is being read
*/
- protected Integer firstChunkLock = new Integer(0);
-
/** Outstanding connection counter */
protected volatile int outstandingConnections = 0;
@@ -1102,19 +1099,16 @@ public class ThrottledFetcher
long currentTime = System.currentTimeMillis();
- synchronized (firstChunkLock)
+ synchronized (this)
{
while (estimateInProgress)
- firstChunkLock.wait();
+ wait();
if (estimateValid == false)
{
seriesStartTime = currentTime;
estimateInProgress = true;
// Add these bytes to the estimated total
- synchronized (this)
- {
- totalBytesRead += (long)byteCount;
- }
+ totalBytesRead += (long)byteCount;
// Exit early; this thread isn't going to do any waiting
//if (Logging.connectors.isTraceEnabled())
// Logging.connectors.trace("RSS: Read begin noted; gathering
stats for '"+serverName+"'");
@@ -1161,11 +1155,21 @@ public class ThrottledFetcher
{
if (!finished)
{
- if (estimateInProgress)
- {
- estimateInProgress = false;
- firstChunkLock.notifyAll();
- }
+ abortRead();
+ }
+ }
+ }
+
+ /** Abort a read in progress.
+ */
+ public void abortRead()
+ {
+ synchronized (this)
+ {
+ if (estimateInProgress)
+ {
+ estimateInProgress = false;
+ notifyAll();
}
}
}
@@ -1183,11 +1187,6 @@ public class ThrottledFetcher
synchronized (this)
{
totalBytesRead = totalBytesRead + (long)actualCount -
(long)originalCount;
- }
-
- // Only one thread should get here if it's the first chunk, but we
synchronize to be sure
- synchronized (firstChunkLock)
- {
if (estimateInProgress)
{
if (actualCount == 0)
@@ -1197,7 +1196,7 @@ public class ThrottledFetcher
rateEstimate = ((double)(currentTime -
seriesStartTime))/(double)actualCount;
estimateValid = true;
estimateInProgress = false;
- firstChunkLock.notifyAll();
+ notifyAll();
}
}
Modified:
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1476354&r1=1476353&r2=1476354&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
(original)
+++
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Fri Apr 26 19:36:03 2013
@@ -769,9 +769,6 @@ public class ThrottledFetcher
/** Total actual bytes read in this series; this includes fetches in
progress */
protected long totalBytesRead = -1L;
- /** This object is used to gate access while the first chunk is being read
*/
- protected Integer firstChunkLock = new Integer(0);
-
/** Constructor. */
public ThrottleBin(String binName)
{
@@ -814,19 +811,16 @@ public class ThrottledFetcher
{
long currentTime = System.currentTimeMillis();
- synchronized (firstChunkLock)
+ synchronized (this)
{
while (estimateInProgress)
- firstChunkLock.wait();
+ wait();
if (estimateValid == false)
{
seriesStartTime = currentTime;
estimateInProgress = true;
// Add these bytes to the estimated total
- synchronized (this)
- {
- totalBytesRead += (long)byteCount;
- }
+ totalBytesRead += (long)byteCount;
// Exit early; this thread isn't going to do any waiting
return;
}
@@ -867,15 +861,25 @@ public class ThrottledFetcher
{
if (!finished)
{
- if (estimateInProgress)
- {
- estimateInProgress = false;
- firstChunkLock.notifyAll();
- }
+ abortRead();
}
}
}
+ /** Abort a read in progress.
+ */
+ public void abortRead()
+ {
+ synchronized (this)
+ {
+ if (estimateInProgress)
+ {
+ estimateInProgress = false;
+ notifyAll();
+ }
+ }
+ }
+
/** Note the end of an individual read from the server. Call this just
after an individual read completes.
* Pass the actual number of bytes read to the method.
*/
@@ -886,11 +890,6 @@ public class ThrottledFetcher
synchronized (this)
{
totalBytesRead = totalBytesRead + (long)actualCount -
(long)originalCount;
- }
-
- // Only one thread should get here if it's the first chunk, but we
synchronize to be sure
- synchronized (firstChunkLock)
- {
if (estimateInProgress)
{
if (actualCount == 0)
@@ -900,7 +899,7 @@ public class ThrottledFetcher
rateEstimate = ((double)(currentTime -
seriesStartTime))/(double)actualCount;
estimateValid = true;
estimateInProgress = false;
- firstChunkLock.notifyAll();
+ notifyAll();
}
}
}
@@ -1179,11 +1178,24 @@ public class ThrottledFetcher
throws InterruptedException
{
// Consult with throttle bins
- int i = 0;
- while (i < throttleBinArray.length)
+ int lastOneDone = 0;
+ try
{
- throttleBinArray[i].beginRead(len,minMillisecondsPerByte[i]);
- i++;
+ for (int i = 0; i < throttleBinArray.length; i++)
+ {
+ throttleBinArray[i].beginRead(len,minMillisecondsPerByte[i]);
+ lastOneDone = i + 1;
+ }
+ }
+ finally
+ {
+ if (lastOneDone != throttleBinArray.length)
+ {
+ for (int i = 0; i < lastOneDone; i++)
+ {
+ throttleBinArray[i].abortRead();
+ }
+ }
}
}
@@ -1191,11 +1203,9 @@ public class ThrottledFetcher
public void endRead(int origLen, int actualAmt)
{
// Consult with throttle bins
- int i = 0;
- while (i < throttleBinArray.length)
+ for (int i = 0; i < throttleBinArray.length; i++)
{
throttleBinArray[i].endRead(origLen,actualAmt);
- i++;
}
}
