Author: kwright
Date: Fri Aug 14 11:19:59 2015
New Revision: 1695858
URL: http://svn.apache.org/r1695858
Log:
Improve scheduling debug logging
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/QueueTracker.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocumentSet.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Aug 14 11:19:59 2015
@@ -125,4 +125,4 @@
/manifoldcf/branches/CONNECTORS-981:1605049-1605773
/manifoldcf/branches/CONNECTORS-989:1611600-1612101
/manifoldcf/branches/CONNECTORS-990:1610284-1610707
-/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605,1649628,1649794,1650351,1650722,1650741-1650742,1650745,1650747,1650911,1650954,1651332,1651539,1651907,1651921,1652071,1652974,1653175,1653899,1654651,1655205,1655261,1655264,1655377,1655411,1655618,1655914,1657346,1657443,1658004,1658036,1658121,1658155,1658188,1658463,1658476,
1660258,1660276,1661454,1665848,1666160,1666781,1666820,1668312,1669100,1669238,1669487,1669523,1669586,1669660,1670614,1670625,1670715,1671496,1672169,1672301,1672616,1672737,1673559,1673573,1673579,1673722,1675781,1675898,1676094,1676882,1676910,1678300,1678329,1678471,1678551,1679730,1679826,1681390,1681735,1682232,1682252,1682410,1682602,1682622,1682719,1683208,1683506,1683529,1683768,1684015,1684017,1684153,1684156,1684712,1684866,1685547,1687097,1687303,1687768,1688070,1688348,1688432,1689608,1690665,1692679,1693653,1695675,1695696,1695808,1695839
+/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605,1649628,1649794,1650351,1650722,1650741-1650742,1650745,1650747,1650911,1650954,1651332,1651539,1651907,1651921,1652071,1652974,1653175,1653899,1654651,1655205,1655261,1655264,1655377,1655411,1655618,1655914,1657346,1657443,1658004,1658036,1658121,1658155,1658188,1658463,1658476,
1660258,1660276,1661454,1665848,1666160,1666781,1666820,1668312,1669100,1669238,1669487,1669523,1669586,1669660,1670614,1670625,1670715,1671496,1672169,1672301,1672616,1672737,1673559,1673573,1673579,1673722,1675781,1675898,1676094,1676882,1676910,1678300,1678329,1678471,1678551,1679730,1679826,1681390,1681735,1682232,1682252,1682410,1682602,1682622,1682719,1683208,1683506,1683529,1683768,1684015,1684017,1684153,1684156,1684712,1684866,1685547,1687097,1687303,1687768,1688070,1688348,1688432,1689608,1690665,1692679,1693653,1695675,1695696,1695808,1695839,1695857
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Aug 14 11:19:59 2015
@@ -113,4 +113,4 @@
/manifoldcf/branches/CONNECTORS-989/framework:1611600-1612101
/manifoldcf/branches/CONNECTORS-990/framework:1610284-1610707
/manifoldcf/trunk:1629122
-/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605,1650351,1650911,1651332,1651539,1651921,1655377,1655411,1657346,1658004,1658036,1660258,1660276,1669487,1670614,1672616,1672737,1676094,1681390,1681735,1682232,1682252,1682410,1682719,1683208,1683506,1683768,1684712,1685547,1687097,1687303,1688432,1690665,1695675,1695696,1695808,1695839
+/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605,1650351,1650911,1651332,1651539,1651921,1655377,1655411,1657346,1658004,1658036,1660258,1660276,1669487,1670614,1672616,1672737,1676094,1681390,1681735,1682232,1682252,1682410,1682719,1683208,1683506,1683768,1684712,1685547,1687097,1687303,1688432,1690665,1695675,1695696,1695808,1695839,1695857
Modified:
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/QueueTracker.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/QueueTracker.java?rev=1695858&r1=1695857&r2=1695858&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/QueueTracker.java
(original)
+++
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/QueueTracker.java
Fri Aug 14 11:19:59 2015
@@ -18,7 +18,6 @@
*/
package org.apache.manifoldcf.crawler.interfaces;
-import org.apache.manifoldcf.crawler.system.Logging;
import org.apache.manifoldcf.core.interfaces.*;
import java.io.*;
import java.util.*;
@@ -80,16 +79,6 @@ public class QueueTracker
*/
public void addRecord(String[] binNames)
{
- if (Logging.scheduling.isDebugEnabled())
- {
- StringBuilder sb = new StringBuilder();
- int j = 0;
- while (j < binNames.length)
- {
- sb.append(binNames[j++]).append(" ");
- }
- Logging.scheduling.debug("Putting document with bins ["+sb.toString()+"]
onto active queue");
- }
int i = 0;
while (i < binNames.length)
{
@@ -114,9 +103,6 @@ public class QueueTracker
*/
public void noteConnectionPerformance(int docCount, String connectionName,
long elapsedTime)
{
- if (Logging.scheduling.isDebugEnabled())
- Logging.scheduling.debug("Worker thread for connection
"+connectionName+" took "+new Long(elapsedTime).toString()+"ms to handle
"+Integer.toString(docCount)+" documents");
-
performanceStatistics.noteDocumentsCompleted(connectionName,docCount,elapsedTime);
}
@@ -131,17 +117,6 @@ public class QueueTracker
*/
public void beginProcessing(String[] binNames)
{
- if (Logging.scheduling.isDebugEnabled())
- {
- StringBuilder sb = new StringBuilder();
- int j = 0;
- while (j < binNames.length)
- {
- sb.append(binNames[j++]).append(" ");
- }
- Logging.scheduling.debug("Handing document with bins ["+sb.toString()+"]
to worker thread");
- }
-
// Effectively, we are moving the document from one status to another, so
we adjust the bin counts of the source and
// the target both.
@@ -181,17 +156,6 @@ public class QueueTracker
*/
public void endProcessing(String[] binNames)
{
- if (Logging.scheduling.isDebugEnabled())
- {
- StringBuilder sb = new StringBuilder();
- int j = 0;
- while (j < binNames.length)
- {
- sb.append(binNames[j++]).append(" ");
- }
- Logging.scheduling.debug("Worker thread done document with bins
["+sb.toString()+"]");
- }
-
// Remove the document from the active queue, by decrementing the
corresponding active bin counts.
int i = 0;
@@ -242,20 +206,7 @@ public class QueueTracker
}
// Take the ith root of the bin rating, and leave it in log form
- double rval = ratingLog/(double)i;
-
- if (false && Logging.scheduling.isDebugEnabled())
- {
- StringBuilder sb = new StringBuilder();
- int j = 0;
- while (j < binNames.length)
- {
- sb.append(binNames[j++]).append(" ");
- }
- Logging.scheduling.debug("Document with bins ["+sb.toString()+"] given
assignment rating "+new Double(rval).toString());
- }
-
- return rval;
+ return ratingLog/(double)i;
}
Modified:
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1695858&r1=1695857&r2=1695858&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Fri Aug 14 11:19:59 2015
@@ -2771,7 +2771,7 @@ public class JobManager implements IJobM
ThrottleLimit vList = new ThrottleLimit(n);
IResultSet jobconnections = jobs.getActiveJobConnections();
- HashMap connectionSet = new HashMap();
+ Set<String> connectionSet = new HashSet<String>();
int i = 0;
while (i < jobconnections.getRowCount())
{
@@ -2779,29 +2779,26 @@ public class JobManager implements IJobM
Long jobid = (Long)row.getValue("jobid");
String connectionName = (String)row.getValue("connectionname");
vList.addJob(jobid,connectionName);
- connectionSet.put(connectionName,connectionName);
+ connectionSet.add(connectionName);
}
// Find the active connection names. We'll load these, and then get
throttling info
// from each one.
String[] activeConnectionNames = new String[connectionSet.size()];
- Iterator iter = connectionSet.keySet().iterator();
i = 0;
- while (iter.hasNext())
+ for (String connectionName : connectionSet)
{
- activeConnectionNames[i++] = (String)iter.next();
+ activeConnectionNames[i++] = connectionName;
}
IRepositoryConnection[] connections =
connectionMgr.loadMultiple(activeConnectionNames);
// Accumulate a sum of the max_connection_count * avg_connection_rate
values, so we can calculate the appropriate adjustment
// factor and set the connection limits.
- HashMap rawFetchCounts = new HashMap();
+ Map<String,Double> rawFetchCounts = new HashMap<String,Double>();
double rawFetchCountTotal = 0.0;
- i = 0;
- while (i < connections.length)
+ for (IRepositoryConnection connection : connections)
{
- IRepositoryConnection connection = connections[i++];
String connectionName = connection.getName();
int maxConnections = connection.getMaxConnections();
double avgFetchRate =
statistics.calculateConnectionFetchRate(connectionName);
@@ -2818,18 +2815,14 @@ public class JobManager implements IJobM
// and also randomly select an extra fetch based on the fractional
probability. (This latter is
// necessary for the case where the maximum fetch rate is specified to be
pretty low.)
//
- i = 0;
- while (i < connections.length)
+ for (IRepositoryConnection connection : connections)
{
- IRepositoryConnection connection = connections[i++];
String connectionName = connection.getName();
// Check if throttled...
String[] throttles = connection.getThrottles();
- int k = 0;
- while (k < throttles.length)
+ for (String throttle : throttles)
{
// The key is the regexp value itself
- String throttle = throttles[k++];
float throttleValue = connection.getThrottleValue(throttle);
// For the given connection, set the fetch limit per bin. This is
calculated using the time interval
// and the desired fetch rate. The fractional remainder is used to
conditionally provide an "extra fetch"
@@ -2848,7 +2841,7 @@ public class JobManager implements IJobM
vList.addLimit(connectionName,throttle,fetches);
}
// For the overall connection, we also have a limit which is based on
the number of connections there are actually available.
- Double weightedRawFetchCount =
(Double)rawFetchCounts.get(connectionName);
+ Double weightedRawFetchCount = rawFetchCounts.get(connectionName);
double adjustedFetchCount = weightedRawFetchCount.doubleValue() *
fetchCountAdjustmentFactor;
// Note well: Queuing starvation that results from there being very few
available documents for high-priority connections is dealt with here by simply
allowing
@@ -2903,7 +2896,7 @@ public class JobManager implements IJobM
// have any chance of getting returned twice.
// Accumulate the answers here
- ArrayList answers = new ArrayList();
+ List<DocumentDescription> answers = new ArrayList<DocumentDescription>();
// The current time value
Long currentTimeValue = new Long(currentTime);
@@ -2934,10 +2927,9 @@ public class JobManager implements IJobM
// Convert the saved answers to an array
DocumentDescription[] rval = new DocumentDescription[answers.size()];
i = 0;
- while (i < rval.length)
+ for (DocumentDescription answer : answers)
{
- rval[i] = (DocumentDescription)answers.get(i);
- i++;
+ rval[i++] = answer;
}
// After we're done pulling stuff from the queue, find the eligible row
with the best priority on the queue, and save the bins for assessment.
@@ -3030,7 +3022,7 @@ public class JobManager implements IJobM
}
/** Fetch and process documents matching the passed-in criteria */
- protected void fetchAndProcessDocuments(ArrayList answers, Long
currentTimeValue, Long currentPriorityValue,
+ protected void fetchAndProcessDocuments(List<DocumentDescription> answers,
Long currentTimeValue, Long currentPriorityValue,
ThrottleLimit vList, IRepositoryConnection[] connections, String processID)
throws ManifoldCFException
{
@@ -3158,8 +3150,7 @@ public class JobManager implements IJobM
Map<String,DocumentDescription> storageMap = new
HashMap<String,DocumentDescription>();
Map<String,Integer> statusMap = new HashMap<String,Integer>();
- int i = 0;
- while (i < set.getRowCount())
+ for (int i = 0; i < set.getRowCount(); i++)
{
IResultRow row = set.getRow(i);
Long id = (Long)row.getValue(jobQueue.idField);
@@ -3189,26 +3180,21 @@ public class JobManager implements IJobM
Double docPriority =
(Double)row.getValue(jobQueue.docPriorityField);
Logging.scheduling.debug("Stuffing document '"+docID+"' that
has priority "+docPriority.toString()+" onto active list");
}
- i++;
}
// No duplicates are possible here
java.util.Arrays.sort(docIDHashes);
- i = 0;
- while (i < docIDHashes.length)
+ for (String docIDHash : docIDHashes)
{
- String docIDHash = docIDHashes[i];
- DocumentDescription dd =
(DocumentDescription)storageMap.get(docIDHash);
+ DocumentDescription dd = storageMap.get(docIDHash);
Long id = dd.getID();
- int status = ((Integer)statusMap.get(docIDHash)).intValue();
+ int status = statusMap.get(docIDHash).intValue();
// Set status to "ACTIVE".
jobQueue.updateActiveRecord(id,status,processID);
answers.add(dd);
-
- i++;
}
TrackerClass.notePrecommit();
database.performCommit();
Modified:
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocumentSet.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocumentSet.java?rev=1695858&r1=1695857&r2=1695858&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocumentSet.java
(original)
+++
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocumentSet.java
Fri Aug 14 11:19:59 2015
@@ -73,6 +73,16 @@ public class QueuedDocumentSet
while (l < documents.length)
{
QueuedDocument d = documents[l++];
+ if (Logging.scheduling.isDebugEnabled())
+ {
+ StringBuilder sb = new StringBuilder();
+ for (String binName : d.getBinNames())
+ {
+ sb.append(binName).append(" ");
+ }
+ Logging.scheduling.debug("Handing document
'"+d.getDocumentDescription().getDocumentIdentifier()+"' with bins
["+sb.toString()+"] to worker thread");
+ }
+
queueTracker.beginProcessing(d.getBinNames());
}
}
@@ -84,6 +94,16 @@ public class QueuedDocumentSet
while (l < documents.length)
{
QueuedDocument d = documents[l++];
+ if (Logging.scheduling.isDebugEnabled())
+ {
+ StringBuilder sb = new StringBuilder();
+ for (String binName : d.getBinNames())
+ {
+ sb.append(binName).append(" ");
+ }
+ Logging.scheduling.debug("Worker thread done document
'"+d.getDocumentDescription().getDocumentIdentifier()+"' with bins
["+sb.toString()+"]");
+ }
+
queueTracker.endProcessing(d.getBinNames());
}
@@ -100,8 +120,19 @@ public class QueuedDocumentSet
int i = 0;
while (i < documents.length)
{
- String[] binNames = documents[i++].getBinNames();
- ratingAccumulator +=
overlapCalculator.calculateAssignmentRating(binNames,connection);
+ QueuedDocument d = documents[i++];
+ double rating =
overlapCalculator.calculateAssignmentRating(d.getBinNames(),connection);
+ if (false && Logging.scheduling.isDebugEnabled())
+ {
+ StringBuilder sb = new StringBuilder();
+ for (String binName : d.getBinNames())
+ {
+ sb.append(binName).append(" ");
+ }
+ Logging.scheduling.debug("Document
'"+d.getDocumentDescription().getDocumentIdentifier()+"' with bins
["+sb.toString()+"] given assignment rating "+new Double(rating).toString());
+ }
+
+ ratingAccumulator += rating;
}
return ratingAccumulator / (double)documents.length;
Modified:
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1695858&r1=1695857&r2=1695858&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
(original)
+++
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Fri Aug 14 11:19:59 2015
@@ -376,6 +376,16 @@ public class StufferThread extends Threa
set.add(qd);
// Note the queuing activity
+ if (Logging.scheduling.isDebugEnabled())
+ {
+ StringBuilder sb = new StringBuilder();
+ int j = 0;
+ while (j < binNames.length)
+ {
+ sb.append(binNames[j++]).append(" ");
+ }
+ Logging.scheduling.debug("Putting document
'"+descs[i].getDocumentIdentifier()+"' with bins ["+sb.toString()+"] onto
active queue");
+ }
queueTracker.addRecord(binNames);
if (set.size() >= maxDocuments)
Modified:
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1695858&r1=1695857&r2=1695858&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Fri Aug 14 11:19:59 2015
@@ -735,7 +735,10 @@ public class WorkerThread extends Thread
// Successful processing of the set
// We count 'get version' time in the average, so even if
we decide not to process a doc
// it still counts.
-
queueTracker.noteConnectionPerformance(qds.getCount(),connectionName,System.currentTimeMillis()
- processingStartTime);
+ long elapsedTime = System.currentTimeMillis() -
processingStartTime;
+ if (Logging.scheduling.isDebugEnabled())
+ Logging.scheduling.debug("Worker thread for connection
"+connectionName+" took "+new Long(elapsedTime).toString()+"ms to handle
"+Integer.toString(qds.getCount())+" documents");
+
queueTracker.noteConnectionPerformance(qds.getCount(),connectionName,elapsedTime);
}
