Author: kwright
Date: Mon Jan 21 23:18:32 2013
New Revision: 1436686
URL: http://svn.apache.org/viewvc?rev=1436686&view=rev
Log:
Fix for CONNECTORS-618.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1436686&r1=1436685&r2=1436686&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Mon Jan 21 23:18:32 2013
@@ -8,6 +8,11 @@ CONNECTORS-617: Simple report NPE under
======================= Release 1.1 =====================
+CONNECTORS-618: MySQL orders indexes so that NULL values are first.
+This is a problem for the stuffer query, which then must go through
+millions of rows before it finds the one it is looking for.
+(Shigeki Kobayashi, Karl Wright)
+
CONNECTORS-616: Work around Solr 4.0 or Jetty bug where connections
are dropped randomly under multithreaded load. Broken pipe exceptions
are now retried after a minute, for up to three times, before the Solr
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1436686&r1=1436685&r2=1436686&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Mon Jan 21 23:18:32 2013
@@ -2132,7 +2132,7 @@ public class JobManager implements IJobM
{
IResultRow row = set.getRow(0);
Double docPriority = (Double)row.getValue(jobQueue.docPriorityField);
- if (docPriority != null)
+ if (docPriority != null && docPriority.doubleValue() <
jobQueue.noDocPriorityValue)
scanRecord.addBins(docPriority);
}
return rval;
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1436686&r1=1436685&r2=1436686&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Mon Jan 21 23:18:32 2013
@@ -110,6 +110,9 @@ public class JobQueue extends org.apache
public static final String prioritySetField = "priorityset";
public static final String checkActionField = "checkaction";
+ public static final double noDocPriorityValue = 1e9;
+ public static final Double nullDocPriority = new Double(noDocPriorityValue +
1.0);
+
protected static Map statusMap;
static
@@ -198,7 +201,10 @@ public class JobQueue extends org.apache
}
else
{
- // Upgrade code goes here, if needed
+ // Upgrade; null docpriority fields bashed to 'infinity', so they
don't slow down MySQL
+ Map map = new HashMap();
+ map.put(docPriorityField,nullDocPriority);
+ performUpdate(map,"WHERE "+docPriorityField+" IS NULL",null,null);
}
// Secondary table installation
@@ -688,7 +694,7 @@ public class JobQueue extends org.apache
{
HashMap map = new HashMap();
map.put(prioritySetField,null);
- map.put(docPriorityField,null);
+ map.put(docPriorityField,nullDocPriority);
ArrayList list = new ArrayList();
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(jobIDField,jobID)});
@@ -715,7 +721,7 @@ public class JobQueue extends org.apache
actionFieldValue = null;
checkTimeValue = null;
// Remove document priority; we don't want to pollute the queue. See
CONNECTORS-290.
- map.put(docPriorityField,null);
+ map.put(docPriorityField,nullDocPriority);
map.put(prioritySetField,null);
break;
case STATUS_ACTIVENEEDRESCAN:
