Author: kwright
Date: Wed Jul 16 22:26:29 2014
New Revision: 1611211
URL: http://svn.apache.org/r1611211
Log:
Minor refactoring improvements
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1611211&r1=1611210&r2=1611211&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Wed Jul 16 22:26:29 2014
@@ -3071,16 +3071,14 @@ public class JobManager implements IJobM
// order the "select for update" operations appropriately.
//
- HashMap indexMap = new HashMap();
+ Map<String,Integer> indexMap = new HashMap<String,Integer>();
String[] docIDHashes = new String[documentDescriptions.length];
- int i = 0;
- while (i < documentDescriptions.length)
+ for (int i = 0; i < documentDescriptions.length; i++)
{
String documentIDHash =
documentDescriptions[i].getDocumentIdentifierHash() + ":" +
documentDescriptions[i].getJobID();
docIDHashes[i] = documentIDHash;
indexMap.put(documentIDHash,new Integer(i));
- i++;
}
java.util.Arrays.sort(docIDHashes);
@@ -3095,13 +3093,10 @@ public class JobManager implements IJobM
try
{
// Do one row at a time, to avoid deadlocking things
- i = 0;
- while (i < docIDHashes.length)
+ for (String docIDHash : docIDHashes)
{
- String docIDHash = docIDHashes[i];
-
// Get the DocumentDescription object
- DocumentDescription dd =
documentDescriptions[((Integer)indexMap.get(docIDHash)).intValue()];
+ DocumentDescription dd =
documentDescriptions[indexMap.get(docIDHash).intValue()];
// Query for the status
ArrayList list = new ArrayList();
@@ -3119,10 +3114,10 @@ public class JobManager implements IJobM
// Update the jobqueue table
jobQueue.updateCompletedRecord(dd.getID(),status);
}
- i++;
}
TrackerClass.notePrecommit();
database.performCommit();
+ System.out.println("Committed updated status");
TrackerClass.noteCommit();
break;
}
@@ -3146,6 +3141,12 @@ public class JobManager implements IJobM
TrackerClass.noteRollback();
throw e;
}
+ catch (RuntimeException e)
+ {
+ database.signalRollback();
+ TrackerClass.noteRollback();
+ throw e;
+ }
finally
{
database.endTransaction();
@@ -5037,7 +5038,7 @@ public class JobManager implements IJobM
" docs and hopcounts for job "+jobID.toString()+" parent identifier
hash "+parentIdentifierHash);
// Go through document id's one at a time, in order - mainly to
prevent deadlock as much as possible. Search for any existing row in jobqueue
first (for update)
- HashMap existingRows = new HashMap();
+ Map<String,JobqueueRecord> existingRows = new
HashMap<String,JobqueueRecord>();
for (int z = 0; z < reorderedDocIDHashes.length; z++)
{
@@ -5091,7 +5092,7 @@ public class JobManager implements IJobM
for (int z = 0; z < reorderedDocIDHashes.length; z++)
{
String docIDHash = reorderedDocIDHashes[z];
- JobqueueRecord jr = (JobqueueRecord)existingRows.get(docIDHash);
+ JobqueueRecord jr = existingRows.get(docIDHash);
if (jr != null)
{
// It was an existing row; do the update logic
@@ -5112,6 +5113,7 @@ public class JobManager implements IJobM
TrackerClass.notePrecommit();
database.performCommit();
+ System.out.println("Committed carrydown updates");
TrackerClass.noteCommit();
if (Logging.perf.isDebugEnabled())
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1611211&r1=1611210&r2=1611211&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Wed Jul 16 22:26:29 2014
@@ -418,6 +418,7 @@ public class WorkerThread extends Thread
}
else if (serviceInterruption != null)
{
+
// Service interruption has precedence over
unchanged, because we might have been interrupted while scanning the document
// for references
DocumentDescription dd = qd.getDocumentDescription();
@@ -449,6 +450,7 @@ public class WorkerThread extends Thread
}
else if
(activity.wasDocumentUnchanged(qd.getDocumentDescription().getDocumentIdentifier()))
{
+
finishList.add(qd);
ingesterCheckList.add(qd.getDocumentDescription().getDocumentIdentifierHash());
}
@@ -494,7 +496,7 @@ public class WorkerThread extends Thread
}
DocumentDescription[] requeueCandidates =
jobManager.finishDocuments(job.getID(),legalLinkTypes,documentIDHashes,job.getHopcountMode());
ManifoldCF.requeueDocumentsDueToCarrydown(jobManager,requeueCandidates,connector,connection,rt,currentTime);
-
+
// In both job types, we have to go through the
finishList to figure out what to do with the documents.
// In the case of a document that was aborted, we must
requeue it for immediate reprocessing in BOTH job types.
switch (job.getType())
@@ -584,9 +586,8 @@ public class WorkerThread extends Thread
// Separate the ones we actually finished from the
ones we need to requeue because they were aborted
List<DocumentDescription> completedList = new
ArrayList<DocumentDescription>();
List<DocumentDescription> abortedList = new
ArrayList<DocumentDescription>();
- for (int i = 0; i < finishList.size(); i++)
+ for (QueuedDocument qd : finishList)
{
- QueuedDocument qd = finishList.get(i);
DocumentDescription dd =
qd.getDocumentDescription();
if
(activity.wasDocumentAborted(dd.getDocumentIdentifier()))
{
@@ -622,7 +623,7 @@ public class WorkerThread extends Thread
DocumentDescription[] docDescriptions = new
DocumentDescription[completedList.size()];
for (int i = 0; i < docDescriptions.length; i++)
{
- docDescriptions[i] =
(DocumentDescription)completedList.get(i);
+ docDescriptions[i] = completedList.get(i);
}
jobManager.markDocumentCompletedMultiple(docDescriptions);
@@ -672,9 +673,8 @@ public class WorkerThread extends Thread
}
// Handle rescanning
- for (int i = 0; i < rescanList.size(); i++)
+ for (QueuedDocument qd : rescanList)
{
- QueuedDocument qd = rescanList.get(i);
jobManager.resetDocument(qd.getDocumentDescription(),0L,IJobManager.ACTION_RESCAN,-1L,-1);
qd.setProcessed();
}
@@ -711,10 +711,9 @@ public class WorkerThread extends Thread
{
// Go through qds and requeue any that aren't closed out in one
way or another. This allows the job
// to be aborted; no dangling entries are left around.
- int i = 0;
- while (i < qds.getCount())
+ for (int i = 0; i < qds.getCount(); i++)
{
- QueuedDocument qd = qds.getDocument(i++);
+ QueuedDocument qd = qds.getDocument(i);
if (!qd.wasProcessed())
{
jobManager.resetDocument(qd.getDocumentDescription(),0L,IJobManager.ACTION_RESCAN,-1L,-1);
