Author: kwright
Date: Sun Aug 10 12:33:03 2014
New Revision: 1617101
URL: http://svn.apache.org/r1617101
Log:
Flesh out some comments
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java?rev=1617101&r1=1617100&r2=1617101&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java
Sun Aug 10 12:33:03 2014
@@ -381,7 +381,6 @@ public abstract class BaseRepositoryConn
else
{
// These documents must go away permanently
- // MHL to collect these and do them as a group
activities.deleteDocument(documentIdentifier);
}
}
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1617101&r1=1617100&r2=1617101&view=diff
==============================================================================
---
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Sun Aug 10 12:33:03 2014
@@ -1578,10 +1578,9 @@ public class WorkerThread extends Thread
String componentIdentifierHash =
computeComponentIDHash(componentIdentifier);
checkMultipleDispositions(documentIdentifier,componentIdentifier,componentIdentifierHash);
+ // First, we need to add into the metadata the stuff from the job
description.
if (data != null)
{
- //Map<String,Set<String>> forcedMetadata = job.getForcedMetadata();
-
// Modify the repository document with forced parameters.
for (String paramName : forcedMetadata.keySet())
{
@@ -1596,7 +1595,14 @@ public class WorkerThread extends Thread
}
}
- // First, we need to add into the metadata the stuff from the job
description.
+ // This method currently signals whether the document is accepted or
rejected permanently.
+ // Permanent rejection should involve leaving the document in the queue
(since it probably
+ // will be rediscovered), and noting its version (so that we don't try
doing anything with it
+ // again). But the documentIngest method will already take care of the
latter, so we really need to
+ // do nothing different based on the results of the documentIngest
invocation.
+ // The only time it would be great to do something different would be if
the document's version
+ // indicates that it should always be refetched. But I have no way to
describe this situation
+ // in the database at the moment.
ingester.documentIngest(
computePipelineSpecification(documentIdentifierHash,componentIdentifierHash,documentIdentifier),
connectionName,documentIdentifierHash,componentIdentifierHash,
