Author: kwright
Date: Fri Jul 4 13:34:44 2014
New Revision: 1607853
URL: http://svn.apache.org/r1607853
Log:
Initial work on various APIs
Modified:
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Modified:
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Fri Jul 4 13:34:44 2014
@@ -687,6 +687,40 @@ public class IncrementalIngester extends
sb.append(delim);
}
+ /** Note the start of processing of a set of documents. This method does
whatever is needed to handle the
+ * bookkeeping for the documents' indexing records, and the indexing records
for multiple virtual child documents.
+ * The documents must all have the same identifier class.
+ *@param pipelineSpecificationBasic is the basic pipeline specification for
the set of documents.
+ *@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
+ *@param identifierHashes are the document identifier hashes that are about
to be processed.
+ */
+ @Override
+ public void beginDocumentProcessing(
+ IPipelineSpecificationBasic pipelineSpecificationBasic,
+ String identifierClass, String[] identifierHashes)
+ throws ManifoldCFException
+ {
+ // MHL
+ }
+
+ /** Note the end of processing of a set of documents. This method completes
bookkeeping for the documents'
+ * indexing records, and the indexing records for multiple virtual child
documents. This method may cause
+ * documents to be removed from the specified output connections, should that
be indicated.
+ *@param pipelineSpecificationBasic is the basic pipeline specification for
the set of documents.
+ *@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
+ *@param identifierHashes are the document identifier hashes that are about
to be processed.
+ *@param activities is the object to use to log the details of any removals.
May be null.
+ */
+ @Override
+ public void endDocumentProcessing(
+ IPipelineSpecificationBasic pipelineSpecificationBasic,
+ String identifierClass, String[] identifierHashes,
+ IOutputRemoveActivity activities)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ // MHL
+ }
+
/** Record a document version, but don't ingest it.
* The purpose of this method is to keep track of the frequency at which
ingestion "attempts" take place.
* ServiceInterruption is thrown if this action must be rescheduled.
@@ -707,7 +741,7 @@ public class IncrementalIngester extends
{
documentRecord(
new RuntPipelineSpecificationBasic(outputConnectionName),
- identifierClass, identifierHash,
+ identifierClass, identifierHash, null,
documentVersion,
recordTime, activities);
}
@@ -718,6 +752,7 @@ public class IncrementalIngester extends
*@param pipelineSpecificationBasic is the basic pipeline specification
needed.
*@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
*@param identifierHash is the hashed document identifier.
+ *@param childIdentifierHash is the hashed virtual child document identifier.
Pass null if this is a primary record.
*@param documentVersion is the document version.
*@param recordTime is the time at which the recording took place, in
milliseconds since epoch.
*@param activities is the object used in case a document needs to be removed
from the output index as the result of this operation.
@@ -725,11 +760,12 @@ public class IncrementalIngester extends
@Override
public void documentRecord(
IPipelineSpecificationBasic pipelineSpecificationBasic,
- String identifierClass, String identifierHash,
+ String identifierClass, String identifierHash, String childIdentifierHash,
String documentVersion, long recordTime,
IOutputActivity activities)
throws ManifoldCFException, ServiceInterruption
{
+ // MHL
String docKey = makeKey(identifierClass,identifierHash);
String[] outputConnectionNames =
extractOutputConnectionNames(pipelineSpecificationBasic);
@@ -917,7 +953,7 @@ public class IncrementalIngester extends
return documentIngest(
new
RuntPipelineSpecificationWithVersions(outputConnectionName,outputVersion,
"","","","",""),
- identifierClass, identifierHash,
+ identifierClass, identifierHash, null,
documentVersion,
parameterVersion,
authorityName,
@@ -1026,6 +1062,7 @@ public class IncrementalIngester extends
*@param pipelineSpecificationWithVersions is the pipeline specification with
already-fetched output versioning information.
*@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
*@param identifierHash is the hashed document identifier.
+ *@param childIdentifierHash is the hashed virtual child document identifier.
Pass null if this is a primary record.
*@param documentVersion is the document version.
*@param parameterVersion is the version string for the forced parameters.
*@param authorityName is the name of the authority associated with the
document, if any.
@@ -1039,7 +1076,7 @@ public class IncrementalIngester extends
@Override
public boolean documentIngest(
IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
- String identifierClass, String identifierHash,
+ String identifierClass, String identifierHash, String childIdentifierHash,
String documentVersion,
String parameterVersion,
String authorityName,
@@ -1048,6 +1085,8 @@ public class IncrementalIngester extends
IOutputActivity activities)
throws ManifoldCFException, ServiceInterruption, IOException
{
+ // MHL
+
PipelineConnectionsWithVersions pipelineConnectionsWithVersions = new
PipelineConnectionsWithVersions(pipelineSpecificationWithVersions);
String docKey = makeKey(identifierClass,identifierHash);
@@ -1894,7 +1933,8 @@ public class IncrementalIngester extends
if (authorityName == null)
authorityName = "";
int indexValue = position.intValue();
- rval.put(new
OutputKey(identifierClasses[indexValue],identifierHashes[indexValue],outputConnectionName),
+ // MHL
+ rval.put(new
OutputKey(identifierClasses[indexValue],identifierHashes[indexValue],null,outputConnectionName),
new
DocumentIngestStatus(lastVersion,lastTransformationVersion,lastOutputVersion,paramVersion,authorityName));
}
}
Modified:
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
(original)
+++
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
Fri Jul 4 13:34:44 2014
@@ -200,6 +200,32 @@ public interface IIncrementalIngester
String newParameterVersion,
String newAuthorityNameString);
+ /** Note the start of processing of a set of documents. This method does
whatever is needed to handle the
+ * bookkeeping for the documents' indexing records, and the indexing records
for multiple virtual child documents.
+ * The documents must all have the same identifier class.
+ *@param pipelineSpecificationBasic is the basic pipeline specification for
the set of documents.
+ *@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
+ *@param identifierHashes are the document identifier hashes that are about
to be processed.
+ */
+ public void beginDocumentProcessing(
+ IPipelineSpecificationBasic pipelineSpecificationBasic,
+ String identifierClass, String[] identifierHashes)
+ throws ManifoldCFException;
+
+ /** Note the end of processing of a set of documents. This method completes
bookkeeping for the documents'
+ * indexing records, and the indexing records for multiple virtual child
documents. This method may cause
+ * documents to be removed from the specified output connections, should that
be indicated.
+ *@param pipelineSpecificationBasic is the basic pipeline specification for
the set of documents.
+ *@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
+ *@param identifierHashes are the document identifier hashes that are about
to be processed.
+ *@param activities is the object to use to log the details of any removals.
May be null.
+ */
+ public void endDocumentProcessing(
+ IPipelineSpecificationBasic pipelineSpecificationBasic,
+ String identifierClass, String[] identifierHashes,
+ IOutputRemoveActivity activities)
+ throws ManifoldCFException, ServiceInterruption;
+
/** Record a document version, but don't ingest it.
* The purpose of this method is to keep track of the frequency at which
ingestion "attempts" take place.
* ServiceInterruption is thrown if this action must be rescheduled.
@@ -223,13 +249,14 @@ public interface IIncrementalIngester
*@param pipelineSpecificationBasic is the basic pipeline specification
needed.
*@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
*@param identifierHash is the hashed document identifier.
+ *@param childIdentifierHash is the hashed virtual child document identifier.
Pass null if this is a primary record.
*@param documentVersion is the document version.
*@param recordTime is the time at which the recording took place, in
milliseconds since epoch.
*@param activities is the object used in case a document needs to be removed
from the output index as the result of this operation.
*/
public void documentRecord(
IPipelineSpecificationBasic pipelineSpecificationBasic,
- String identifierClass, String identifierHash,
+ String identifierClass, String identifierHash, String childIdentifierHash,
String documentVersion, long recordTime,
IOutputActivity activities)
throws ManifoldCFException, ServiceInterruption;
@@ -300,6 +327,7 @@ public interface IIncrementalIngester
*@param pipelineSpecificationWithVersions is the pipeline specification with
already-fetched output versioning information.
*@param identifierClass is the name of the space in which the identifier
hash should be interpreted.
*@param identifierHash is the hashed document identifier.
+ *@param childIdentifierHash is the hashed virtual child document identifier.
Pass null if this is a primary record.
*@param documentVersion is the document version.
*@param parameterVersion is the version string for the forced parameters.
*@param authorityName is the name of the authority associated with the
document, if any.
@@ -312,7 +340,7 @@ public interface IIncrementalIngester
*/
public boolean documentIngest(
IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
- String identifierClass, String identifierHash,
+ String identifierClass, String identifierHash, String childIdentifierHash,
String documentVersion,
String parameterVersion,
String authorityName,
Modified:
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
(original)
+++
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/OutputKey.java
Fri Jul 4 13:34:44 2014
@@ -32,14 +32,16 @@ public class OutputKey
protected final String documentClass;
protected final String documentIDHash;
+ protected final String childIDHash;
protected final String outputConnectionName;
/** Constructor */
- public OutputKey(String documentClass, String documentIDHash, String
outputConnectionName)
+ public OutputKey(String documentClass, String documentIDHash, String
childIDHash, String outputConnectionName)
{
// Identifying information
this.documentClass = documentClass;
this.documentIDHash = documentIDHash;
+ this.childIDHash = childIDHash;
this.outputConnectionName = outputConnectionName;
}
@@ -55,6 +57,12 @@ public class OutputKey
return documentIDHash;
}
+ /** Get the child ID hash */
+ public String getChildIDHash()
+ {
+ return childIDHash;
+ }
+
/** Get the output connection name */
public String getOutputConnectionName()
{
@@ -63,7 +71,9 @@ public class OutputKey
public int hashCode()
{
- return documentClass.hashCode() + documentIDHash.hashCode() +
outputConnectionName.hashCode();
+ return documentClass.hashCode() + documentIDHash.hashCode() +
+ ((childIDHash==null)?0:childIDHash.hashCode()) +
+ outputConnectionName.hashCode();
}
public boolean equals(Object o)
@@ -73,6 +83,7 @@ public class OutputKey
OutputKey dis = (OutputKey)o;
return dis.documentClass.equals(documentClass) &&
dis.documentIDHash.equals(documentIDHash) &&
+ (((dis.childIDHash == null || childIDHash == null) && dis.childIDHash ==
childIDHash) || (dis.childIDHash != null && childIDHash != null &&
dis.childIDHash.equals(childIDHash))) &&
dis.outputConnectionName.equals(outputConnectionName);
}
Modified:
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
(original)
+++
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PipelineSpecificationWithVersions.java
Fri Jul 4 13:34:44 2014
@@ -49,7 +49,8 @@ public class PipelineSpecificationWithVe
protected DocumentIngestStatus getStatus(int index)
{
IPipelineSpecificationBasic basic =
pipelineSpecification.getBasicPipelineSpecification();
- return
queuedDocument.getLastIngestedStatus(basic.getStageConnectionName(basic.getOutputStage(index)));
+ // MHL
+ return
queuedDocument.getLastIngestedStatus(basic.getStageConnectionName(basic.getOutputStage(index)),"");
}
/** For a given output index, return a document version string.
Modified:
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
(original)
+++
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/QueuedDocument.java
Fri Jul 4 13:34:44 2014
@@ -38,8 +38,8 @@ public class QueuedDocument
/** The document description. */
protected final DocumentDescription documentDescription;
- /** The last ingested status, null meaning "never ingested". */
- protected final Map<String,DocumentIngestStatus> lastIngestedStatus;
+ /** The last ingested status for all child records, arranged by output
connection name. null means "never ingested". */
+ protected final Map<String,Map<String,DocumentIngestStatus>>
lastIngestedStatus;
/** The binnames for the document, according to the connector */
protected final String[] binNames;
/** This flag indicates whether the document has been processed or not. */
@@ -47,10 +47,11 @@ public class QueuedDocument
/** Constructor.
*@param documentDescription is the document description.
- *@param lastIngestedStatus is the document's last ingested status.
+ *@param lastIngestedStatus is the document's last ingested status. Map is
keyed by output connection name, and child map is
+ * keyed by child document id hash value, or a blank value for a primary
row.
*@param binNames are the bins associated with the document.
*/
- public QueuedDocument(DocumentDescription documentDescription,
Map<String,DocumentIngestStatus> lastIngestedStatus, String[] binNames)
+ public QueuedDocument(DocumentDescription documentDescription,
Map<String,Map<String,DocumentIngestStatus>> lastIngestedStatus, String[]
binNames)
{
this.documentDescription = documentDescription;
this.lastIngestedStatus = lastIngestedStatus;
@@ -65,15 +66,34 @@ public class QueuedDocument
return documentDescription;
}
+ /** Get an iterator over the child document IDs available for this document
identifier and a specified
+ * output connection name.
+ *@param outputConnectionName is the name of the output connection.
+ *@return an iterator over child document IDs. For the primary ID, an empty
value will be returned by this iterator.
+ */
+ public Iterator<String> getLastIngestedChildIDs(String outputConnectionName)
+ {
+ if (lastIngestedStatus == null)
+ return new HashSet<String>().iterator();
+ Map<String,DocumentIngestStatus> rval =
lastIngestedStatus.get(outputConnectionName);
+ if (rval == null)
+ return new HashSet<String>().iterator();
+ return rval.keySet().iterator();
+ }
+
/** Get the last ingested status.
*@param outputConnectionName is the name of the output connection.
+ *@param childIDHash is the child identifier hash, using an empty value for
the primary.
*@return the last ingested status for that output, or null if not found.
*/
- public DocumentIngestStatus getLastIngestedStatus(String
outputConnectionName)
+ public DocumentIngestStatus getLastIngestedStatus(String
outputConnectionName, String childIDHash)
{
if (lastIngestedStatus == null)
return null;
- return lastIngestedStatus.get(outputConnectionName);
+ Map<String,DocumentIngestStatus> rval =
lastIngestedStatus.get(outputConnectionName);
+ if (rval == null)
+ return null;
+ return rval.get(childIDHash);
}
/** Return true if there are *any* last ingested records.
Modified:
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
(original)
+++
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Fri Jul 4 13:34:44 2014
@@ -250,18 +250,36 @@ public class StufferThread extends Threa
Map<OutputKey,DocumentIngestStatus> statuses = new
HashMap<OutputKey,DocumentIngestStatus>();
ingester.getPipelineDocumentIngestDataMultiple(statuses,pipelineSpecifications,documentClasses,documentIDHashes);
- // Break apart the result.
+
+ // Process the results. Ideally, we'd just use a get operation to
find the right document status, but the child id messes
+ // that up, so we need to create a way of finding the right map to
put things into.
+ Map<String,Integer> indexMap = new HashMap<String,Integer>();
for (int i = 0; i < descs.length; i++)
{
- versions[i] = new HashMap<String,DocumentIngestStatus>();
- for (int j = 0; j < pipelineSpecifications[i].getOutputCount();
j++)
- {
- String outputName =
pipelineSpecifications[i].getStageConnectionName(pipelineSpecifications[i].getOutputStage(j));
- OutputKey key = new
OutputKey(documentClasses[i],documentIDHashes[i],outputName);
- DocumentIngestStatus status = statuses.get(key);
- if (status != null)
- versions[i].put(outputName,status);
- }
+ indexMap.put(documentClasses[i] + documentIDHashes[i],new
Integer(i));
+ versions[i] = null;
+ }
+
+ // Populate each map in the way we need it.
+ for (OutputKey key : statuses.keySet())
+ {
+ int index = indexMap.get(key.getDocumentClass() +
key.getDocumentIDHash()).intValue();
+ Map<String,Map<String,DocumentIngestStatus>> entry =
versions[index];
+ if (entry == null)
+ {
+ entry = new HashMap<String,Map<String,DocumentIngestStatus>>();
+ versions[index] = entry;
+ }
+ Map<String,DocumentIngestStatus> childAndStatus =
entry.get(key.getOutputConnectionName());
+ if (childAndStatus == null)
+ {
+ childAndStatus = new HashMap<String,DocumentIngestStatus>();
+ entry.put(key.getOutputConnectionName(),childAndStatus);
+ }
+ String childIDHash = key.getChildIDHash();
+ if (childIDHash == null)
+ childIDHash = "";
+ childAndStatus.put(childIDHash,statuses.get(key));
}
// We need to go through the list, and segregate them by job, so the
individual
@@ -335,7 +353,7 @@ public class StufferThread extends Threa
binNames = new String[]{""};
}
- QueuedDocument qd = new
QueuedDocument(descs[i],(Map<String,DocumentIngestStatus>)versions[i],binNames);
+ QueuedDocument qd = new
QueuedDocument(descs[i],(Map<String,Map<String,DocumentIngestStatus>>)versions[i],binNames);
// Grab the arraylist that's there, or create it.
List<QueuedDocument> set = documentSets.get(jobID);
Modified:
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1607853&r1=1607852&r2=1607853&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Fri Jul 4 13:34:44 2014
@@ -307,7 +307,8 @@ public class WorkerThread extends Thread
QueuedDocument qd = activeDocuments.get(i);
currentDocIDHashArray[i] =
qd.getDocumentDescription().getDocumentIdentifierHash();
currentDocIDArray[i] =
qd.getDocumentDescription().getDocumentIdentifier();
- DocumentIngestStatus dis =
qd.getLastIngestedStatus(lastIndexedOutputConnectionName);
+ // MHL
+ DocumentIngestStatus dis =
qd.getLastIngestedStatus(lastIndexedOutputConnectionName,"");
if (dis == null)
oldVersionStringArray[i] = null;
else
@@ -996,6 +997,7 @@ public class WorkerThread extends Thread
{
QueuedDocument qd = deleteList.get(i);
// See if we need to delete from index
+ // MHL
if (qd.anyLastIngestedRecords())
{
// Queue up to issue deletion
@@ -1746,9 +1748,10 @@ public class WorkerThread extends Thread
throws ManifoldCFException, ServiceInterruption
{
String documentIdentifierHash = ManifoldCF.hash(documentIdentifier);
+ // MHL
ingester.documentRecord(
pipelineSpecification.getBasicPipelineSpecification(),
- connectionName,documentIdentifierHash,
+ connectionName,documentIdentifierHash,null,
version,currentTime,ingestLogger);
}
@@ -1816,9 +1819,10 @@ public class WorkerThread extends Thread
}
// First, we need to add into the metadata the stuff from the job
description.
+ // MHL
ingester.documentIngest(
fetchPipelineSpecifications.get(documentIdentifierHash),
- connectionName,documentIdentifierHash,
+ connectionName,documentIdentifierHash,null,
version,parameterVersion,
connection.getACLAuthority(),
data,currentTime,
