Author: kwright
Date: Sat Sep 13 12:40:13 2014
New Revision: 1624731
URL: http://svn.apache.org/r1624731
Log:
Rework the Livelink connector. Part of CONNECTORS-977.
Modified:
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Modified:
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1624731&r1=1624730&r2=1624731&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
(original)
+++
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Sat Sep 13 12:40:13 2014
@@ -96,11 +96,6 @@ public class LivelinkConnector extends o
/** Deny access token for default authority */
private final static String defaultAuthorityDenyToken = GLOBAL_DENY_TOKEN;
- // Livelink does not have "deny" permissions, and there is no such thing as
a document with no tokens, so it is safe to not have a local "deny" token.
- // However, people feel that a suspenders-and-belt approach is called for,
so this restriction has been added.
- // Livelink tokens are numbers, "SYSTEM", or "GUEST", so they can't collide
with the standard form.
- private static final String denyToken = "DEAD_AUTHORITY";
-
// A couple of very important points.
// First, the canonical document identifier has the following form:
// <D|F>[<volume_id>:]<object_id>
@@ -1086,31 +1081,25 @@ public class LivelinkConnector extends o
return "";
}
-
- /** Get document versions given an array of document identifiers.
- * This method is called for EVERY document that is considered. It is
- * therefore important to perform as little work as possible here.
- *@param documentIdentifiers is the array of local document identifiers, as
understood by this connector.
- *@param oldVersions is the corresponding array of version strings that have
been saved for the document identifiers.
- * A null value indicates that this is a first-time fetch, while an empty
string indicates that the previous document
- * had an empty version string.
- *@param activities is the interface this method should use to perform
whatever framework actions are desired.
- *@param spec is the current document specification for the current job. If
there is a dependency on this
- * specification, then the version string should include the pertinent data,
so that reingestion will occur
- * when the specification changes. This is primarily useful for metadata.
+ /** Process a set of documents.
+ * This is the method that should cause each document to be fetched,
processed, and the results either added
+ * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
+ * The document specification allows this class to filter what is done based
on the job.
+ * The connector will be connected before this method can be called.
+ *@param documentIdentifiers is the set of document identifiers to process.
+ *@param statuses are the currently-stored document versions for each
document in the set of document identifiers
+ * passed in above.
+ *@param activities is the interface this method should use to queue up new
document references
+ * and ingest documents.
*@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
*@param usesDefaultAuthority will be true only if the authority in use for
these documents is the default one.
- *@return the corresponding version strings, with null in the places where
the document no longer exists.
- * Empty version strings indicate that there is no versioning ability for the
corresponding document, and the document
- * will always be processed.
*/
@Override
- public String[] getDocumentVersions(String[] documentIdentifiers, String[]
oldVersions, IVersionActivity activities,
- DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
+ public void processDocuments(String[] documentIdentifiers, IExistingVersions
statuses, Specification spec,
+ IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption
{
- getSession();
-
+
// Initialize a "livelink context", to minimize the number of objects we
have to fetch
LivelinkContext llc = new LivelinkContext();
@@ -1189,22 +1178,23 @@ public class LivelinkConnector extends o
// Prepare the specified metadata
StringBuilder metadataString = null;
+ String[] specifiedMetadataAttributes = null;
CategoryPathAccumulator catAccum = null;
if (!includeAllMetadata)
{
metadataString = new StringBuilder();
// Put into an array
- String[] sortArray = new String[holder.size()];
+ specifiedMetadataAttributes = new String[holder.size()];
i = 0;
for (String attrName : holder)
{
- sortArray[i++] = attrName;
+ specifiedMetadataAttributes[i++] = attrName;
}
// Sort!
- java.util.Arrays.sort(sortArray);
+ java.util.Arrays.sort(specifiedMetadataAttributes);
// Build the metadata string piece now
- packList(metadataString,sortArray,'+');
+ packList(metadataString,specifiedMetadataAttributes,'+');
}
else
catAccum = new CategoryPathAccumulator(llc);
@@ -1215,23 +1205,24 @@ public class LivelinkConnector extends o
if (pathAttributeName != null)
pathNameAttributeVersion.append("=").append(pathAttributeName).append(":").append(pathSeparator).append(":").append(matchMap);
- // The version string includes the following:
- // 1) The modify date for the document
- // 2) The rights for the document, ordered (which can change without
changing the ModifyDate field)
- // 3) The requested metadata fields (category and attribute, ordered) for
the document
- //
- // The document identifiers are object id's.
+ // Initialize the table of catid's.
+ // Keeping this around will allow us to benefit from batching of documents.
+ MetadataDescription desc = new MetadataDescription(llc);
+
+ // Build the node/path cache
+ SystemMetadataDescription sDesc = new SystemMetadataDescription(llc,spec);
- String[] rval = new String[documentIdentifiers.length];
- i = 0;
- while (i < documentIdentifiers.length)
+ // Since the identifier indicates it is a directory, then queue up all the
current children which pass the filter.
+ String filterString = buildFilterString(spec);
+
+ for (String documentIdentifier : documentIdentifiers)
{
// Since each livelink access is time-consuming, be sure that we abort
if the job has gone inactive
activities.checkJobStillActive();
// Read the document or folder metadata, which includes the ModifyDate
- String docID = documentIdentifiers[i];
-
+ String docID = documentIdentifier;
+
boolean isFolder = docID.startsWith("F");
int colonPos = docID.indexOf(":",1);
@@ -1250,7 +1241,7 @@ public class LivelinkConnector extends o
vol = new Integer(docID.substring(1,colonPos)).intValue();
}
- rval[i] = null;
+ getSession();
ObjectInformation value = llc.getObjectInformation(vol,objID);
if (value.exists())
{
@@ -1265,88 +1256,191 @@ public class LivelinkConnector extends o
{
// We were able to get rights, so object still exists.
- // I rearranged this on 11/7/2006 so that it would be more
parseable, since
- // we want to pull the transient information out of the version
string where
- // possible (so there is no mismatch between version checking and
ingestion).
-
- StringBuilder sb = new StringBuilder();
-
- // On 1/17/2008 I changed the version generation code to NOT
include metadata, view info, etc. for folders, since
- // folders make absolutely no use of this info.
- if (!isFolder)
+ // Changed folder versioning for MCF 2.0
+ if (isFolder)
+ {
+ // === Livelink folder ===
+ // I'm still not sure if Livelink folder modified dates are
one-level or hierarchical.
+ // The code below assumes one-level only, so we always scan
folders and there's no versioning
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
+
+ int sanityRetryCount = FAILURE_RETRY_COUNT;
+ while (true)
+ {
+ ListObjectsThread t = new
ListObjectsThread(vol,objID,filterString);
+ try
+ {
+ t.start();
+ LLValue childrenDocs;
+ try
+ {
+ childrenDocs = t.finishUp();
+ }
+ catch (ManifoldCFException e)
+ {
+ sanityRetryCount = assessRetry(sanityRetryCount,e);
+ continue;
+ }
+
+ int size = 0;
+
+ if (childrenDocs.isRecord())
+ size = 1;
+ if (childrenDocs.isTable())
+ size = childrenDocs.size();
+
+ // System.out.println("Total child count =
"+Integer.toString(size));
+ // Do the scan
+ for (int j = 0; j < size; j++)
+ {
+ int childID = childrenDocs.toInteger(j, "ID");
+
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Found a child of
folder "+Integer.toString(vol)+":"+Integer.toString(objID)+" :
ID="+Integer.toString(childID));
+
+ int subtype = childrenDocs.toInteger(j, "SubType");
+ boolean childIsFolder = (subtype ==
LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
+ subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
+
+ // If it's a folder, we just let it through for now
+ if (!childIsFolder &&
checkInclude(childrenDocs.toString(j,"Name") + "." +
childrenDocs.toString(j,"FileType"), spec) == false)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" was excluded by inclusion criteria");
+ continue;
+ }
+
+ if (childIsFolder)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a folder, project, or compound document;
adding a reference");
+ if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
+ {
+ // If we pick up a project object, we need to describe
the volume object (which
+ // will be the root of all documents beneath)
+ activities.addDocumentReference("F"+new
Integer(childID).toString()+":"+new Integer(-childID).toString());
+ }
+ else
+ activities.addDocumentReference("F"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
+ }
+ else
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a simple document; adding a reference");
+
+ activities.addDocumentReference("D"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
+ }
+
+ }
+ break;
+ }
+ catch (InterruptedException e)
+ {
+ t.interrupt();
+ throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (RuntimeException e)
+ {
+ sanityRetryCount =
handleLivelinkRuntimeException(e,sanityRetryCount,true);
+ continue;
+ }
+ }
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Done processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
+ }
+ else
{
+ // === Livelink document ===
+
+ // The version string includes the following:
+ // 1) The modify date for the document
+ // 2) The rights for the document, ordered (which can change
without changing the ModifyDate field)
+ // 3) The requested metadata fields (category and attribute,
ordered) for the document
+ //
+ // The document identifiers are object id's.
+
+ StringBuilder sb = new StringBuilder();
+
+ String[] categoryPaths;
if (includeAllMetadata)
{
// Find all the metadata associated with this object, and then
// find the set of category pathnames that correspond to it.
int[] catIDs = getObjectCategoryIDs(vol,objID);
- String[] categoryPaths =
catAccum.getCategoryPathsAttributeNames(catIDs);
+ categoryPaths =
catAccum.getCategoryPathsAttributeNames(catIDs);
// Sort!
java.util.Arrays.sort(categoryPaths);
// Build the metadata string piece now
packList(sb,categoryPaths,'+');
}
else
+ {
+ categoryPaths = specifiedMetadataAttributes;
sb.append(metadataString);
- }
-
- String[] actualAcls;
- String denyAcl;
- if (acls != null && acls.length == 0)
- {
- // No forced acls. Read the actual acls from livelink, as a set
of rights.
- // We need also to add in support for the special rights
objects. These are:
- // -1: RIGHT_WORLD
- // -2: RIGHT_SYSTEM
- // -3: RIGHT_OWNER
- // -4: RIGHT_GROUP
- //
- // RIGHT_WORLD means guest access.
- // RIGHT_SYSTEM is "Public Access".
- // RIGHT_OWNER is access by the owner of the object.
- // RIGHT_GROUP is access by a member of the base group
containing the owner
- //
- // These objects are returned by the GetObjectRights() call made
above, and NOT
- // returned by LLUser.ListObjects(). We have to figure out how
to map these to
- // things that are
- // the equivalent of acls.
-
- actualAcls = lookupTokens(rights, value);
- java.util.Arrays.sort(actualAcls);
- // If security is on, no deny acl is needed for the local
authority, since the repository does not support "deny". But this was added
- // to be really really really sure.
- denyAcl = denyToken;
+ }
+
+ String[] actualAcls;
+ String[] denyAcls;
+
+ String denyAcl;
+ if (acls != null && acls.length == 0)
+ {
+ // No forced acls. Read the actual acls from livelink, as a
set of rights.
+ // We need also to add in support for the special rights
objects. These are:
+ // -1: RIGHT_WORLD
+ // -2: RIGHT_SYSTEM
+ // -3: RIGHT_OWNER
+ // -4: RIGHT_GROUP
+ //
+ // RIGHT_WORLD means guest access.
+ // RIGHT_SYSTEM is "Public Access".
+ // RIGHT_OWNER is access by the owner of the object.
+ // RIGHT_GROUP is access by a member of the base group
containing the owner
+ //
+ // These objects are returned by the GetObjectRights() call
made above, and NOT
+ // returned by LLUser.ListObjects(). We have to figure out
how to map these to
+ // things that are
+ // the equivalent of acls.
+
+ actualAcls = lookupTokens(rights, value);
+ java.util.Arrays.sort(actualAcls);
+ // If security is on, no deny acl is needed for the local
authority, since the repository does not support "deny". But this was added
+ // to be really really really sure.
+ denyAcl = defaultAuthorityDenyToken;
- }
- else if (acls != null && acls.length > 0)
- {
- // Forced acls
- actualAcls = acls;
- denyAcl = defaultAuthorityDenyToken;
- }
- else
- {
- // Security is OFF
- actualAcls = acls;
- denyAcl = null;
- }
+ }
+ else if (acls != null && acls.length > 0)
+ {
+ // Forced acls
+ actualAcls = acls;
+ denyAcl = defaultAuthorityDenyToken;
+ }
+ else
+ {
+ // Security is OFF
+ actualAcls = acls;
+ denyAcl = null;
+ }
- // Now encode the acls. If null, we write a special value.
- if (actualAcls == null)
- sb.append('-');
- else
- {
- sb.append('+');
- packList(sb,actualAcls,'+');
- // This was added on 4/21/2008 to support forced acls working
with the global default authority.
- pack(sb,denyAcl,'+');
- }
+ // Now encode the acls. If null, we write a special value.
+ if (actualAcls == null)
+ {
+ sb.append('-');
+ denyAcls = null;
+ }
+ else
+ {
+ sb.append('+');
+ packList(sb,actualAcls,'+');
+ // This was added on 4/21/2008 to support forced acls working
with the global default authority.
+ pack(sb,denyAcl,'+');
+ denyAcls = new String[]{denyAcl};
+ }
- // The date does not need to be parseable
- sb.append(dt.toString());
+ // The date does not need to be parseable
+ sb.append(dt.toString());
- if (!isFolder)
- {
// PathNameAttributeVersion comes completely from the spec, so
we don't
// have to worry about it changing. No need, therefore, to
parse it during
// processDocuments.
@@ -1354,33 +1448,60 @@ public class LivelinkConnector extends o
// Tack on ingestCgiPath, to insulate us against changes to the
repository connection setup. Added 9/7/07.
sb.append("_").append(viewBasePath);
- }
- rval[i] = sb.toString();
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Successfully calculated
version string for object "+Integer.toString(vol)+":"+Integer.toString(objID)+"
: '"+rval[i]+"'");
+ String versionString = sb.toString();
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Successfully calculated
version string for document
"+Integer.toString(vol)+":"+Integer.toString(objID)+" : '"+versionString+"'");
+
+ if
(activities.checkDocumentNeedsReindexing(documentIdentifier,versionString))
+ {
+ // Index the document
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Processing document
"+Integer.toString(vol)+":"+Integer.toString(objID));
+ if (checkIngest(llc,objID,spec))
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Decided to ingest
document "+Integer.toString(vol)+":"+Integer.toString(objID));
+
+ // Grab the access tokens for this file from the version
string, inside ingest method.
+
ingestFromLiveLink(llc,documentIdentifier,versionString,actualAcls,denyAcls,categoryPaths,activities,desc,sDesc);
+ }
+ else
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Decided not to ingest
document "+Integer.toString(vol)+":"+Integer.toString(objID)+" - Did not match
ingestion criteria");
+ activities.noDocument(documentIdentifier,versionString);
+ }
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Done processing document
"+Integer.toString(vol)+":"+Integer.toString(objID));
+ }
+ }
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("Livelink: Could not get rights for
object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
+ activities.deleteDocument(documentIdentifier);
+ continue;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("Livelink: Crawl user cannot see contents
of object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
+ activities.deleteDocument(documentIdentifier);
+ continue;
}
}
else
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("Livelink: Object
"+Integer.toString(vol)+":"+Integer.toString(objID)+" has no information -
deleting");
+ activities.deleteDocument(documentIdentifier);
+ continue;
}
- i++;
}
- return rval;
}
protected class ListObjectsThread extends Thread
@@ -1438,186 +1559,6 @@ public class LivelinkConnector extends o
}
}
- /** Process a set of documents.
- * This is the method that should cause each document to be fetched,
processed, and the results either added
- * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
- * The document specification allows this class to filter what is done based
on the job.
- *@param documentIdentifiers is the set of document identifiers to process.
- *@param activities is the interface this method should use to queue up new
document references
- * and ingest documents.
- *@param spec is the document specification.
- *@param scanOnly is an array corresponding to the document identifiers. It
is set to true to indicate when the processing
- * should only find other references, and should not actually call the
ingestion methods.
- */
- @Override
- public void processDocuments(String[] documentIdentifiers, String[]
versions, IProcessActivity activities, DocumentSpecification spec, boolean[]
scanOnly)
- throws ManifoldCFException, ServiceInterruption
- {
- getSession();
-
- // Initialize livelink context
- LivelinkContext llc = new LivelinkContext();
-
- // First, initialize the table of catid's.
- // Keeping this around will allow us to benefit from batching of documents.
- MetadataDescription desc = new MetadataDescription(llc);
-
- // Build the node/path cache
- SystemMetadataDescription sDesc = new SystemMetadataDescription(llc,spec);
-
- int i = 0;
- while (i < documentIdentifiers.length)
- {
- // Since each livelink access is time-consuming, be sure that we abort
if the job has gone inactive
- activities.checkJobStillActive();
-
- String documentIdentifier = documentIdentifiers[i];
- String version = versions[i];
-
- boolean doScanOnly = scanOnly[i];
-
- boolean isFolder = documentIdentifier.startsWith("F");
- int colonPosition = documentIdentifier.indexOf(":",1);
- int vol;
- int objID;
- if (colonPosition == -1)
- {
- vol = LLENTWK_VOL;
- objID = new Integer(documentIdentifier.substring(1)).intValue();
- }
- else
- {
- vol = new
Integer(documentIdentifier.substring(1,colonPosition)).intValue();
- objID = new
Integer(documentIdentifier.substring(colonPosition+1)).intValue();
- }
-
- if (isFolder)
- {
- // We don't know if LL will let us version folders properly, so ALWAYS
process folders.
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
-
- // Since the identifier indicates it is a directory, then queue up all
the current children which pass the filter.
- String filterString = buildFilterString(spec);
-
- int sanityRetryCount = FAILURE_RETRY_COUNT;
- while (true)
- {
- ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
- try
- {
- t.start();
- LLValue childrenDocs;
- try
- {
- childrenDocs = t.finishUp();
- }
- catch (ManifoldCFException e)
- {
- sanityRetryCount = assessRetry(sanityRetryCount,e);
- continue;
- }
-
- int size = 0;
-
- if (childrenDocs.isRecord())
- size = 1;
- if (childrenDocs.isTable())
- size = childrenDocs.size();
-
- // System.out.println("Total child count =
"+Integer.toString(size));
- // Do the scan
- int j = 0;
- while (j < size)
- {
- int childID = childrenDocs.toInteger(j, "ID");
-
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Found a child of folder
"+Integer.toString(vol)+":"+Integer.toString(objID)+" :
ID="+Integer.toString(childID));
-
- int subtype = childrenDocs.toInteger(j, "SubType");
- boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE
|| subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
- subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
-
- // If it's a folder, we just let it through for now
- if (!childIsFolder &&
checkInclude(childrenDocs.toString(j,"Name") + "." +
childrenDocs.toString(j,"FileType"), spec) == false)
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" was excluded by inclusion criteria");
- j++;
- continue;
- }
-
- if (childIsFolder)
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a folder, project, or compound document;
adding a reference");
- if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
- {
- // If we pick up a project object, we need to describe the
volume object (which
- // will be the root of all documents beneath)
- activities.addDocumentReference("F"+new
Integer(childID).toString()+":"+new Integer(-childID).toString());
- }
- else
- activities.addDocumentReference("F"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
- }
- else
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a simple document; adding a reference");
-
- activities.addDocumentReference("D"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
- }
-
- j++;
- }
- break;
- }
- catch (InterruptedException e)
- {
- t.interrupt();
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (RuntimeException e)
- {
- sanityRetryCount =
handleLivelinkRuntimeException(e,sanityRetryCount,true);
- continue;
- }
- }
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Done processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
- }
- else
- {
- // It's a known file, and we've already checked whether it's allowed
or not (except any
- // checks based on the file data)
-
- if (doScanOnly == false)
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Processing document
"+Integer.toString(vol)+":"+Integer.toString(objID));
- if (checkIngest(llc,objID,spec))
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Decided to ingest document
"+Integer.toString(vol)+":"+Integer.toString(objID));
-
- // Grab the access tokens for this file from the version string,
inside ingest method.
-
ingestFromLiveLink(llc,documentIdentifiers[i],version,activities,desc,sDesc);
- }
- else
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Decided not to ingest
document "+Integer.toString(vol)+":"+Integer.toString(objID)+" - Did not match
ingestion criteria");
-
- }
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Done processing document
"+Integer.toString(vol)+":"+Integer.toString(objID));
- }
- }
- i++;
- }
- }
-
/** Get the maximum number of documents to amalgamate together into one
batch, for this connector.
*@return the maximum number. 0 indicates "unlimited".
*/
@@ -4267,6 +4208,8 @@ public class LivelinkConnector extends o
*/
protected void ingestFromLiveLink(LivelinkContext llc,
String documentIdentifier, String version,
+ String[] actualAcls, String[] denyAcls,
+ String[] categoryPaths,
IProcessActivity activities,
MetadataDescription desc, SystemMetadataDescription sDesc)
throws ManifoldCFException, ServiceInterruption
@@ -4371,16 +4314,10 @@ public class LivelinkConnector extends o
// Iterate over the metadata items. These are organized by
category
// for speed of lookup.
- // Unpack version string
- int startPos = 0;
-
- // Metadata items first
- ArrayList metadataItems = new ArrayList();
- startPos = unpackList(metadataItems,version,startPos,'+');
- Iterator catIter = desc.getItems(metadataItems);
+ Iterator<MetadataItem> catIter = desc.getItems(categoryPaths);
while (catIter.hasNext())
{
- MetadataItem item = (MetadataItem)catIter.next();
+ MetadataItem item = catIter.next();
MetadataPathItem pathItem = item.getPathItem();
if (pathItem != null)
{
@@ -4390,10 +4327,10 @@ public class LivelinkConnector extends o
if (catVersion != null)
{
// Go through attributes now
- Iterator attrIter = item.getAttributeNames();
+ Iterator<String> attrIter = item.getAttributeNames();
while (attrIter.hasNext())
{
- String attrName = (String)attrIter.next();
+ String attrName = attrIter.next();
// Create a unique metadata name
String metadataName = pathItem.getCatName()+":"+attrName;
// Fetch the metadata and stuff it into the RepositoryData
structure
@@ -4408,32 +4345,8 @@ public class LivelinkConnector extends o
}
}
- // Unpack acls (conditionally)
- if (startPos < version.length())
- {
- char x = version.charAt(startPos++);
- if (x == '+')
- {
- ArrayList acls = new ArrayList();
- startPos = unpackList(acls,version,startPos,'+');
- // Turn into acls and add into description
- String[] aclArray = new String[acls.size()];
- int j = 0;
- while (j < aclArray.length)
- {
- aclArray[j] = (String)acls.get(j);
- j++;
- }
-
- StringBuilder denyBuffer = new StringBuilder();
- startPos = unpack(denyBuffer,version,startPos,'+');
- String denyAcl = denyBuffer.toString();
- String[] denyAclArray = new String[1];
- denyAclArray[0] = denyAcl;
-
-
rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,aclArray,denyAclArray);
- }
- }
+ if (actualAcls != null && denyAcls != null)
+
rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,actualAcls,denyAcls);
// Add the path metadata item into the mix, if enabled
String pathAttributeName = sDesc.getPathAttributeName();
@@ -6708,7 +6621,7 @@ public class LivelinkConnector extends o
*@param spec is the specification string.
*@return the correct part of the filter string. Empty string is returned if
there is no filtering.
*/
- protected static String buildFilterString(DocumentSpecification spec)
+ protected static String buildFilterString(Specification spec)
{
StringBuilder rval = new StringBuilder();
@@ -6758,7 +6671,7 @@ public class LivelinkConnector extends o
*@param spec is the document specification.
*@return the acls.
*/
- protected static String[] getAcls(DocumentSpecification spec)
+ protected static String[] getAcls(Specification spec)
{
HashMap map = new HashMap();
int i = 0;
@@ -6798,7 +6711,7 @@ public class LivelinkConnector extends o
*@param documentSpecification is the specification.
*@return true if it should be included.
*/
- protected static boolean checkInclude(String filename, DocumentSpecification
documentSpecification)
+ protected static boolean checkInclude(String filename, Specification
documentSpecification)
throws ManifoldCFException
{
// Scan includes to insure we match
@@ -6842,7 +6755,7 @@ public class LivelinkConnector extends o
*@param objID is the file ID.
*@param documentSpecification is the specification.
*/
- protected boolean checkIngest(LivelinkContext llc, int objID,
DocumentSpecification documentSpecification)
+ protected boolean checkIngest(LivelinkContext llc, int objID, Specification
documentSpecification)
throws ManifoldCFException
{
// Since the only exclusions at this point are not based on file contents,
this is a no-op.
@@ -6988,7 +6901,7 @@ public class LivelinkConnector extends o
protected static class MetadataItem
{
protected MetadataPathItem pathItem;
- protected Map attributeNames = new HashMap();
+ protected Set<String> attributeNames = new HashSet<String>();
/** Constructor.
*/
@@ -7001,7 +6914,7 @@ public class LivelinkConnector extends o
*/
public void addAttribute(String attributeName)
{
- attributeNames.put(attributeName,attributeName);
+ attributeNames.add(attributeName);
}
/** Get the path object.
@@ -7015,9 +6928,9 @@ public class LivelinkConnector extends o
/** Get an iterator over the attribute names.
*@return the iterator.
*/
- public Iterator getAttributeNames()
+ public Iterator<String> getAttributeNames()
{
- return attributeNames.keySet().iterator();
+ return attributeNames.iterator();
}
}
@@ -7043,7 +6956,7 @@ public class LivelinkConnector extends o
protected MatchMap matchMap = new MatchMap();
/** Constructor */
- public SystemMetadataDescription(LivelinkContext llc,
DocumentSpecification spec)
+ public SystemMetadataDescription(LivelinkContext llc, Specification spec)
throws ManifoldCFException
{
this.llc = llc;
@@ -7172,7 +7085,7 @@ public class LivelinkConnector extends o
protected final LivelinkContext llc;
// This is a map of category name to category ID and attributes
- protected Map categoryMap = new HashMap();
+ protected final Map<String,MetadataPathItem> categoryMap = new
HashMap<String,MetadataPathItem>();
/** Constructor.
*/
@@ -7184,21 +7097,19 @@ public class LivelinkConnector extends o
/** Iterate over the metadata items represented by the specified chunk of
version string.
*@return an iterator over MetadataItem objects.
*/
- public Iterator getItems(ArrayList metadataItems)
+ public Iterator<MetadataItem> getItems(String[] metadataItems)
throws ManifoldCFException, ServiceInterruption
{
// This is the map that will be iterated over for a return value.
// It gets built out of (hopefully cached) data from categoryMap.
- HashMap newMap = new HashMap();
+ Map<String,MetadataItem> newMap = new HashMap<String,MetadataItem>();
// Start at root
ObjectInformation rootValue = null;
// Walk through string and process each metadata element in turn.
- int i = 0;
- while (i < metadataItems.size())
+ for (String metadataSpec : metadataItems)
{
- String metadataSpec = (String)metadataItems.get(i++);
StringBuilder categoryBuffer = new StringBuilder();
StringBuilder attributeBuffer = new StringBuilder();
unpackCategoryAttribute(categoryBuffer,attributeBuffer,metadataSpec);
@@ -7206,12 +7117,12 @@ public class LivelinkConnector extends o
String attributeName = attributeBuffer.toString();
// If there's already an entry for this category in the return map,
use it
- MetadataItem mi = (MetadataItem)newMap.get(category);
+ MetadataItem mi = newMap.get(category);
if (mi == null)
{
// Now, look up the node information
// Convert category to cat id.
- MetadataPathItem item = (MetadataPathItem)categoryMap.get(category);
+ MetadataPathItem item = categoryMap.get(category);
if (item == null)
{
RootValue rv = new RootValue(llc,category);
