Author: kwright
Date: Tue Jul 29 09:42:49 2014
New Revision: 1614295
URL: http://svn.apache.org/r1614295
Log:
Remove versioning for folders for HDFS and Livelink: HDFS because it emulates
unix and windows, and Livelink because we don't know and won't know in time for
1.7
Modified:
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Modified:
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1614295&r1=1614294&r2=1614295&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Tue Jul 29 09:42:49 2014
@@ -418,26 +418,19 @@ public class HDFSRepositoryConnector ext
}
if (fileStatus.isDirectory()) {
- // Since we believe that downstream changes affect the current node's
version string,
- // then we only have to add references when there are detected changes.
- if (!scanOnly[i]) {
- activities.noDocument(documentIdentifier,version);
- /*
- * Queue up stuff for directory
- */
- String entityReference = documentIdentifier;
- FileStatus[] fileStatuses = getChildren(fileStatus.getPath());
- if (fileStatuses == null) {
- // Directory was deleted, so remove
- activities.deleteDocument(documentIdentifier);
- continue;
- }
- for (int j = 0; j < fileStatuses.length; j++) {
- FileStatus fs = fileStatuses[j++];
- String canonicalPath = fs.getPath().toString();
- if
(checkInclude(session.getUri().toString(),fs,canonicalPath,spec)) {
-
activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD);
- }
+ /*
+ * Queue up stuff for directory
+ */
+ String entityReference = documentIdentifier;
+ FileStatus[] fileStatuses = getChildren(fileStatus.getPath());
+ if (fileStatuses == null) {
+ continue;
+ }
+ for (int j = 0; j < fileStatuses.length; j++) {
+ FileStatus fs = fileStatuses[j++];
+ String canonicalPath = fs.getPath().toString();
+ if (checkInclude(session.getUri().toString(),fs,canonicalPath,spec))
{
+
activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD);
}
}
} else {
Modified:
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1614295&r1=1614294&r2=1614295&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
(original)
+++
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Tue Jul 29 09:42:49 2014
@@ -1473,103 +1473,99 @@ public class LivelinkConnector extends o
if (isFolder)
{
- if (doScanOnly == false)
- {
- activities.noDocument(documentIdentifier,version);
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
+ // We don't know if LL will let us version folders properly, so ALWAYS
process folders.
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
- // Since the identifier indicates it is a directory, then queue up
all the current children which pass the filter.
- String filterString = buildFilterString(spec);
+ // Since the identifier indicates it is a directory, then queue up all
the current children which pass the filter.
+ String filterString = buildFilterString(spec);
- int sanityRetryCount = FAILURE_RETRY_COUNT;
- while (true)
+ int sanityRetryCount = FAILURE_RETRY_COUNT;
+ while (true)
+ {
+ ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
+ try
{
- ListObjectsThread t = new
ListObjectsThread(vol,objID,filterString);
+ t.start();
+ LLValue childrenDocs;
try
{
- t.start();
- LLValue childrenDocs;
- try
- {
- childrenDocs = t.finishUp();
- }
- catch (ManifoldCFException e)
- {
- sanityRetryCount = assessRetry(sanityRetryCount,e);
- continue;
- }
+ childrenDocs = t.finishUp();
+ }
+ catch (ManifoldCFException e)
+ {
+ sanityRetryCount = assessRetry(sanityRetryCount,e);
+ continue;
+ }
- int size = 0;
+ int size = 0;
+
+ if (childrenDocs.isRecord())
+ size = 1;
+ if (childrenDocs.isTable())
+ size = childrenDocs.size();
- if (childrenDocs.isRecord())
- size = 1;
- if (childrenDocs.isTable())
- size = childrenDocs.size();
-
- // System.out.println("Total child count =
"+Integer.toString(size));
-
- // Do the scan
- int j = 0;
- while (j < size)
- {
- int childID = childrenDocs.toInteger(j, "ID");
+ // System.out.println("Total child count =
"+Integer.toString(size));
+ // Do the scan
+ int j = 0;
+ while (j < size)
+ {
+ int childID = childrenDocs.toInteger(j, "ID");
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Found a child of folder
"+Integer.toString(vol)+":"+Integer.toString(objID)+" :
ID="+Integer.toString(childID));
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Found a child of folder
"+Integer.toString(vol)+":"+Integer.toString(objID)+" :
ID="+Integer.toString(childID));
- int subtype = childrenDocs.toInteger(j, "SubType");
- boolean childIsFolder = (subtype ==
LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
- subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
+ int subtype = childrenDocs.toInteger(j, "SubType");
+ boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE
|| subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
+ subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
- // If it's a folder, we just let it through for now
- if (!childIsFolder &&
checkInclude(childrenDocs.toString(j,"Name") + "." +
childrenDocs.toString(j,"FileType"), spec) == false)
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" was excluded by inclusion criteria");
- j++;
- continue;
- }
+ // If it's a folder, we just let it through for now
+ if (!childIsFolder &&
checkInclude(childrenDocs.toString(j,"Name") + "." +
childrenDocs.toString(j,"FileType"), spec) == false)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" was excluded by inclusion criteria");
+ j++;
+ continue;
+ }
- if (childIsFolder)
+ if (childIsFolder)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a folder, project, or compound document;
adding a reference");
+ if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
{
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a folder, project, or compound document;
adding a reference");
- if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
- {
- // If we pick up a project object, we need to describe the
volume object (which
- // will be the root of all documents beneath)
- activities.addDocumentReference("F"+new
Integer(childID).toString()+":"+new Integer(-childID).toString());
- }
- else
- activities.addDocumentReference("F"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
+ // If we pick up a project object, we need to describe the
volume object (which
+ // will be the root of all documents beneath)
+ activities.addDocumentReference("F"+new
Integer(childID).toString()+":"+new Integer(-childID).toString());
}
else
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a simple document; adding a reference");
-
- activities.addDocumentReference("D"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
- }
+ activities.addDocumentReference("F"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
+ }
+ else
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Child identifier
"+Integer.toString(childID)+" is a simple document; adding a reference");
- j++;
+ activities.addDocumentReference("D"+new
Integer(vol).toString()+":"+new Integer(childID).toString());
}
- break;
- }
- catch (InterruptedException e)
- {
- t.interrupt();
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (RuntimeException e)
- {
- sanityRetryCount =
handleLivelinkRuntimeException(e,sanityRetryCount,true);
- continue;
+
+ j++;
}
+ break;
+ }
+ catch (InterruptedException e)
+ {
+ t.interrupt();
+ throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (RuntimeException e)
+ {
+ sanityRetryCount =
handleLivelinkRuntimeException(e,sanityRetryCount,true);
+ continue;
}
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Done processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
}
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Done processing folder
"+Integer.toString(vol)+":"+Integer.toString(objID));
}
else
{
