Author: kwright
Date: Sat Sep 13 12:39:17 2014
New Revision: 1624730
URL: http://svn.apache.org/r1624730
Log:
Rework Jira connector. Part of CONNECTORS-977.
Modified:
manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
Modified:
manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java?rev=1624730&r1=1624729&r2=1624730&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
(original)
+++
manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
Sat Sep 13 12:39:17 2014
@@ -51,6 +51,7 @@ import org.apache.manifoldcf.core.interf
import org.apache.manifoldcf.crawler.interfaces.DocumentSpecification;
import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
+import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
import java.util.Map.Entry;
/**
@@ -907,88 +908,97 @@ public class JiraRepositoryConnector ext
}
-
- /**
- * Process a set of documents. This is the method that should cause each
- * document to be fetched, processed, and the results either added to the
- * queue of documents for the current job, and/or entered into the
- * incremental ingestion manager. The document specification allows this
- * class to filter what is done based on the job.
- *
- * @param documentIdentifiers is the set of document identifiers to process.
- * @param versions is the corresponding document versions to process, as
- * returned by getDocumentVersions() above. The implementation may choose to
- * ignore this parameter and always process the current version.
- * @param activities is the interface this method should use to queue up new
- * document references and ingest documents.
- * @param spec is the document specification.
- * @param scanOnly is an array corresponding to the document identifiers. It
- * is set to true to indicate when the processing should only find other
- * references, and should not actually call the ingestion methods.
- * @param jobMode is an integer describing how the job is being run, whether
- * continuous or once-only.
- */
- @SuppressWarnings("unchecked")
+ /** Process a set of documents.
+ * This is the method that should cause each document to be fetched,
processed, and the results either added
+ * to the queue of documents for the current job, and/or entered into the
incremental ingestion manager.
+ * The document specification allows this class to filter what is done based
on the job.
+ * The connector will be connected before this method can be called.
+ *@param documentIdentifiers is the set of document identifiers to process.
+ *@param statuses are the currently-stored document versions for each
document in the set of document identifiers
+ * passed in above.
+ *@param activities is the interface this method should use to queue up new
document references
+ * and ingest documents.
+ *@param jobMode is an integer describing how the job is being run, whether
continuous or once-only.
+ *@param usesDefaultAuthority will be true only if the authority in use for
these documents is the default one.
+ */
@Override
- public void processDocuments(String[] documentIdentifiers, String[] versions,
- IProcessActivity activities, DocumentSpecification spec,
- boolean[] scanOnly) throws ManifoldCFException, ServiceInterruption {
-
- Logging.connectors.debug("JIRA: Inside processDocuments");
-
- for (int i = 0; i < documentIdentifiers.length; i++) {
- String nodeId = documentIdentifiers[i];
- String version = versions[i];
-
- long startTime = System.currentTimeMillis();
- String errorCode = "FAILED";
- String errorDesc = StringUtils.EMPTY;
- Long fileSize = null;
- boolean doLog = false;
+ public void processDocuments(String[] documentIdentifiers, IExistingVersions
statuses, Specification spec,
+ IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
+ throws ManifoldCFException, ServiceInterruption {
+
+ // Forced acls
+ String[] acls = getAcls(spec);
+ if (acls != null)
+ java.util.Arrays.sort(acls);
+
+ for (String documentIdentifier : documentIdentifiers) {
- try {
- if (Logging.connectors.isDebugEnabled()) {
- Logging.connectors.debug("JIRA: Processing document identifier '"
- + nodeId + "'");
+ if (documentIdentifier.startsWith("I-")) {
+ // It is an issue
+ String versionString;
+ String[] aclsToUse;
+ String issueID;
+ JiraIssue jiraFile;
+
+ issueID = documentIdentifier.substring(2);
+ jiraFile = getIssue(issueID);
+ if (jiraFile == null) {
+ activities.deleteDocument(documentIdentifier);
+ continue;
}
+ Date rev = jiraFile.getUpdatedDate();
+ if (rev != null) {
+ StringBuilder sb = new StringBuilder();
- if (!scanOnly[i]) {
- doLog = true;
+ if (acls == null) {
+ // Get acls from issue
+ List<String> users = getUsers(issueID);
+ aclsToUse = (String[])users.toArray(new String[0]);
+ java.util.Arrays.sort(aclsToUse);
+ } else {
+ aclsToUse = acls;
+ }
+
+ // Acls
+ packList(sb,aclsToUse,'+');
+ if (aclsToUse.length > 0) {
+ sb.append('+');
+ pack(sb,defaultAuthorityDenyToken,'+');
+ } else
+ sb.append('-');
+ sb.append(rev.toString());
+ versionString = sb.toString();
+ } else {
+ //a jira document that doesn't contain versioning information will
NEVER be processed.
+ // I don't know what this means, and whether it can ever occur.
+ activities.deleteDocument(documentIdentifier);
+ continue;
+ }
- if (nodeId.startsWith("I-")) {
- // It's an issue
- String issueKey = nodeId.substring(2);
- JiraIssue jiraFile = getIssue(issueKey);
- if (jiraFile == null) {
- activities.deleteDocument(nodeId);
- continue;
- }
-
+ if
(activities.checkDocumentNeedsReindexing(documentIdentifier,versionString))
+ {
+
+ long startTime = System.currentTimeMillis();
+ String errorCode = "FAILED";
+ String errorDesc = StringUtils.EMPTY;
+ Long fileSize = null;
+
+ try {
if (Logging.connectors.isDebugEnabled()) {
- Logging.connectors.debug("JIRA: This issue exists: " +
jiraFile.getKey());
- }
-
- // Unpack the version string
- ArrayList acls = new ArrayList();
- StringBuilder denyAclBuffer = new StringBuilder();
- int index = unpackList(acls,version,0,'+');
- if (index < version.length() && version.charAt(index++) == '+') {
- index = unpack(denyAclBuffer,version,index,'+');
+ Logging.connectors.debug("JIRA: Processing document identifier '"
+ + documentIdentifier + "'");
}
//otherwise process
RepositoryDocument rd = new RepositoryDocument();
// Turn into acls and add into description
- String[] aclArray = new String[acls.size()];
- for (int j = 0; j < aclArray.length; j++) {
- aclArray[j] = (String)acls.get(j);
- }
-
rd.setSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,aclArray);
- if (denyAclBuffer.length() > 0) {
- String[] denyAclArray = new String[]{denyAclBuffer.toString()};
-
rd.setSecurityDenyACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,denyAclArray);
- }
+ String[] denyAclsToUse;
+ if (aclsToUse.length > 0)
+ denyAclsToUse = new String[]{defaultAuthorityDenyToken};
+ else
+ denyAclsToUse = new String[0];
+
rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,aclsToUse,denyAclsToUse);
// Now do standard stuff
@@ -1020,7 +1030,7 @@ public class JiraRepositoryConnector ext
InputStream is = new ByteArrayInputStream(documentBytes);
try {
rd.setBinary(is, documentBytes.length);
- activities.ingestDocumentWithException(nodeId, version,
documentURI, rd);
+ activities.ingestDocumentWithException(documentIdentifier,
versionString, documentURI, rd);
// No errors. Record the fact that we made it.
errorCode = "OK";
fileSize = new Long(documentBytes.length);
@@ -1030,16 +1040,20 @@ public class JiraRepositoryConnector ext
} catch (java.io.IOException e) {
handleIOException(e);
}
+ } finally {
+ activities.recordActivity(new Long(startTime), ACTIVITY_READ,
+ fileSize, documentIdentifier, errorCode, errorDesc, null);
}
+
}
- } finally {
- if (doLog)
- activities.recordActivity(new Long(startTime), ACTIVITY_READ,
- fileSize, nodeId, errorCode, errorDesc, null);
+ } else {
+ // Unrecognized identifier type
+ activities.deleteDocument(documentIdentifier);
+ continue;
}
}
}
-
+
protected static String getJiraBody(JiraIssue jiraFile) {
String summary = jiraFile.getSummary();
String description = jiraFile.getDescription();
@@ -1054,78 +1068,11 @@ public class JiraRepositoryConnector ext
return body.toString();
}
- /**
- * The short version of getDocumentVersions. Get document versions given an
- * array of document identifiers. This method is called for EVERY document
- * that is considered. It is therefore important to perform as little work
- * as possible here.
- *
- * @param documentIdentifiers is the array of local document identifiers, as
- * understood by this connector.
- * @param spec is the current document specification for the current job. If
- * there is a dependency on this specification, then the version string
- * should include the pertinent data, so that reingestion will occur when
- * the specification changes. This is primarily useful for metadata.
- * @return the corresponding version strings, with null in the places where
- * the document no longer exists. Empty version strings indicate that there
- * is no versioning ability for the corresponding document, and the document
- * will always be processed.
- */
- @Override
- public String[] getDocumentVersions(String[] documentIdentifiers,
- DocumentSpecification spec) throws ManifoldCFException,
- ServiceInterruption {
-
- // Forced acls
- String[] acls = getAcls(spec);
- if (acls != null)
- java.util.Arrays.sort(acls);
-
- String[] rval = new String[documentIdentifiers.length];
- for (int i = 0; i < rval.length; i++) {
- String nodeId = documentIdentifiers[i];
- if (nodeId.startsWith("I-")) {
- // It is an issue
- String issueID = nodeId.substring(2);
- JiraIssue jiraFile = getIssue(issueID);
- Date rev = jiraFile.getUpdatedDate();
- if (rev != null) {
- StringBuilder sb = new StringBuilder();
-
- String[] aclsToUse;
- if (acls == null) {
- // Get acls from issue
- List<String> users = getUsers(issueID);
- aclsToUse = (String[])users.toArray(new String[0]);
- java.util.Arrays.sort(aclsToUse);
- } else {
- aclsToUse = acls;
- }
-
- // Acls
- packList(sb,aclsToUse,'+');
- if (aclsToUse.length > 0) {
- sb.append('+');
- pack(sb,defaultAuthorityDenyToken,'+');
- } else
- sb.append('-');
- sb.append(rev.toString());
- rval[i] = sb.toString();
- } else {
- //a jira document that doesn't contain versioning information will
NEVER be processed.
- // I don't know what this means, and whether it can ever occur.
- rval[i] = null;
- }
- }
- }
- return rval;
- }
-
/** Grab forced acl out of document specification.
*@param spec is the document specification.
*@return the acls, or null if security is on (and the acls need to be
fetched)
*/
- protected static String[] getAcls(DocumentSpecification spec) {
+ protected static String[] getAcls(Specification spec) {
Set<String> map = new HashSet<String>();
for (int i = 0; i < spec.getChildCount(); i++) {
SpecificationNode sn = spec.getChild(i);
