Modified: manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java?rev=1626228&r1=1626227&r2=1626228&view=diff ============================================================================== --- manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java (original) +++ manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java Fri Sep 19 14:22:27 2014 @@ -57,10 +57,11 @@ import org.apache.manifoldcf.core.interf import org.apache.manifoldcf.core.interfaces.Configuration; import org.apache.manifoldcf.core.interfaces.ConfigurationNode; import org.apache.manifoldcf.core.interfaces.LockManagerFactory; -import org.apache.manifoldcf.crawler.interfaces.DocumentSpecification; import org.apache.manifoldcf.crawler.interfaces.IDocumentIdentifierStream; import org.apache.manifoldcf.crawler.interfaces.IProcessActivity; +import org.apache.manifoldcf.crawler.interfaces.IExistingVersions; import org.apache.manifoldcf.crawler.interfaces.IFingerprintActivity; +import org.apache.manifoldcf.crawler.interfaces.DocumentSpecification; import org.apache.manifoldcf.core.interfaces.Specification; import org.apache.manifoldcf.core.interfaces.SpecificationNode; import org.apache.manifoldcf.crawler.interfaces.IVersionActivity; @@ -450,30 +451,24 @@ public class SharedDriveConnector extend return new IdentifierStream(spec); } - - /** Get document versions given an array of document identifiers. - * This method is called for EVERY document that is considered. It is - * therefore important to perform as little work as possible here. - *@param documentIdentifiers is the array of local document identifiers, as understood by this connector. - *@param oldVersions is the corresponding array of version strings that have been saved for the document identifiers. - * A null value indicates that this is a first-time fetch, while an empty string indicates that the previous document - * had an empty version string. - *@param activities is the interface this method should use to perform whatever framework actions are desired. - *@param spec is the current document specification for the current job. If there is a dependency on this - * specification, then the version string should include the pertinent data, so that reingestion will occur - * when the specification changes. This is primarily useful for metadata. + /** Process a set of documents. + * This is the method that should cause each document to be fetched, processed, and the results either added + * to the queue of documents for the current job, and/or entered into the incremental ingestion manager. + * The document specification allows this class to filter what is done based on the job. + * The connector will be connected before this method can be called. + *@param documentIdentifiers is the set of document identifiers to process. + *@param statuses are the currently-stored document versions for each document in the set of document identifiers + * passed in above. + *@param activities is the interface this method should use to queue up new document references + * and ingest documents. *@param jobMode is an integer describing how the job is being run, whether continuous or once-only. *@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one. - *@return the corresponding version strings, with null in the places where the document no longer exists. - * Empty version strings indicate that there is no versioning ability for the corresponding document, and the document - * will always be processed. */ @Override - public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities, - DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority) + public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec, + IProcessActivity activities, int jobMode, boolean usesDefaultAuthority) throws ManifoldCFException, ServiceInterruption { - getSession(); // Read the forced acls. A null return indicates that security is disabled!!! // A zero-length return indicates that the native acls should be used. // All of this is germane to how we ingest the document, so we need to note it in @@ -515,17 +510,34 @@ public class SharedDriveConnector extend } } - String[] rval = new String[documentIdentifiers.length]; - String documentIdentifier = null; - i = 0; - while (i < rval.length) + for (String documentIdentifier : documentIdentifiers) { - documentIdentifier = documentIdentifiers[i]; + getSession(); + + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("JCIFS: Processing '"+documentIdentifier+"'"); + + String versionString; + SmbFile file; + + String ingestionURI = null; + String pathAttributeValue = null; + + String[] shareAllow = null; + String[] shareDeny = null; + boolean shareSecurityOn = false; + + String[] parentAllow = null; + String[] parentDeny = null; + boolean parentSecurityOn = false; + + String[] documentAllow = null; + String[] documentDeny = null; + boolean documentSecurityOn = false; + try { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: getVersions(): documentIdentifiers[" + i + "] is: " + documentIdentifier); - SmbFile file = new SmbFile(documentIdentifier,pa); + file = new SmbFile(documentIdentifier,pa); // File has to exist AND have a non-null canonical path to be readable. If the canonical path is // null, it means that the windows permissions are not right and directory/file is not readable!!! @@ -535,17 +547,43 @@ public class SharedDriveConnector extend { if (fileIsDirectory(file)) { + // Hmm, this is not correct; version string should be empty for windows directories, since + // they are not hierarchical in modified date propagation. // It's a directory. The version ID will be the // last modified date. - long lastModified = fileLastModified(file); - rval[i] = new Long(lastModified).toString(); + //long lastModified = fileLastModified(file); + //versionString = new Long(lastModified).toString(); + versionString = ""; } else { // It's a file of acceptable length. // The ability to get ACLs, list files, and an inputstream under DFS all work now. + // The SmbFile for parentFolder acls. + SmbFile parentFolder = new SmbFile(file.getParent(),pa); + // Compute the security information + String[] modelArray = new String[0]; + + List<String> allowList = new ArrayList<String>(); + List<String> denyList = new ArrayList<String>(); + shareSecurityOn = getFileShareSecuritySet(allowList, denyList, file, shareAcls); + shareAllow = allowList.toArray(modelArray); + shareDeny = denyList.toArray(modelArray); + + allowList.clear(); + denyList.clear(); + parentSecurityOn = getFileSecuritySet(allowList, denyList, parentFolder, parentFolderAcls); + parentAllow = allowList.toArray(modelArray); + parentDeny = denyList.toArray(modelArray); + + allowList.clear(); + denyList.clear(); + documentSecurityOn = getFileSecuritySet(allowList, denyList, file, acls); + documentAllow = allowList.toArray(modelArray); + documentDeny = denyList.toArray(modelArray); + // The format of this string changed on 11/8/2006 to be comformant with the standard way // acls and metadata descriptions are being stuffed into the version string across connectors. @@ -554,11 +592,9 @@ public class SharedDriveConnector extend StringBuilder sb = new StringBuilder(); - // The SmbFile for parentFolder acls. - SmbFile parentFolder = new SmbFile(file.getParent(),pa); - - // Parseable stuff goes first. There's no metadata for jcifs, so this will just be the acls - describeDocumentSecurity(sb,file,parentFolder,acls,shareAcls,parentFolderAcls); + addSecuritySet(sb,shareSecurityOn,shareAllow,shareDeny); + addSecuritySet(sb,parentSecurityOn,parentAllow,parentDeny); + addSecuritySet(sb,documentSecurityOn,documentAllow,documentDeny); // Include the path attribute name and value in the parseable area. if (pathAttributeName != null) @@ -566,7 +602,7 @@ public class SharedDriveConnector extend sb.append('+'); pack(sb,pathAttributeName,'+'); // Calculate path string; we'll include that wholesale in the version - String pathAttributeValue = documentIdentifier; + pathAttributeValue = documentIdentifier; // 3/13/2008 // In looking at what comes into the path metadata attribute by default, and cogitating a bit, I've concluded that // the smb:// and the server/domain name at the start of the path are just plain old noise, and should be stripped. @@ -588,7 +624,7 @@ public class SharedDriveConnector extend sb.append('-'); // Calculate the ingestion IRI/URI, and include that in the parseable area. - String ingestionURI = convertToURI(documentIdentifier,fileMap,uriMap); + ingestionURI = convertToURI(documentIdentifier,fileMap,uriMap); pack(sb,ingestionURI,'+'); // The stuff from here on down is non-parseable. @@ -608,16 +644,20 @@ public class SharedDriveConnector extend sb.append("I"); else sb.append(ifIndexable?"Y":"N"); - rval[i] = sb.toString(); + versionString = sb.toString(); } } else - rval[i] = null; + { + activities.deleteDocument(documentIdentifier); + continue; + } } catch (jcifs.smb.SmbAuthException e) { Logging.connectors.warn("JCIFS: Authorization exception reading version information for "+documentIdentifier+" - skipping"); - rval[i] = null; + activities.deleteDocument(documentIdentifier); + continue; } catch (MalformedURLException mue) { @@ -627,7 +667,8 @@ public class SharedDriveConnector extend catch (SmbException se) { processSMBException(se,documentIdentifier,"getting document version","fetching share security"); - rval[i] = null; + activities.deleteDocument(documentIdentifier); + continue; } catch (java.net.SocketTimeoutException e) { @@ -647,83 +688,42 @@ public class SharedDriveConnector extend throw new ServiceInterruption("Timeout or other service interruption: "+e.getMessage(),e,currentTime + 300000L, currentTime + 3 * 60 * 60000L,-1,false); } - i++; - } - return rval; - } - - - /** - * Process a set of documents. This is the method that should cause each - * document to be fetched, processed, and the results either added to the - * queue of documents for the current job, and/or entered into the - * incremental ingestion manager. The document specification allows this - * class to filter what is done based on the job. - * - * @param documentIdentifiers - * is the set of document identifiers to process. - * @param activities - * is the interface this method should use to queue up new - * document references and ingest documents. - * @param spec - * is the document specification. - * @param scanOnly - * is an array corresponding to the document identifiers. It is - * set to true to indicate when the processing should only find - * other references, and should not actually call the ingestion - * methods. - */ - @Override - public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities, - DocumentSpecification spec, boolean[] scanOnly) throws ManifoldCFException, ServiceInterruption - { - getSession(); - - byte[] transferBuffer = null; - - int i = 0; - while (i < documentIdentifiers.length) - { - String documentIdentifier = documentIdentifiers[i]; - String version = versions[i]; - - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Processing '"+documentIdentifier+"'"); - try + + if (versionString.length() == 0 || activities.checkDocumentNeedsReindexing(documentIdentifier,versionString)) { + byte[] transferBuffer = null; - SmbFile file = new SmbFile(documentIdentifier,pa); - - if (fileExists(file)) + try { - if (fileIsDirectory(file)) - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: '"+documentIdentifier+"' is a directory"); - // Queue up stuff for directory - // DFS special support no longer needed, because JCifs now does the right thing. - - // This is the string we replace in the child canonical paths. - // String matchPrefix = ""; - // This is what we replace it with, to get back to a DFS path. - // String matchReplace = ""; - - // DFS resolved. - - // Use a filter to actually do the work here. This prevents large arrays from being - // created when there are big directories. - ProcessDocumentsFilter filter = new ProcessDocumentsFilter(activities,spec); - fileListFiles(file,filter); - filter.checkAndThrow(); - } - else + if (fileExists(file)) { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: '"+documentIdentifier+"' is a file"); + if (fileIsDirectory(file)) + { + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("JCIFS: '"+documentIdentifier+"' is a directory"); + + // Queue up stuff for directory + // DFS special support no longer needed, because JCifs now does the right thing. - if (!scanOnly[i]) + // This is the string we replace in the child canonical paths. + // String matchPrefix = ""; + // This is what we replace it with, to get back to a DFS path. + // String matchReplace = ""; + + // DFS resolved. + + // Use a filter to actually do the work here. This prevents large arrays from being + // created when there are big directories. + ProcessDocumentsFilter filter = new ProcessDocumentsFilter(activities,spec); + fileListFiles(file,filter); + filter.checkAndThrow(); + } + else { + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("JCIFS: '"+documentIdentifier+"' is a file"); + // We've already avoided queuing documents that we // don't want, based on file specifications. // We still need to check based on file data. @@ -734,12 +734,17 @@ public class SharedDriveConnector extend String fileName = getFileCanonicalPath(file); if (fileName != null && !file.isHidden()) { - // Initialize repository document with common stuff, and find the URI - RepositoryDocument rd = new RepositoryDocument(); - String uri = prepareForIndexing(rd,file,version); + String uri = ingestionURI; if (activities.checkURLIndexable(uri)) { + // Initialize repository document with common stuff, and find the URI + RepositoryDocument rd = new RepositoryDocument(); + prepareForIndexing(rd,file, + shareAllow,shareDeny, + parentAllow,parentDeny, + documentAllow,documentDeny, + pathAttributeName,pathAttributeValue); // manipulate path to include the DFS alias, not the literal path // String newPath = matchPrefix + fileName.substring(matchReplace.length()); @@ -792,7 +797,7 @@ public class SharedDriveConnector extend { rd.setBinary(inputStream, tempFile.length()); - activities.ingestDocumentWithException(documentIdentifier, version, uri, rd); + activities.ingestDocumentWithException(documentIdentifier, versionString, uri, rd); } finally { @@ -813,7 +818,7 @@ public class SharedDriveConnector extend // method has no way of signalling this, since it does not do the fingerprinting. if (Logging.connectors.isDebugEnabled()) Logging.connectors.debug("JCIFS: Decided to remove '"+documentIdentifier+"'"); - activities.deleteDocument(documentIdentifier, version); + activities.noDocument(documentIdentifier, versionString); // We should record the access here as well, since this is a non-exception way through the code path. // (I noticed that this was not being recorded in the history while fixing 25477.) activities.recordActivity(new Long(startFetchTime),ACTIVITY_ACCESS, @@ -839,7 +844,7 @@ public class SharedDriveConnector extend { rd.setBinary(inputStream, fileLength(file)); - activities.ingestDocumentWithException(documentIdentifier, version, uri, rd); + activities.ingestDocumentWithException(documentIdentifier, versionString, uri, rd); } finally { @@ -854,6 +859,7 @@ public class SharedDriveConnector extend Logging.connectors.debug("JCIFS: Skipping file because output connector cannot accept it"); activities.recordActivity(null,ACTIVITY_ACCESS, null,documentIdentifier,"Skip","Output connector refused",null); + activities.noDocument(documentIdentifier,versionString); } } else @@ -861,130 +867,131 @@ public class SharedDriveConnector extend Logging.connectors.debug("JCIFS: Skipping file because canonical path is null, or because file is hidden"); activities.recordActivity(null,ACTIVITY_ACCESS, null,documentIdentifier,"Skip","Null canonical path or hidden file",null); + activities.noDocument(documentIdentifier,versionString); } } } } - } - catch (MalformedURLException mue) - { - Logging.connectors.error("MalformedURLException tossed",mue); - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Error","Malformed URL: "+mue.getMessage(),null); - throw new ManifoldCFException("MalformedURLException tossed: "+mue.getMessage(),mue); - } - catch (jcifs.smb.SmbAuthException e) - { - Logging.connectors.warn("JCIFS: Authorization exception reading document/directory "+documentIdentifier+" - skipping"); - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Skip","Authorization: "+e.getMessage(),null); - // We call the delete even if it's a directory; this is harmless. - activities.deleteDocument(documentIdentifier, version); - } - catch (SmbException se) - { - // At least some of these are transport errors, and should be treated as service - // interruptions. - long currentTime = System.currentTimeMillis(); - Throwable cause = se.getRootCause(); - if (cause != null && (cause instanceof jcifs.util.transport.TransportException)) + catch (MalformedURLException mue) { - // See if it's an interruption - jcifs.util.transport.TransportException te = (jcifs.util.transport.TransportException)cause; - if (te.getRootCause() != null && te.getRootCause() instanceof java.lang.InterruptedException) - throw new ManifoldCFException(te.getRootCause().getMessage(),te.getRootCause(),ManifoldCFException.INTERRUPTED); - - Logging.connectors.warn("JCIFS: Timeout processing document/directory "+documentIdentifier+": retrying...",se); + Logging.connectors.error("MalformedURLException tossed",mue); activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Retry","Transport: "+cause.getMessage(),null); - throw new ServiceInterruption("Timeout or other service interruption: "+cause.getMessage(),cause,currentTime + 300000L, - currentTime + 12 * 60 * 60000L,-1,false); + null,documentIdentifier,"Error","Malformed URL: "+mue.getMessage(),null); + throw new ManifoldCFException("MalformedURLException tossed: "+mue.getMessage(),mue); } - if (se.getMessage().indexOf("busy") != -1) + catch (jcifs.smb.SmbAuthException e) { - Logging.connectors.warn("JCIFS: 'Busy' response when processing document/directory for "+documentIdentifier+": retrying...",se); + Logging.connectors.warn("JCIFS: Authorization exception reading document/directory "+documentIdentifier+" - skipping"); activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Retry","Busy: "+se.getMessage(),null); - throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, - currentTime + 3 * 60 * 60000L,-1,false); + null,documentIdentifier,"Skip","Authorization: "+e.getMessage(),null); + // We call the delete even if it's a directory; this is harmless. + activities.noDocument(documentIdentifier, versionString); + } + catch (SmbException se) + { + // At least some of these are transport errors, and should be treated as service + // interruptions. + long currentTime = System.currentTimeMillis(); + Throwable cause = se.getRootCause(); + if (cause != null && (cause instanceof jcifs.util.transport.TransportException)) + { + // See if it's an interruption + jcifs.util.transport.TransportException te = (jcifs.util.transport.TransportException)cause; + if (te.getRootCause() != null && te.getRootCause() instanceof java.lang.InterruptedException) + throw new ManifoldCFException(te.getRootCause().getMessage(),te.getRootCause(),ManifoldCFException.INTERRUPTED); + + Logging.connectors.warn("JCIFS: Timeout processing document/directory "+documentIdentifier+": retrying...",se); + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Retry","Transport: "+cause.getMessage(),null); + throw new ServiceInterruption("Timeout or other service interruption: "+cause.getMessage(),cause,currentTime + 300000L, + currentTime + 12 * 60 * 60000L,-1,false); + } + if (se.getMessage().indexOf("busy") != -1) + { + Logging.connectors.warn("JCIFS: 'Busy' response when processing document/directory for "+documentIdentifier+": retrying...",se); + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Retry","Busy: "+se.getMessage(),null); + throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, + currentTime + 3 * 60 * 60000L,-1,false); + } + else if (se.getMessage().indexOf("handle is invalid") != -1) + { + Logging.connectors.warn("JCIFS: 'Handle is invalid' response when processing document/directory for "+documentIdentifier+": retrying...",se); + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Retry","Expiration: "+se.getMessage(),null); + throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, + currentTime + 3 * 60 * 60000L,-1,false); + } + else if (se.getMessage().indexOf("parameter is incorrect") != -1) + { + Logging.connectors.warn("JCIFS: 'Parameter is incorrect' response when processing document/directory for "+documentIdentifier+": retrying...",se); + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Retry","Expiration: "+se.getMessage(),null); + throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, + currentTime + 3 * 60 * 60000L,-1,false); + } + else if (se.getMessage().indexOf("no longer available") != -1) + { + Logging.connectors.warn("JCIFS: 'No longer available' response when processing document/directory for "+documentIdentifier+": retrying...",se); + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Retry","Expiration: "+se.getMessage(),null); + throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, + currentTime + 3 * 60 * 60000L,-1,false); + } + else if (se.getMessage().indexOf("cannot find") != -1 || se.getMessage().indexOf("cannot be found") != -1) + { + if (Logging.connectors.isDebugEnabled()) + Logging.connectors.debug("JCIFS: Skipping document/directory "+documentIdentifier+" because it cannot be found"); + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Not found",null,null); + activities.noDocument(documentIdentifier, versionString); + } + else if (se.getMessage().indexOf("is denied") != -1) + { + Logging.connectors.warn("JCIFS: Access exception reading document/directory "+documentIdentifier+" - skipping"); + // We call the delete even if it's a directory; this is harmless and it cleans up the jobqueue row. + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Skip","Authorization: "+se.getMessage(),null); + activities.noDocument(documentIdentifier, versionString); + } + else + { + Logging.connectors.error("JCIFS: SmbException tossed processing "+documentIdentifier,se); + activities.recordActivity(null,ACTIVITY_ACCESS, + null,documentIdentifier,"Error","Unknown: "+se.getMessage(),null); + throw new ManifoldCFException("SmbException tossed: "+se.getMessage(),se); + } } - else if (se.getMessage().indexOf("handle is invalid") != -1) + catch (java.net.SocketTimeoutException e) { - Logging.connectors.warn("JCIFS: 'Handle is invalid' response when processing document/directory for "+documentIdentifier+": retrying...",se); + long currentTime = System.currentTimeMillis(); + Logging.connectors.warn("JCIFS: Socket timeout processing "+documentIdentifier+": "+e.getMessage(),e); activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Retry","Expiration: "+se.getMessage(),null); - throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, + null,documentIdentifier,"Retry","Socket timeout: "+e.getMessage(),null); + throw new ServiceInterruption("Timeout or other service interruption: "+e.getMessage(),e,currentTime + 300000L, currentTime + 3 * 60 * 60000L,-1,false); } - else if (se.getMessage().indexOf("parameter is incorrect") != -1) + catch (InterruptedIOException e) { - Logging.connectors.warn("JCIFS: 'Parameter is incorrect' response when processing document/directory for "+documentIdentifier+": retrying...",se); - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Retry","Expiration: "+se.getMessage(),null); - throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, - currentTime + 3 * 60 * 60000L,-1,false); + throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED); } - else if (se.getMessage().indexOf("no longer available") != -1) + catch (IOException e) { - Logging.connectors.warn("JCIFS: 'No longer available' response when processing document/directory for "+documentIdentifier+": retrying...",se); + long currentTime = System.currentTimeMillis(); + Logging.connectors.warn("JCIFS: IO error processing "+documentIdentifier+": "+e.getMessage(),e); activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Retry","Expiration: "+se.getMessage(),null); - throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L, + null,documentIdentifier,"Retry","IO Error: "+e.getMessage(),null); + throw new ServiceInterruption("Timeout or other service interruption: "+e.getMessage(),e,currentTime + 300000L, currentTime + 3 * 60 * 60000L,-1,false); } - else if (se.getMessage().indexOf("cannot find") != -1 || se.getMessage().indexOf("cannot be found") != -1) - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Skipping document/directory "+documentIdentifier+" because it cannot be found"); - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Not found",null,null); - activities.deleteDocument(documentIdentifier, version); - } - else if (se.getMessage().indexOf("is denied") != -1) - { - Logging.connectors.warn("JCIFS: Access exception reading document/directory "+documentIdentifier+" - skipping"); - // We call the delete even if it's a directory; this is harmless and it cleans up the jobqueue row. - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Skip","Authorization: "+se.getMessage(),null); - activities.deleteDocument(documentIdentifier, version); - } - else - { - Logging.connectors.error("JCIFS: SmbException tossed processing "+documentIdentifier,se); - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Error","Unknown: "+se.getMessage(),null); - throw new ManifoldCFException("SmbException tossed: "+se.getMessage(),se); - } - } - catch (java.net.SocketTimeoutException e) - { - long currentTime = System.currentTimeMillis(); - Logging.connectors.warn("JCIFS: Socket timeout processing "+documentIdentifier+": "+e.getMessage(),e); - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Retry","Socket timeout: "+e.getMessage(),null); - throw new ServiceInterruption("Timeout or other service interruption: "+e.getMessage(),e,currentTime + 300000L, - currentTime + 3 * 60 * 60000L,-1,false); } - catch (InterruptedIOException e) - { - throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED); - } - catch (IOException e) - { - long currentTime = System.currentTimeMillis(); - Logging.connectors.warn("JCIFS: IO error processing "+documentIdentifier+": "+e.getMessage(),e); - activities.recordActivity(null,ACTIVITY_ACCESS, - null,documentIdentifier,"Retry","IO Error: "+e.getMessage(),null); - throw new ServiceInterruption("Timeout or other service interruption: "+e.getMessage(),e,currentTime + 300000L, - currentTime + 3 * 60 * 60000L,-1,false); - } - - i++; } - } - protected static String prepareForIndexing(RepositoryDocument rd, SmbFile file, String version) + + protected static void prepareForIndexing(RepositoryDocument rd, SmbFile file, + String[] shareAllow, String[] shareDeny, String[] parentAllow, String[] parentDeny, String[] allow, String[] deny, + String pathAttributeName, String pathAttributeValue) throws ManifoldCFException, SmbException { String fileNameString = file.getName(); @@ -1012,13 +1019,8 @@ public class SharedDriveConnector extend rd.addField("attributes", Integer.toString(attributes)); rd.addField("shareName", shareName); - - int index = 0; - index = setDocumentSecurity(rd,version,index); - index = setPathMetadata(rd,version,index); - StringBuilder ingestURI = new StringBuilder(); - index = unpack(ingestURI,version,index,'+'); - return ingestURI.toString(); + setDocumentSecurity(rd,shareAllow,shareDeny,parentAllow,parentDeny,allow,deny); + setPathMetadata(rd,pathAttributeName,pathAttributeValue); } /** Map an extension to a mime type */ @@ -1033,258 +1035,93 @@ public class SharedDriveConnector extend return ExtensionMimeMap.mapToMimeType(fileName.substring(dotIndex+1).toLowerCase(java.util.Locale.ROOT)); } - /** This method calculates an ACL string based on whether there are forced acls and also based on - * the acls in place for a file. - */ - protected void describeDocumentSecurity(StringBuilder description, - SmbFile file, SmbFile parentFolder, - String[] forcedacls, String[] forcedShareAcls, String[] forcedParentFolderAcls) - throws ManifoldCFException, IOException + protected static void addSecuritySet(StringBuilder description, + boolean enabled, String[] allowTokens, String[] denyTokens) { - String[] shareAllowAcls; - String[] shareDenyAcls; - String[] parentAllowAcls; - String[] parentDenyAcls; - String[] allowAcls; - String[] denyAcls; - - int j; - int allowCount; - int denyCount; - ACE[] aces; - - if (forcedShareAcls!=null) + if (enabled) { description.append("+"); + java.util.Arrays.sort(allowTokens); + java.util.Arrays.sort(denyTokens); + // Stuff the acls into the description string. + packList(description,allowTokens,'+'); + packList(description,denyTokens,'+'); + } + else + description.append("-"); - if (forcedShareAcls.length==0) + } + + protected boolean getFileSecuritySet(List<String> allowList, List<String> denyList, SmbFile file, String[] forced) + throws ManifoldCFException, IOException + { + if (forced != null) + { + if (forced.length == 0) { - // Do the share acls first. Note that the smbfile passed in has been dereferenced, - // so if this is a DFS path, we will be looking up the permissions on the share - // that is actually used to contain the file. However, there's no guarantee that the - // url generated from the original share will work to get there; the permissions on - // the original share may prohibit users that the could nevertheless see the document - // if they went in the direct way. - - - // Grab the share permissions. - aces = getFileShareSecurity(file, useSIDs); - - if (aces == null) - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Share has no ACL for '"+getFileCanonicalPath(file)+"'"); - - // "Public" share: S-1-1-0 - shareAllowAcls = new String[]{"S-1-1-0"}; - shareDenyAcls = new String[]{defaultAuthorityDenyToken}; - } - else - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Found "+Integer.toString(aces.length)+" share access tokens for '"+getFileCanonicalPath(file)+"'"); - - // We are interested in the read permission, and take - // a keen interest in allow/deny - allowCount = 0; - denyCount = 0; - j = 0; - while (j < aces.length) - { - ACE ace = aces[j++]; - if ((ace.getAccessMask() & ACE.FILE_READ_DATA) != 0) - { - if (ace.isAllow()) - allowCount++; - else - denyCount++; - } - } - - shareAllowAcls = new String[allowCount]; - shareDenyAcls = new String[denyCount+1]; - j = 0; - allowCount = 0; - denyCount = 0; - shareDenyAcls[denyCount++] = defaultAuthorityDenyToken; - while (j < aces.length) - { - ACE ace = aces[j++]; - if ((ace.getAccessMask() & ACE.FILE_READ_DATA) != 0) - { - if (ace.isAllow()) - shareAllowAcls[allowCount++] = useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName(); - else - shareDenyAcls[denyCount++] = useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName(); - } - } - } + convertACEs(allowList,denyList,getFileSecurity(file, useSIDs)); } else { - shareAllowAcls = forcedShareAcls; - if (forcedShareAcls.length == 0) - shareDenyAcls = new String[0]; - else - shareDenyAcls = new String[]{defaultAuthorityDenyToken}; + for (String forcedToken : forced) + { + allowList.add(forcedToken); + } + denyList.add(defaultAuthorityDenyToken); } - java.util.Arrays.sort(shareAllowAcls); - java.util.Arrays.sort(shareDenyAcls); - // Stuff the acls into the description string. - packList(description,shareAllowAcls,'+'); - packList(description,shareDenyAcls,'+'); + return true; } else - description.append('-'); + return false; + } - if (forcedParentFolderAcls!=null) + protected boolean getFileShareSecuritySet(List<String> allowList, List<String> denyList, SmbFile file, String[] forced) + throws ManifoldCFException, IOException + { + if (forced != null) { - description.append("+"); - - if (forcedParentFolderAcls.length==0) + if (forced.length == 0) { - aces = getFileSecurity(parentFolder, useSIDs); - if (aces == null) - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Parent folder has no ACL for '"+getFileCanonicalPath(parentFolder)+"'"); - - // Parent folder is "public", meaning we want S-1-1-0 and the deny token - parentAllowAcls = new String[]{"S-1-1-0"}; - parentDenyAcls = new String[]{defaultAuthorityDenyToken}; - } - else - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Found "+Integer.toString(aces.length)+" parent folder access tokens for '"+getFileCanonicalPath(parentFolder)+"'"); - - // We are interested in the read permission, and take - // a keen interest in allow/deny - allowCount = 0; - denyCount = 0; - j = 0; - while (j < aces.length) - { - ACE ace = aces[j++]; - if ((ace.getAccessMask() & ACE.FILE_READ_DATA) != 0) - { - if (ace.isAllow()) - allowCount++; - else - denyCount++; - } - } - - parentAllowAcls = new String[allowCount]; - parentDenyAcls = new String[denyCount+1]; - j = 0; - allowCount = 0; - denyCount = 0; - parentDenyAcls[denyCount++] = defaultAuthorityDenyToken; - while (j < aces.length) - { - ACE ace = aces[j++]; - if ((ace.getAccessMask() & ACE.FILE_READ_DATA) != 0) - { - if (ace.isAllow()) - parentAllowAcls[allowCount++] = useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName(); - else - parentDenyAcls[denyCount++] = useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName(); - } - } - } + convertACEs(allowList,denyList,getFileShareSecurity(file, useSIDs)); } else { - parentAllowAcls = forcedParentFolderAcls; - if (forcedParentFolderAcls.length == 0) - parentDenyAcls = new String[0]; - else - parentDenyAcls = new String[]{defaultAuthorityDenyToken}; + for (String forcedToken : forced) + { + allowList.add(forcedToken); + } + denyList.add(defaultAuthorityDenyToken); } - java.util.Arrays.sort(parentAllowAcls); - java.util.Arrays.sort(parentDenyAcls); - packList(description,parentAllowAcls,'+'); - packList(description,parentDenyAcls,'+'); + return true; } else - description.append('-'); - - if (forcedacls!=null) + return false; + } + + protected void convertACEs(List<String> allowList, List<String> denyList, ACE[] aces) + { + if (aces == null) { - description.append("+"); - - if (forcedacls.length==0) + // "Public" share: S-1-1-0 + allowList.add("S-1-1-0"); + denyList.add(defaultAuthorityDenyToken); + } + else + { + denyList.add(defaultAuthorityDenyToken); + for (ACE ace : aces) { - aces = getFileSecurity(file, useSIDs); - if (aces == null) + if ((ace.getAccessMask() & ACE.FILE_READ_DATA) != 0) { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Document has no ACL for '"+getFileCanonicalPath(file)+"'"); - - // Document is "public", meaning we want S-1-1-0 and the deny token - allowAcls = new String[]{"S-1-1-0"}; - denyAcls = new String[]{defaultAuthorityDenyToken}; - } - else - { - if (Logging.connectors.isDebugEnabled()) - Logging.connectors.debug("JCIFS: Found "+Integer.toString(aces.length)+" document access tokens for '"+getFileCanonicalPath(file)+"'"); - - // We are interested in the read permission, and take - // a keen interest in allow/deny - allowCount = 0; - denyCount = 0; - j = 0; - while (j < aces.length) - { - ACE ace = aces[j++]; - if ((ace.getAccessMask() & ACE.FILE_READ_DATA) != 0) - { - if (ace.isAllow()) - allowCount++; - else - denyCount++; - } - } - - allowAcls = new String[allowCount]; - denyAcls = new String[denyCount+1]; - j = 0; - allowCount = 0; - denyCount = 0; - denyAcls[denyCount++] = defaultAuthorityDenyToken; - while (j < aces.length) - { - ACE ace = aces[j++]; - if ((ace.getAccessMask() & ACE.FILE_READ_DATA) != 0) - { - if (ace.isAllow()) - allowAcls[allowCount++] = useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName(); - else - denyAcls[denyCount++] = useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName(); - } - } + if (ace.isAllow()) + allowList.add(useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName()); + else + denyList.add(useSIDs ? ace.getSID().toString() : ace.getSID().getAccountName()); } } - else - { - allowAcls = forcedacls; - if (forcedacls.length == 0) - denyAcls = new String[0]; - else - denyAcls = new String[]{defaultAuthorityDenyToken}; - } - java.util.Arrays.sort(allowAcls); - java.util.Arrays.sort(denyAcls); - packList(description,allowAcls,'+'); - packList(description,denyAcls,'+'); } - else - description.append('-'); - } - + protected static void processSMBException(SmbException se, String documentIdentifier, String activity, String operation) throws ManifoldCFException, ServiceInterruption @@ -1361,97 +1198,26 @@ public class SharedDriveConnector extend } } - protected static int setDocumentSecurity(RepositoryDocument rd, String version, int startPosition) + protected static void setDocumentSecurity(RepositoryDocument rd, + String[] shareAllow, String[] shareDeny, + String[] parentAllow, String[] parentDeny, + String[] allow, String[] deny) { - if (startPosition < version.length() && version.charAt(startPosition++) == '+') - { - // Unpack share allow and share deny - ArrayList shareAllowAcls = new ArrayList(); - startPosition = unpackList(shareAllowAcls,version,startPosition,'+'); - ArrayList shareDenyAcls = new ArrayList(); - startPosition = unpackList(shareDenyAcls,version,startPosition,'+'); - String[] shareAllow = new String[shareAllowAcls.size()]; - String[] shareDeny = new String[shareDenyAcls.size()]; - int i = 0; - while (i < shareAllow.length) - { - shareAllow[i] = (String)shareAllowAcls.get(i); - i++; - } - i = 0; - while (i < shareDeny.length) - { - shareDeny[i] = (String)shareDenyAcls.get(i); - i++; - } - - // set share acls + // set share acls + if (shareAllow.length > 0 || shareDeny.length > 0) rd.setSecurity(RepositoryDocument.SECURITY_TYPE_SHARE,shareAllow,shareDeny); - } - if (startPosition < version.length() && version.charAt(startPosition++) == '+') - { - // Unpack parent allow and deny acls - ArrayList parentAllowAcls = new ArrayList(); - startPosition = unpackList(parentAllowAcls,version,startPosition,'+'); - ArrayList parentDenyAcls = new ArrayList(); - startPosition = unpackList(parentDenyAcls,version,startPosition,'+'); - String[] parentAllow = new String[parentAllowAcls.size()]; - String[] parentDeny = new String[parentDenyAcls.size()]; - int i = 0; - while (i < parentAllow.length) - { - parentAllow[i] = (String)parentAllowAcls.get(i); - i++; - } - i = 0; - while (i < parentDeny.length) - { - parentDeny[i] = (String)parentDenyAcls.get(i); - i++; - } - - // set parent folder acls + // set parent folder acls + if (parentAllow.length > 0 || parentDeny.length > 0) rd.setSecurity(RepositoryDocument.SECURITY_TYPE_PARENT,parentAllow,parentDeny); - } - if (startPosition < version.length() && version.charAt(startPosition++) == '+') - { - // Unpack allow and deny acls - ArrayList allowAcls = new ArrayList(); - startPosition = unpackList(allowAcls,version,startPosition,'+'); - ArrayList denyAcls = new ArrayList(); - startPosition = unpackList(denyAcls,version,startPosition,'+'); - String[] allow = new String[allowAcls.size()]; - String[] deny = new String[denyAcls.size()]; - int i = 0; - while (i < allow.length) - { - allow[i] = (String)allowAcls.get(i); - i++; - } - i = 0; - while (i < deny.length) - { - deny[i] = (String)denyAcls.get(i); - i++; - } - - // set native file acls + // set native file acls + if (allow.length > 0 || deny.length > 0) rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,allow,deny); - } - return startPosition; } - protected static int setPathMetadata(RepositoryDocument rd, String version, int index) + protected static void setPathMetadata(RepositoryDocument rd, String pathAttributeName, String pathAttributeValue) throws ManifoldCFException { - if (version.length() > index && version.charAt(index++) == '+') - { - StringBuilder pathAttributeNameBuffer = new StringBuilder(); - StringBuilder pathAttributeValueBuffer = new StringBuilder(); - index = unpack(pathAttributeNameBuffer,version,index,'+'); - index = unpack(pathAttributeValueBuffer,version,index,'+'); - String pathAttributeName = pathAttributeNameBuffer.toString(); - String pathAttributeValue = pathAttributeValueBuffer.toString(); + if (pathAttributeName != null && pathAttributeValue != null) { if (Logging.connectors.isDebugEnabled()) Logging.connectors.debug("JCIFS: Path attribute name is '"+pathAttributeName+"'"); if (Logging.connectors.isDebugEnabled()) @@ -1460,7 +1226,6 @@ public class SharedDriveConnector extend } else Logging.connectors.debug("JCIFS: Path attribute name is null"); - return index; } /** Check status of connection. @@ -1525,7 +1290,7 @@ public class SharedDriveConnector extend *@param documentSpecification is the specification. *@return true if it should be included. */ - protected boolean checkInclude(SmbFile file, String fileName, DocumentSpecification documentSpecification, IFingerprintActivity activities) + protected boolean checkInclude(SmbFile file, String fileName, Specification documentSpecification, IFingerprintActivity activities) throws ManifoldCFException, ServiceInterruption { if (Logging.connectors.isDebugEnabled()) @@ -1741,7 +1506,7 @@ public class SharedDriveConnector extend * or false otherwise. *@return true if the file would be ingested given the parameters. */ - protected boolean wouldFileBeIncluded(String fileName, DocumentSpecification documentSpecification, + protected boolean wouldFileBeIncluded(String fileName, Specification documentSpecification, boolean pretendIndexable) throws ManifoldCFException { @@ -1876,7 +1641,7 @@ public class SharedDriveConnector extend *@param documentSpecification is the document specification. *@return true if the file needs to be fingerprinted. */ - protected boolean checkNeedFileData(String fileName, DocumentSpecification documentSpecification) + protected boolean checkNeedFileData(String fileName, Specification documentSpecification) throws ManifoldCFException { return wouldFileBeIncluded(fileName,documentSpecification,true) != wouldFileBeIncluded(fileName,documentSpecification,false); @@ -1891,7 +1656,7 @@ public class SharedDriveConnector extend *@param activities are the activities available to determine indexability. *@return true if the file should be ingested. */ - protected boolean checkIngest(File localFile, String fileName, DocumentSpecification documentSpecification, IFingerprintActivity activities) + protected boolean checkIngest(File localFile, String fileName, Specification documentSpecification, IFingerprintActivity activities) throws ManifoldCFException, ServiceInterruption { if (Logging.connectors.isDebugEnabled()) @@ -2132,7 +1897,7 @@ public class SharedDriveConnector extend *@param spec is the document specification. *@return the acls. */ - protected static String[] getForcedAcls(DocumentSpecification spec) + protected static String[] getForcedAcls(Specification spec) { HashMap map = new HashMap(); int i = 0; @@ -2171,7 +1936,7 @@ public class SharedDriveConnector extend *@param spec is the document specification. *@return the acls. */ - protected static String[] getForcedShareAcls(DocumentSpecification spec) + protected static String[] getForcedShareAcls(Specification spec) { HashMap map = new HashMap(); int i = 0; @@ -2209,7 +1974,7 @@ public class SharedDriveConnector extend *@param spec is the document specification. *@return the acls. */ - protected static String[] getForcedParentFolderAcls(DocumentSpecification spec) + protected static String[] getForcedParentFolderAcls(Specification spec) { HashMap map = new HashMap(); int i = 0; @@ -5027,15 +4792,15 @@ public class SharedDriveConnector extend { /** This is the activities object, where matching references will be logged */ - protected IProcessActivity activities; + protected final IProcessActivity activities; /** Document specification */ - protected DocumentSpecification spec; + protected final Specification spec; /** Exceptions that we saw. These are saved here so that they can be rethrown when done */ protected ManifoldCFException lcfException = null; protected ServiceInterruption serviceInterruption = null; /** Constructor */ - public ProcessDocumentsFilter(IProcessActivity activities, DocumentSpecification spec) + public ProcessDocumentsFilter(IProcessActivity activities, Specification spec) { this.activities = activities; this.spec = spec;
Modified: manifoldcf/branches/dev_1x/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java?rev=1626228&r1=1626227&r2=1626228&view=diff ============================================================================== --- manifoldcf/branches/dev_1x/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java (original) +++ manifoldcf/branches/dev_1x/connectors/jdbc/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jdbc/JDBCConnector.java Fri Sep 19 14:22:27 2014 @@ -36,36 +36,7 @@ import javax.sql.*; import java.io.*; import java.util.*; -/** This interface describes an instance of a connection between a repository and ManifoldCF's -* standard "pull" ingestion agent. -* -* Each instance of this interface is used in only one thread at a time. Connection Pooling -* on these kinds of objects is performed by the factory which instantiates repository connectors -* from symbolic names and config parameters, and is pooled by these parameters. That is, a pooled connector -* handle is used only if all the connection parameters for the handle match. -* -* Implementers of this interface should provide a default constructor which has this signature: -* -* xxx(); -* -* Connectors are either configured or not. If configured, they will persist in a pool, and be -* reused multiple times. Certain methods of a connector may be called before the connector is -* configured. This includes basically all methods that permit inspection of the connector's -* capabilities. The complete list is: -* -* -* The purpose of the repository connector is to allow documents to be fetched from the repository. -* -* Each repository connector describes a set of documents that are known only to that connector. -* It therefore establishes a space of document identifiers. Each connector will only ever be -* asked to deal with identifiers that have in some way originated from the connector. -* -* Documents are fetched in three stages. First, the getDocuments() method is called in the connector -* implementation. This returns a set of document identifiers. The document identifiers are used to -* obtain the current document version strings in the second stage, using the getDocumentVersions() method. -* The last stage is processDocuments(), which queues up any additional documents needed, and also ingests. -* This method will not be called if the document version seems to indicate that no document change took -* place. +/** JDBC repository connector. */ public class JDBCConnector extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector { @@ -302,31 +273,27 @@ public class JDBCConnector extends org.a } } - /** Get document versions given an array of document identifiers. - * This method is called for EVERY document that is considered. It is - * therefore important to perform as little work as possible here. - *@param documentIdentifiers is the array of local document identifiers, as understood by this connector. - *@param oldVersions is the corresponding array of version strings that have been saved for the document identifiers. - * A null value indicates that this is a first-time fetch, while an empty string indicates that the previous document - * had an empty version string. - *@param activities is the interface this method should use to perform whatever framework actions are desired. - *@param spec is the current document specification for the current job. If there is a dependency on this - * specification, then the version string should include the pertinent data, so that reingestion will occur - * when the specification changes. This is primarily useful for metadata. + /** Process a set of documents. + * This is the method that should cause each document to be fetched, processed, and the results either added + * to the queue of documents for the current job, and/or entered into the incremental ingestion manager. + * The document specification allows this class to filter what is done based on the job. + * The connector will be connected before this method can be called. + *@param documentIdentifiers is the set of document identifiers to process. + *@param statuses are the currently-stored document versions for each document in the set of document identifiers + * passed in above. + *@param activities is the interface this method should use to queue up new document references + * and ingest documents. *@param jobMode is an integer describing how the job is being run, whether continuous or once-only. *@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one. - *@return the corresponding version strings, with null in the places where the document no longer exists. - * Empty version strings indicate that there is no versioning ability for the corresponding document, and the document - * will always be processed. */ @Override - public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities, - DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority) + public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec, + IProcessActivity activities, int jobMode, boolean usesDefaultAuthority) throws ManifoldCFException, ServiceInterruption { - getSession(); TableSpec ts = new TableSpec(spec); - String[] acls = getAcls(spec); + + String[] acls = ts.getAcls(); // Sort these, java.util.Arrays.sort(acls); @@ -344,125 +311,122 @@ public class JDBCConnector extends org.a { versionsReturned[i++] = ""; } - - return versionsReturned; } - - // If there IS a versions query, do it. First set up the variables, then do the substitution. - VariableMap vm = new VariableMap(); - addConstant(vm,JDBCConstants.idReturnVariable,JDBCConstants.idReturnColumnName); - addConstant(vm,JDBCConstants.versionReturnVariable,JDBCConstants.versionReturnColumnName); - if (!addIDList(vm,JDBCConstants.idListVariable,documentIdentifiers,null)) - return new String[0]; - - // Do the substitution - ArrayList paramList = new ArrayList(); - StringBuilder sb = new StringBuilder(); - substituteQuery(ts.versionQuery,vm,sb,paramList); - - // Now, build a result return, and a hash table so we can correlate the returned values with the place to put them. - // We presume that if the row is missing, the document is gone. - Map map = new HashMap(); - int j = 0; - while (j < documentIdentifiers.length) + else { - map.put(documentIdentifiers[j],new Integer(j)); - versionsReturned[j] = ""; - j++; - } + // If there IS a versions query, do it. First set up the variables, then do the substitution. + VariableMap vm = new VariableMap(); + addConstant(vm,JDBCConstants.idReturnVariable,JDBCConstants.idReturnColumnName); + addConstant(vm,JDBCConstants.versionReturnVariable,JDBCConstants.versionReturnColumnName); + if (addIDList(vm,JDBCConstants.idListVariable,documentIdentifiers,null)) + { + // Do the substitution + ArrayList paramList = new ArrayList(); + StringBuilder sb = new StringBuilder(); + substituteQuery(ts.versionQuery,vm,sb,paramList); + + // Now, build a result return, and a hash table so we can correlate the returned values with the place to put them. + // We presume that if the row is missing, the document is gone. + Map<String,Integer> map = new HashMap<String,Integer>(); + int j = 0; + while (j < documentIdentifiers.length) + { + map.put(documentIdentifiers[j],new Integer(j)); + versionsReturned[j] = ""; + j++; + } - // Fire off the query! - IDynamicResultSet result; - String queryText = sb.toString(); - long startTime = System.currentTimeMillis(); - // Get a dynamic resultset. Contract for dynamic resultset is that if - // one is returned, it MUST be closed, or a connection will leak. - try - { - result = connection.executeUncachedQuery(queryText,paramList,-1); - } - catch (ManifoldCFException e) - { - // If failure, record the failure. - activities.recordActivity(new Long(startTime), ACTIVITY_EXTERNAL_QUERY, null, - createQueryString(queryText,paramList), "ERROR", e.getMessage(), null); - throw e; - } - try - { - // If success, record that too. - activities.recordActivity(new Long(startTime), ACTIVITY_EXTERNAL_QUERY, null, - createQueryString(queryText,paramList), "OK", null, null); - // Now, go through resultset - while (true) - { - IDynamicResultRow row = result.getNextRow(); - if (row == null) - break; + // Fire off the query! + getSession(); + IDynamicResultSet result; + String queryText = sb.toString(); + long startTime = System.currentTimeMillis(); + // Get a dynamic resultset. Contract for dynamic resultset is that if + // one is returned, it MUST be closed, or a connection will leak. try { - Object o = row.getValue(JDBCConstants.idReturnColumnName); - if (o == null) - throw new ManifoldCFException("Bad version query; doesn't return $(IDCOLUMN) column. Try using quotes around $(IDCOLUMN) variable, e.g. \"$(IDCOLUMN)\"."); - String idValue = JDBCConnection.readAsString(o); - o = row.getValue(JDBCConstants.versionReturnColumnName); - String versionValue; - // Null version is OK; make it a "" - if (o == null) - versionValue = ""; - else + result = connection.executeUncachedQuery(queryText,paramList,-1); + } + catch (ManifoldCFException e) + { + // If failure, record the failure. + activities.recordActivity(new Long(startTime), ACTIVITY_EXTERNAL_QUERY, null, + createQueryString(queryText,paramList), "ERROR", e.getMessage(), null); + throw e; + } + try + { + // If success, record that too. + activities.recordActivity(new Long(startTime), ACTIVITY_EXTERNAL_QUERY, null, + createQueryString(queryText,paramList), "OK", null, null); + // Now, go through resultset + while (true) { - // A real version string! Any acls must be added to the front, if they are present... - sb = new StringBuilder(); - packList(sb,acls,'+'); - if (acls.length > 0) + IDynamicResultRow row = result.getNextRow(); + if (row == null) + break; + try { - sb.append('+'); - pack(sb,defaultAuthorityDenyToken,'+'); - } - else - sb.append('-'); + Object o = row.getValue(JDBCConstants.idReturnColumnName); + if (o == null) + throw new ManifoldCFException("Bad version query; doesn't return $(IDCOLUMN) column. Try using quotes around $(IDCOLUMN) variable, e.g. \"$(IDCOLUMN)\"."); + String idValue = JDBCConnection.readAsString(o); + o = row.getValue(JDBCConstants.versionReturnColumnName); + String versionValue; + // Null version is OK; make it a "" + if (o == null) + versionValue = ""; + else + { + // A real version string! Any acls must be added to the front, if they are present... + sb = new StringBuilder(); + packList(sb,acls,'+'); + if (acls.length > 0) + { + sb.append('+'); + pack(sb,defaultAuthorityDenyToken,'+'); + } + else + sb.append('-'); - sb.append(JDBCConnection.readAsString(o)).append("=").append(ts.dataQuery); - versionValue = sb.toString(); + sb.append(JDBCConnection.readAsString(o)).append("=").append(ts.dataQuery); + versionValue = sb.toString(); + } + // Versions that are "", when processed, will have their acls fetched at that time... + versionsReturned[map.get(idValue).intValue()] = versionValue; + } + finally + { + row.close(); + } } - // Versions that are "", when processed, will have their acls fetched at that time... - versionsReturned[((Integer)map.get(idValue)).intValue()] = versionValue; } finally { - row.close(); + result.close(); } } } - finally - { - result.close(); + + // Delete the documents that had no version, and work only on ones that did + Set<String> fetchDocuments = new HashSet<String>(); + Map<String,String> map = new HashMap<String,String>(); + for (int i = 0; i < documentIdentifiers.length; i++) + { + String documentIdentifier = documentIdentifiers[i]; + String versionValue = versionsReturned[i]; + if (versionValue == null) + { + activities.deleteDocument(documentIdentifier); + continue; + } + if (versionValue.length() == 0 || activities.checkDocumentNeedsReindexing(documentIdentifier,versionValue)) + { + fetchDocuments.add(documentIdentifier); + map.put(documentIdentifier,versionValue); + } } - - return versionsReturned; - } - - /** Process a set of documents. - * This is the method that should cause each document to be fetched, processed, and the results either added - * to the queue of documents for the current job, and/or entered into the incremental ingestion manager. - * The document specification allows this class to filter what is done based on the job. - *@param documentIdentifiers is the set of document identifiers to process. - *@param versions is the corresponding document versions to process, as returned by getDocumentVersions() above. - * The implementation may choose to ignore this parameter and always process the current version. - *@param activities is the interface this method should use to queue up new document references - * and ingest documents. - *@param spec is the document specification. - *@param scanOnly is an array corresponding to the document identifiers. It is set to true to indicate when the processing - * should only find other references, and should not actually call the ingestion methods. - */ - @Override - public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities, DocumentSpecification spec, boolean[] scanOnly) - throws ManifoldCFException, ServiceInterruption - { - getSession(); - TableSpec ts = new TableSpec(spec); - + // For all the documents not marked "scan only", form a query and pick up the contents. // If the contents is not found, then explicitly call the delete action method. VariableMap vm = new VariableMap(); @@ -470,7 +434,7 @@ public class JDBCConnector extends org.a addConstant(vm,JDBCConstants.urlReturnVariable,JDBCConstants.urlReturnColumnName); addConstant(vm,JDBCConstants.dataReturnVariable,JDBCConstants.dataReturnColumnName); addConstant(vm,JDBCConstants.contentTypeReturnVariable,JDBCConstants.contentTypeReturnColumnName); - if (!addIDList(vm,JDBCConstants.idListVariable,documentIdentifiers,scanOnly)) + if (!addIDList(vm,JDBCConstants.idListVariable,documentIdentifiers,fetchDocuments)) return; // Do the substitution @@ -478,23 +442,8 @@ public class JDBCConnector extends org.a StringBuilder sb = new StringBuilder(); substituteQuery(ts.dataQuery,vm,sb,paramList); - int i; - - // Build a map of versions we are allowed to ingest - Map map = new HashMap(); - i = 0; - while (i < documentIdentifiers.length) - { - if (!scanOnly[i]) - { - // Version strings at this point should never be null; the CF interprets nulls as - // meaning that delete must occur. Empty strings are possible though. - map.put(documentIdentifiers[i],versions[i]); - } - i++; - } - // Execute the query + getSession(); IDynamicResultSet result; String queryText = sb.toString(); long startTime = System.currentTimeMillis(); @@ -528,7 +477,7 @@ public class JDBCConnector extends org.a if (o == null) throw new ManifoldCFException("Bad document query; doesn't return $(IDCOLUMN) column. Try using quotes around $(IDCOLUMN) variable, e.g. \"$(IDCOLUMN)\"."); String id = JDBCConnection.readAsString(o); - String version = (String)map.get(id); + String version = map.get(id); if (version != null) { // This document was marked as "not scan only", so we expect to find it. @@ -581,7 +530,7 @@ public class JDBCConnector extends org.a else rd.setMimeType(contentType); - applyAccessTokens(rd,version,spec); + applyAccessTokens(rd,ts); applyMetadata(rd,row); BinaryInput bi = (BinaryInput)contents; @@ -623,7 +572,7 @@ public class JDBCConnector extends org.a else rd.setMimeType(contentType); - applyAccessTokens(rd,version,spec); + applyAccessTokens(rd,ts); applyMetadata(rd,row); CharacterInput ci = (CharacterInput)contents; @@ -669,7 +618,7 @@ public class JDBCConnector extends org.a else rd.setMimeType(contentType); - applyAccessTokens(rd,version,spec); + applyAccessTokens(rd,ts); applyMetadata(rd,row); InputStream is = new ByteArrayInputStream(bytes); @@ -694,16 +643,28 @@ public class JDBCConnector extends org.a } } else + { Logging.connectors.warn("JDBC: Document '"+id+"' excluded because of mime type - skipping"); + activities.noDocument(id,version); + } } else + { Logging.connectors.warn("JDBC: Document '"+id+"' seems to have null data - skipping"); + activities.noDocument(id,version); + } } else + { Logging.connectors.warn("JDBC: Document '"+id+"' has an illegal url: '"+url+"' - skipping"); + activities.noDocument(id,version); + } } else + { Logging.connectors.warn("JDBC: Document '"+id+"' has a null url - skipping"); + activities.noDocument(id,version); + } } } finally @@ -711,28 +672,28 @@ public class JDBCConnector extends org.a row.close(); } } - // Now, go through the original id's, and see which ones are still in the map. These - // did not appear in the result and are presumed to be gone from the database, and thus must be deleted. - i = 0; - while (i < documentIdentifiers.length) - { - if (!scanOnly[i]) - { - String documentIdentifier = documentIdentifiers[i]; - if (map.get(documentIdentifier) != null) - { - // This means we did not see it (or data for it) in the result set. Delete it! - activities.deleteDocument(documentIdentifier); - } - } - i++; - } } finally { result.close(); } + + // Now, go through the original id's, and see which ones are still in the map. These + // did not appear in the result and are presumed to be gone from the database, and thus must be deleted. + for (String documentIdentifier : documentIdentifiers) + { + if (fetchDocuments.contains(documentIdentifier)) + { + String documentVersion = map.get(documentIdentifier); + if (documentVersion != null) + { + // This means we did not see it (or data for it) in the result set. Delete it! + activities.noDocument(documentIdentifier,documentVersion); + } + } + } + } // UI support methods. @@ -1552,48 +1513,16 @@ public class JDBCConnector extends org.a *@param version is the version string. *@param spec is the document specification. */ - protected void applyAccessTokens(RepositoryDocument rd, String version, DocumentSpecification spec) + protected void applyAccessTokens(RepositoryDocument rd, TableSpec ts) throws ManifoldCFException { - // Set up any acls - String[] accessAcls = null; - String[] denyAcls = null; - - if (version.length() == 0) - { - // Version is empty string, therefore acl information must be gathered from spec - String[] specAcls = getAcls(spec); - accessAcls = specAcls; - if (specAcls.length != 0) - denyAcls = new String[]{defaultAuthorityDenyToken}; - else - denyAcls = new String[0]; - } + String[] accessAcls = ts.getAcls(); + String[] denyAcls; + if (accessAcls.length == 0) + denyAcls = new String[0]; else - { - // Unpack access tokens and the deny token too - ArrayList acls = new ArrayList(); - StringBuilder denyAclBuffer = new StringBuilder(); - int startPos = unpackList(acls,version,0,'+'); - if (startPos < version.length() && version.charAt(startPos++) == '+') - { - startPos = unpack(denyAclBuffer,version,startPos,'+'); - } - // Turn into acls and add into description - accessAcls = new String[acls.size()]; - int j = 0; - while (j < accessAcls.length) - { - accessAcls[j] = (String)acls.get(j); - j++; - } - // Deny acl too - if (denyAclBuffer.length() > 0) - { - denyAcls = new String[]{denyAclBuffer.toString()}; - } - } - + denyAcls = new String[]{defaultAuthorityDenyToken}; + rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,accessAcls,denyAcls); } @@ -1638,24 +1567,21 @@ public class JDBCConnector extends org.a /** Build an idlist variable, and add it to the specified variable map. */ - protected static boolean addIDList(VariableMap map, String varName, String[] documentIdentifiers, boolean[] scanOnly) + protected static boolean addIDList(VariableMap map, String varName, String[] documentIdentifiers, Set<String> fetchDocuments) { ArrayList params = new ArrayList(); StringBuilder sb = new StringBuilder(" ("); - int i = 0; int k = 0; - while (i < documentIdentifiers.length) + for (String documentIdentifier : documentIdentifiers) { - if (scanOnly == null || !scanOnly[i]) + if (fetchDocuments == null || fetchDocuments.contains(documentIdentifier)) { if (k > 0) sb.append(","); - String documentIdentifier = documentIdentifiers[i]; sb.append("?"); params.add(documentIdentifier); k++; } - i++; } sb.append(") "); map.addVariable(varName,sb.toString(),params); @@ -1835,16 +1761,19 @@ public class JDBCConnector extends org.a */ protected static class TableSpec { - public String idQuery; - public String versionQuery; - public String dataQuery; - - public TableSpec(DocumentSpecification ds) - { - int i = 0; - while (i < ds.getChildCount()) + public final String idQuery; + public final String versionQuery; + public final String dataQuery; + public final Set<String> aclMap = new HashSet<String>(); + + public TableSpec(Specification ds) + { + String idQuery = null; + String versionQuery = null; + String dataQuery = null; + for (int i = 0; i < ds.getChildCount(); i++) { - SpecificationNode sn = ds.getChild(i++); + SpecificationNode sn = ds.getChild(i); if (sn.getType().equals(JDBCConstants.idQueryNode)) { idQuery = sn.getValue(); @@ -1863,8 +1792,26 @@ public class JDBCConnector extends org.a if (dataQuery == null) dataQuery = ""; } + else if (sn.getType().equals("access")) + { + String token = sn.getAttributeValue("token"); + aclMap.add(token); + } } + this.idQuery = idQuery; + this.versionQuery = versionQuery; + this.dataQuery = dataQuery; + } + public String[] getAcls() + { + String[] rval = new String[aclMap.size()]; + int i = 0; + for (String token : aclMap) + { + rval[i++] = token; + } + return rval; } }
