Author: kwright
Date: Wed Jun 6 14:16:49 2012
New Revision: 1346903
URL: http://svn.apache.org/viewvc?rev=1346903&view=rev
Log:
Refactor to allow better modularity talking to services
Added:
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java
(with props)
Modified:
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
Added:
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java
URL:
http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java?rev=1346903&view=auto
==============================================================================
---
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java
(added)
+++
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java
Wed Jun 6 14:16:49 2012
@@ -0,0 +1,25 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.sharepoint;
+
+import org.apache.manifoldcf.core.interfaces.*;
+
+public interface IFileStream
+{
+ public void addFile(String relPath)
+ throws ManifoldCFException;
+}
Propchange:
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/IFileStream.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
URL:
http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java?rev=1346903&r1=1346902&r2=1346903&view=diff
==============================================================================
---
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
(original)
+++
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
Wed Jun 6 14:16:49 2012
@@ -505,7 +505,7 @@ public class SPSProxyHelper {
* @throws ManifoldCFException
* @throws ServiceInterruption
*/
- public XMLDoc getDocuments(String site, String docLibrary)
+ public boolean getDocuments(IFileStream fileStream, String site, String
docLibrary, boolean dspStsWorks )
throws ManifoldCFException, ServiceInterruption
{
long currentTime;
@@ -565,7 +565,44 @@ public class SPSProxyHelper {
throw new ManifoldCFException( " No results found." );
}
- return doc;
+ // Now, extract the files from the response document
+ XMLDoc docs = doc;
+ ArrayList nodeDocs = new ArrayList();
+
+ docs.processPath( nodeDocs, "*", null );
+ parent = nodeDocs.get(0); // ns1:dsQueryResponse
+ nodeDocs.clear();
+ docs.processPath(nodeDocs, "*", parent);
+ Object documents = nodeDocs.get(1);
+ nodeDocs.clear();
+ docs.processPath(nodeDocs, "*", documents);
+
+ StringBuilder sb = new StringBuilder();
+ for( int j =0; j < nodeDocs.size(); j++)
+ {
+ Object node = nodeDocs.get(j);
+ Logging.connectors.debug( node.toString() );
+ String relPath = docs.getData( docs.getElement( node, "FileRef" ) );
+
+ // This relative path is apparently from the domain on down; if
there's a location offset we therefore
+ // need to get rid of it before checking the document against the
site/library tuples. The recorded
+ // document identifier should also not include it.
+
+ if (!relPath.toLowerCase().startsWith(serverLocation.toLowerCase()))
+ {
+ // Unexpected processing error; the path to the folder or document
did not start with the location
+ // offset, so throw up.
+ throw new ManifoldCFException("Internal error: Relative path
'"+relPath+"' was expected to start with '"+
+ serverLocation+"'");
+ }
+
+ relPath = relPath.substring(serverLocation.length());
+
+ if ( !relPath.endsWith(".aspx") )
+ fileStream.addFile( relPath );
+ }
+
+ return true;
}
catch (java.net.MalformedURLException e)
{
@@ -594,14 +631,14 @@ public class SPSProxyHelper {
// Page did not exist
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: The page at
"+baseUrl+site+" did not exist; assuming library deleted");
- return null;
+ return false;
}
else if (httpErrorCode.equals("401"))
{
// User did not have permissions for this library to get the acls
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: The crawl user did not
have access to list documents for "+baseUrl+site+"; skipping documents within");
- return null;
+ return false;
}
else if (httpErrorCode.equals("403"))
throw new ManifoldCFException("Http error "+httpErrorCode+" while
reading from "+baseUrl+site+" - check IIS and SharePoint security settings!
"+e.getMessage(),e);
@@ -622,7 +659,7 @@ public class SPSProxyHelper {
// List did not exist
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("SharePoint: The list "+docLibrary+" in
site "+site+" did not exist; assuming library deleted");
- return null;
+ return false;
}
else
{
@@ -635,7 +672,7 @@ public class SPSProxyHelper {
Logging.connectors.debug("SharePoint: Getting child documents
for the list "+docLibrary+" in site "+site+" failed with unexpected SharePoint
error code "+sharepointErrorCode+": "+errorString+" - Skipping",e);
}
- return null;
+ return false;
}
}
if (Logging.connectors.isDebugEnabled())
Modified:
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
URL:
http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1346903&r1=1346902&r2=1346903&view=diff
==============================================================================
---
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
(original)
+++
incubator/lcf/branches/CONNECTORS-470/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
Wed Jun 6 14:16:49 2012
@@ -61,6 +61,7 @@ public class SharePointRepository extend
public final static String ACTIVITY_FETCH = "fetch";
private boolean supportsItemSecurity = false;
+ private boolean dspStsWorks = true;
private String serverProtocol = null;
private String serverUrl = null;
private String fileBaseUrl = null;
@@ -117,7 +118,8 @@ public class SharePointRepository extend
String serverVersion = params.getParameter( "serverVersion" );
if (serverVersion == null)
serverVersion = "2.0";
- supportsItemSecurity = serverVersion.equals("3.0");
+ supportsItemSecurity = !serverVersion.equals("2.0");
+ dspStsWorks = !serverVersion.equals("4.0");
serverProtocol = params.getParameter( "serverProtocol" );
if (serverProtocol == null)
@@ -903,49 +905,9 @@ public class SharePointRepository extend
String libID = proxy.getDocLibID( encodePath(site), site, libName );
if (libID != null)
{
- XMLDoc docs = proxy.getDocuments( encodePath(site) , libID );
- if (docs != null)
- {
- ArrayList nodeDocs = new ArrayList();
-
- docs.processPath( nodeDocs, "*", null );
- Object parent = nodeDocs.get(0); //
ns1:dsQueryResponse
- nodeDocs.clear();
- docs.processPath(nodeDocs, "*", parent);
- Object documents = nodeDocs.get(1);
- nodeDocs.clear();
- docs.processPath(nodeDocs, "*", documents);
-
- StringBuilder sb = new StringBuilder();
- for( int j =0; j < nodeDocs.size(); j++)
- {
- Object node = nodeDocs.get(j);
- Logging.connectors.debug( node.toString() );
- String relPath = docs.getData( docs.getElement( node,
"FileRef" ) );
-
- // This relative path is apparently from the domain on down;
if there's a location offset we therefore
- // need to get rid of it before checking the document against
the site/library tuples. The recorded
- // document identifier should also not include it.
-
- if
(!relPath.toLowerCase().startsWith(serverLocation.toLowerCase()))
- {
- // Unexpected processing error; the path to the folder or
document did not start with the location
- // offset, so throw up.
- throw new ManifoldCFException("Internal error: Relative path
'"+relPath+"' was expected to start with '"+
- serverLocation+"'");
- }
-
- relPath = relPath.substring(serverLocation.length());
-
- // Since the processing for a file needs to know the library
path, we need a way to signal the cutoff between library and folder levels.
- // The way I've chosen to do this is to use a double slash at
that point, as a separator.
- String modifiedPath =
relPath.substring(0,foldersFilePathIndex) + "/" +
relPath.substring(foldersFilePathIndex);
-
- if ( !relPath.endsWith(".aspx") && checkInclude( relPath, spec
) )
- activities.addDocumentReference( modifiedPath );
- }
- }
- else
+ FileStream fs = new FileStream( activities, foldersFilePathIndex,
spec );
+ boolean success = proxy.getDocuments( fs, encodePath(site) ,
libID, dspStsWorks );
+ if (!success)
{
// Site/library no longer exists, so delete entry
if (Logging.connectors.isDebugEnabled())
@@ -1347,6 +1309,34 @@ public class SharePointRepository extend
}
}
+ protected class FileStream implements IFileStream
+ {
+ protected IProcessActivity activities;
+ protected int foldersFilePathIndex;
+ protected DocumentSpecification spec;
+
+ public FileStream(IProcessActivity activities, int foldersFilePathIndex,
DocumentSpecification spec)
+ {
+ this.activities = activities;
+ this.foldersFilePathIndex = foldersFilePathIndex;
+ this.spec = spec;
+ }
+
+ public void addFile(String relPath)
+ throws ManifoldCFException
+ {
+ if ( checkInclude( relPath, spec ) )
+ {
+ // Since the processing for a file needs to know the library path, we
need a way to signal the cutoff between library and folder levels.
+ // The way I've chosen to do this is to use a double slash at that
point, as a separator.
+ String modifiedPath = relPath.substring(0,foldersFilePathIndex) + "/"
+ relPath.substring(foldersFilePathIndex);
+
+ activities.addDocumentReference( modifiedPath );
+ }
+ }
+ }
+
+
// UI support methods.
//
// These support methods come in two varieties. The first bunch is involved
in setting up connection configuration information. The second bunch
@@ -1548,8 +1538,9 @@ public class SharePointRepository extend
" <td class=\"description\"><nobr>" +
Messages.getBodyString(locale,"SharePointRepository.ServerSharePointVersion") +
"</nobr></td>\n"+
" <td class=\"value\">\n"+
" <select name=\"serverVersion\">\n"+
-" <option value=\"2.0\"
"+((serverVersion.equals("2.0"))?"selected=\"true\"":"")+">SharePoint Services
2.0</option>\n"+
-" <option value=\"3.0\"
"+(serverVersion.equals("3.0")?"selected=\"true\"":"")+">SharePoint Services
3.0</option>\n"+
+" <option value=\"2.0\"
"+((serverVersion.equals("2.0"))?"selected=\"true\"":"")+">SharePoint Services
2.0 (2003)</option>\n"+
+" <option value=\"3.0\"
"+(serverVersion.equals("3.0")?"selected=\"true\"":"")+">SharePoint Services
3.0 (2007)</option>\n"+
+" <option value=\"4.0\"
"+(serverVersion.equals("4.0")?"selected=\"true\"":"")+">SharePoint Services
4.0 (2010)</option>\n"+
" </select>\n"+
" </td>\n"+
" </tr>\n"+
