Author: kwright
Date: Wed Sep 5 16:15:33 2012
New Revision: 1381232
URL: http://svn.apache.org/viewvc?rev=1381232&view=rev
Log:
Merge in changes for CONNECTORS-492.
Modified:
manifoldcf/trunk/ (props changed)
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl
Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-492:r1368611-1381230
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1381232&r1=1381231&r2=1381232&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Sep 5 16:15:33 2012
@@ -5,6 +5,10 @@ $Id$
CONNECTORS-521: Fix project site's broken search
(Ahmet Arslan)
+CONNECTORS-492: Replace Lists.GetListItems call with a custom call,
+in SharePoint connector, to dodge limits on size of response.
+(Ahmet Arslan, Piergiorgio Lucidi, Joe Becknell, Karl Wright)
+
CONNECTORS-522: Wiki connector namespace tab was non-functional.
(Maciej Lizewski, Karl Wright)
Modified:
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java?rev=1381232&r1=1381231&r2=1381232&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
(original)
+++
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SPSProxyHelper.java
Wed Sep 5 16:15:33 2012
@@ -296,7 +296,7 @@ public class SPSProxyHelper {
/**
* Get the acls for a document.
- * NOTE that this function only works for SharePoint 3.0 with the
MCPermissions web service installed.
+ * NOTE that this function only works for SharePoint 2007+ with the
MCPermissions web service installed.
* @param site is the encoded subsite path
* @param file is the encoded file url (not including protocol or server or
location, but including encoded subsite, library and folder/file path)
* @return array of document SIDs
@@ -607,98 +607,69 @@ public class SPSProxyHelper {
}
else
{
- // Sharepoint 2010; use Lists service instead
- ListsWS lservice = new ListsWS(baseUrl + site, userName, password,
myFactory, configuration, connectionManager );
- ListsSoapStub stub1 = (ListsSoapStub)lservice.getListsSoapHandler();
+ // New code
+
+ MCPermissionsWS itemService = new MCPermissionsWS( baseUrl + site,
userName, password, myFactory, configuration, connectionManager );
+ com.microsoft.sharepoint.webpartpages.PermissionsSoap itemCall =
itemService.getPermissionsSoapHandler( );
- // This string is the paging chunk description. It gets updated on
every chunk we do,
- // so that the next call finds new data.
- String nextChunkDescription = "";
- // Order by some column we know is indexed.
- GetListItemsQuery orderByQuery = buildOrderedQuery("ID");
- // Set up fields we want
- ArrayList fieldList = new ArrayList();
- fieldList.add("FileRef");
- GetListItemsViewFields viewFields = buildViewFields(fieldList);
- // Pick a request size we know will not exceed the limit as set by the
administrator.
- int requestSize = 2000;
-
+ int startingIndex = 0;
+ int amtToRequest = 1000;
while (true)
{
- GetListItemsResponseGetListItemsResult items =
stub1.getListItems(guid, "", orderByQuery, viewFields,
Integer.toString(requestSize), buildPagingQueryOptions(nextChunkDescription),
null);
- if (items == null)
- return false;
-
- org.apache.axis.message.MessageElement[] list = items.get_any();
+
com.microsoft.sharepoint.webpartpages.GetListItemsResponseGetListItemsResult
itemsResult =
+
itemCall.getListItems(guid,Integer.toString(startingIndex),Integer.toString(amtToRequest));
+
+ MessageElement[] itemsList = itemsResult.get_any();
if (Logging.connectors.isDebugEnabled()){
- Logging.connectors.debug("SharePoint: getListItems xml response:
'" + list[0].toString() + "'");
+ Logging.connectors.debug("SharePoint: getListItems xml response:
'" + itemsList[0].toString() + "'");
}
- ArrayList nodeList = new ArrayList();
- XMLDoc doc = new XMLDoc(list[0].toString());
-
- doc.processPath(nodeList, "*", null);
- if (nodeList.size() != 1) {
- throw new ManifoldCFException("Bad xml - expecting one outer
'ns1:listitems' node - there are " + Integer.toString(nodeList.size()) + "
nodes");
- }
-
- Object parent = nodeList.get(0);
- if (!"ns1:listitems".equals(doc.getNodeName(parent)))
- throw new ManifoldCFException("Bad xml - outer node is not
'ns1:listitems'");
-
-
- nodeList.clear();
- doc.processPath(nodeList, "*", parent);
-
- if (nodeList.size() != 1)
- throw new ManifoldCFException("Expected rsdata result but no
results found.");
-
- Object rsData = nodeList.get(0);
-
- // Get the chunk description
- nextChunkDescription = doc.getValue(rsData,
"ListItemCollectionPositionNext");
-
- int itemCount = Integer.parseInt(doc.getValue(rsData, "ItemCount"));
-
- // Now, extract the files from the response document
- XMLDoc docs = doc;
- ArrayList nodeDocs = new ArrayList();
-
- docs.processPath(nodeDocs, "*", rsData);
-
- if (nodeDocs.size() != itemCount) {
- throw new ManifoldCFException("itemCount does not match with
nodeDocs.size().");
- }
-
- for (int j = 0; j < nodeDocs.size(); j++)
+ if (itemsList.length != 1)
+ throw new ManifoldCFException("Bad response - expecting one outer
'GetListItems' node, saw "+Integer.toString(itemsList.length));
+
+ MessageElement items = itemsList[0];
+ if (!items.getElementName().getLocalName().equals("GetListItems"))
+ throw new ManifoldCFException("Bad response - outer node should
have been 'GetListItems' node");
+
+ int resultCount = 0;
+ Iterator iter = items.getChildElements();
+ while (iter.hasNext())
{
-
- Object node = nodeDocs.get(j);
-
- String relPath = doc.getValue(node, "ows_FileRef");
-
- // This relative path is apparently from the domain on down; if
there's a location offset we therefore
- // need to get rid of it before checking the document against the
site/library tuples. The recorded
- // document identifier should also not include it.
-
- // KDW: Removed the case changes; URL characters should remain
case-sensitive
- if (!relPath.startsWith(serverLocation))
+ MessageElement child = (MessageElement)iter.next();
+ if
(child.getElementName().getLocalName().equals("GetListItemsResponse"))
{
- // Unexpected processing error; the path to the folder or
document did not start with the location
- // offset, so throw up.
- throw new ManifoldCFException("Internal error: Relative path
'"+relPath+"' was expected to start with '"+
- serverLocation+"'");
+ Iterator resultIter = child.getChildElements();
+ while (resultIter.hasNext())
+ {
+ MessageElement result = (MessageElement)resultIter.next();
+ if
(result.getElementName().getLocalName().equals("GetListItemsResult"))
+ {
+ resultCount++;
+ String relPath = result.getAttribute("FileRef");
+ // This includes the document library, so munge the path
accordingly
+ if (!relPath.startsWith(serverLocation))
+ {
+ // Unexpected processing error; the path to the folder or
document did not start with the location
+ // offset, so throw up.
+ throw new ManifoldCFException("Internal error: Relative
path '"+relPath+"' was expected to start with '"+
+ serverLocation+"'");
+ }
+ relPath = relPath.substring(serverLocation.length());
+
+ relPath = "/" + relPath;
+
+ fileStream.addFile( relPath );
+ }
+ }
+
}
- relPath = relPath.substring(serverLocation.length());
-
- relPath = "/" + valueMunge(relPath);
-
- fileStream.addFile( relPath );
}
- if (requestSize > nodeDocs.size())
+ if (resultCount < amtToRequest)
break;
+
+ startingIndex += resultCount;
}
}
Modified: manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl?rev=1381232&r1=1381231&r2=1381232&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl (original)
+++ manifoldcf/trunk/connectors/sharepoint/wsdls/MCPermissions.wsdl Wed Sep 5
16:15:33 2012
@@ -1,18 +1,4 @@
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
<wsdl:definitions xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/";
xmlns:tm="http://microsoft.com/wsdl/mime/textMatching/";
xmlns:soapenc="http://schemas.xmlsoap.org/soap/encoding/";
xmlns:mime="http://schemas.xmlsoap.org/wsdl/mime/";
xmlns:tns="http://microsoft.com/sharepoint/webpartpages/";
xmlns:s="http://www.w3.org/2001/XMLSchema";
xmlns:soap12="http://schemas.xmlsoap.org/wsdl/soap12/";
xmlns:http="http://schemas.xmlsoap.org/wsdl/http/";
targetNamespace="http://microsoft.com/sharepoint/webpartpages/";
xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/";>
<wsdl:types>
<s:schema elementFormDefault="qualified"
targetNamespace="http://microsoft.com/sharepoint/webpartpages/";>
@@ -22,7 +8,6 @@
<s:element minOccurs="0" maxOccurs="1" name="objectName"
type="s:string" />
<s:element minOccurs="0" maxOccurs="1" name="objectType"
type="s:string" />
</s:sequence>
-
</s:complexType>
</s:element>
<s:element name="GetPermissionCollectionResponse">
@@ -32,7 +17,28 @@
<s:complexType mixed="true">
<s:sequence>
<s:any />
-
+ </s:sequence>
+ </s:complexType>
+ </s:element>
+ </s:sequence>
+ </s:complexType>
+ </s:element>
+ <s:element name="GetListItems">
+ <s:complexType>
+ <s:sequence>
+ <s:element minOccurs="0" maxOccurs="1" name="listName"
type="s:string" />
+ <s:element minOccurs="0" maxOccurs="1" name="startRow"
type="s:string" />
+ <s:element minOccurs="0" maxOccurs="1" name="rowLimit"
type="s:string" />
+ </s:sequence>
+ </s:complexType>
+ </s:element>
+ <s:element name="GetListItemsResponse">
+ <s:complexType>
+ <s:sequence>
+ <s:element minOccurs="0" maxOccurs="1" name="GetListItemsResult">
+ <s:complexType mixed="true">
+ <s:sequence>
+ <s:any />
</s:sequence>
</s:complexType>
</s:element>
@@ -42,26 +48,43 @@
</s:schema>
</wsdl:types>
<wsdl:message name="GetPermissionCollectionSoapIn">
-
<wsdl:part name="parameters" element="tns:GetPermissionCollection" />
</wsdl:message>
<wsdl:message name="GetPermissionCollectionSoapOut">
<wsdl:part name="parameters" element="tns:GetPermissionCollectionResponse"
/>
</wsdl:message>
+ <wsdl:message name="GetListItemsSoapIn">
+ <wsdl:part name="parameters" element="tns:GetListItems" />
+ </wsdl:message>
+ <wsdl:message name="GetListItemsSoapOut">
+ <wsdl:part name="parameters" element="tns:GetListItemsResponse" />
+ </wsdl:message>
<wsdl:portType name="PermissionsSoap">
<wsdl:operation name="GetPermissionCollection">
<wsdl:documentation
xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/";>Returns the collection of
permissions for a site, list, or list item.</wsdl:documentation>
-
<wsdl:input message="tns:GetPermissionCollectionSoapIn" />
<wsdl:output message="tns:GetPermissionCollectionSoapOut" />
</wsdl:operation>
+ <wsdl:operation name="GetListItems">
+ <wsdl:documentation
xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/";>Returns the list of contents of a
library without interference from administrator-set limits.</wsdl:documentation>
+ <wsdl:input message="tns:GetListItemsSoapIn" />
+ <wsdl:output message="tns:GetListItemsSoapOut" />
+ </wsdl:operation>
</wsdl:portType>
<wsdl:binding name="PermissionsSoap" type="tns:PermissionsSoap">
<soap:binding transport="http://schemas.xmlsoap.org/soap/http"; />
<wsdl:operation name="GetPermissionCollection">
<soap:operation
soapAction="http://microsoft.com/sharepoint/webpartpages/GetPermissionCollection";
style="document" />
<wsdl:input>
-
+ <soap:body use="literal" />
+ </wsdl:input>
+ <wsdl:output>
+ <soap:body use="literal" />
+ </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="GetListItems">
+ <soap:operation
soapAction="http://microsoft.com/sharepoint/webpartpages/GetListItems";
style="document" />
+ <wsdl:input>
<soap:body use="literal" />
</wsdl:input>
<wsdl:output>
@@ -71,7 +94,6 @@
</wsdl:binding>
<wsdl:binding name="PermissionsSoap12" type="tns:PermissionsSoap">
<soap12:binding transport="http://schemas.xmlsoap.org/soap/http"; />
-
<wsdl:operation name="GetPermissionCollection">
<soap12:operation
soapAction="http://microsoft.com/sharepoint/webpartpages/GetPermissionCollection";
style="document" />
<wsdl:input>
@@ -81,15 +103,22 @@
<soap12:body use="literal" />
</wsdl:output>
</wsdl:operation>
-
+ <wsdl:operation name="GetListItems">
+ <soap12:operation
soapAction="http://microsoft.com/sharepoint/webpartpages/GetListItems";
style="document" />
+ <wsdl:input>
+ <soap12:body use="literal" />
+ </wsdl:input>
+ <wsdl:output>
+ <soap12:body use="literal" />
+ </wsdl:output>
+ </wsdl:operation>
</wsdl:binding>
<wsdl:service name="Permissions">
<wsdl:port name="PermissionsSoap" binding="tns:PermissionsSoap">
- <soap:address
location="http://w2k3-shp-76-2.qa-ad-76.metacarta.com/_vti_bin/MCPermissions.asmx";
/>
+ <soap:address
location="http://win-0hb0c25kl3n:33445/_vti_bin/MCPermissions.asmx"; />
</wsdl:port>
<wsdl:port name="PermissionsSoap12" binding="tns:PermissionsSoap12">
- <soap12:address
location="http://w2k3-shp-76-2.qa-ad-76.metacarta.com/_vti_bin/MCPermissions.asmx";
/>
+ <soap12:address
location="http://win-0hb0c25kl3n:33445/_vti_bin/MCPermissions.asmx"; />
</wsdl:port>
</wsdl:service>
-
-</wsdl:definitions>
+</wsdl:definitions>
\ No newline at end of file
