Author: kwright
Date: Tue Jan 15 11:01:58 2013
New Revision: 1433350
URL: http://svn.apache.org/viewvc?rev=1433350&view=rev
Log:
Fix for CONNECTORS-612.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Tue Jan 15 11:01:58 2013
@@ -3,6 +3,9 @@ $Id$
======================= 1.1-dev =====================
+CONNECTORS-612: Disable HttpClient retries everywhere.
+(Karl Wright)
+
CONNECTORS-611: Handle inability to get permissions for funky docs
by skipping those docs, rather than throwing an NPE.
(Ahmet Arslan, Karl Wright)
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Tue Jan 15 11:01:58 2013
@@ -30,6 +30,8 @@ import org.apache.http.conn.ClientConnec
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
import org.apache.commons.io.FilenameUtils;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
@@ -104,6 +106,18 @@ public class ElasticSearchConnector exte
localConnectionManager.setMaxTotal(1);
connectionManager = localConnectionManager;
DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+ // No retries
+ localClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
client = localClient;
}
Modified:
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
(original)
+++
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Tue Jan 15 11:01:58 2013
@@ -64,6 +64,8 @@ import org.apache.http.client.params.Htt
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.RedirectException;
@@ -420,10 +422,23 @@ public class LivelinkConnector extends o
params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
- params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+ params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,300000);
params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS,true);
DefaultHttpClient localHttpClient = new
DefaultHttpClient(connectionManager,params);
+ // No retries
+ localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+
localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
// Set up authentication to use
if (ntlmDomain != null)
Modified:
manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
(original)
+++
manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
Tue Jan 15 11:01:58 2013
@@ -87,6 +87,8 @@ import org.apache.http.params.HttpParams
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.NTCredentials;
import org.apache.http.HttpHost;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
public class MeridioWrapper
@@ -259,9 +261,22 @@ public class MeridioWrapper
BasicHttpParams dmwsParams = new BasicHttpParams();
dmwsParams.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
dmwsParams.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
- dmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+ dmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
dmwsParams.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
DefaultHttpClient localDmwsHttpClient = new
DefaultHttpClient(connectionManager,dmwsParams);
+ // No retries
+ localDmwsHttpClient.setHttpRequestRetryHandler(new
HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+
localDmwsHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
if (domainUser != null)
{
@@ -290,9 +305,22 @@ public class MeridioWrapper
BasicHttpParams rmwsParams = new BasicHttpParams();
rmwsParams.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
rmwsParams.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
- rmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+ rmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
rmwsParams.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
DefaultHttpClient localRmwsHttpClient = new
DefaultHttpClient(connectionManager,rmwsParams);
+ // No retries
+ localRmwsHttpClient.setHttpRequestRetryHandler(new
HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+
localRmwsHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
if (domainUser != null)
{
@@ -323,9 +351,22 @@ public class MeridioWrapper
BasicHttpParams mcwsParams = new BasicHttpParams();
mcwsParams.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
mcwsParams.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
- mcwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+ mcwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
mcwsParams.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
DefaultHttpClient localMcwsHttpClient = new
DefaultHttpClient(connectionManager,mcwsParams);
+ // No retries
+ localMcwsHttpClient.setHttpRequestRetryHandler(new
HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+
localMcwsHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
if (domainUser != null)
{
Modified:
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
(original)
+++
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
Tue Jan 15 11:01:58 2013
@@ -53,6 +53,8 @@ import org.apache.http.params.HttpProtoc
import org.apache.http.ProtocolVersion;
import org.apache.http.util.EntityUtils;
import org.apache.http.message.BasicHeader;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.RedirectException;
@@ -111,6 +113,19 @@ public class OpenSearchServerConnection
localConnectionManager.setMaxTotal(1);
connectionManager = localConnectionManager;
DefaultHttpClient localHttpClient = new
DefaultHttpClient(connectionManager);
+ // No retries
+ localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+
httpClient = localHttpClient;
}
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
(original)
+++
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
Tue Jan 15 11:01:58 2013
@@ -53,6 +53,8 @@ import org.apache.http.conn.params.ConnR
import org.apache.http.message.BasicHeader;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.CircularRedirectException;
@@ -287,6 +289,19 @@ public class ThrottledFetcher
params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,connectionTimeoutMilliseconds);
params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
DefaultHttpClient localHttpClient = new
DefaultHttpClient(connectionManager,params);
+ // No retries
+ localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+
localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
// If there's a proxy, set that too.
Modified:
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
(original)
+++
manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
Tue Jan 15 11:01:58 2013
@@ -55,6 +55,8 @@ import org.apache.http.params.BasicHttpP
import org.apache.http.params.HttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
/** This is the "repository connector" for Microsoft SharePoint.
* Document identifiers for this connector come in three forms:
@@ -218,9 +220,21 @@ public class SharePointRepository extend
BasicHttpParams params = new BasicHttpParams();
params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
- params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+ params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
DefaultHttpClient localHttpClient = new
DefaultHttpClient(connectionManager,params);
+ // No retries
+ localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
if (strippedUserName != null)
{
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Tue Jan 15 11:01:58 2013
@@ -226,7 +226,7 @@ public class HttpPoster
public boolean retryRequest(
IOException exception,
int executionCount,
- HttpContext context)
+ HttpContext context)
{
return false;
}
Modified:
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
(original)
+++
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Tue Jan 15 11:01:58 2013
@@ -77,6 +77,7 @@ import org.apache.http.protocol.HttpCont
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.cookie.CookieIdentityComparator;
+import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.cookie.MalformedCookieException;
import org.apache.http.conn.ConnectTimeoutException;
@@ -1344,6 +1345,18 @@ public class ThrottledFetcher
params.setBooleanParameter(CookieSpecPNames.SINGLE_COOKIE_HEADER,new
Boolean(true));
DefaultHttpClient localHttpClient = new
DefaultHttpClient(connManager,params);
+ // No retries
+ localHttpClient.setHttpRequestRetryHandler(new
HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
localHttpClient.getCookieSpecs().register(CookiePolicy.BROWSER_COMPATIBILITY,
new CookieSpecFactory()
{
Modified:
manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
(original)
+++
manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
Tue Jan 15 11:01:58 2013
@@ -50,6 +50,8 @@ import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;
import org.apache.http.message.BasicHeader;
import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.CircularRedirectException;
@@ -173,10 +175,23 @@ public class WikiConnector extends org.a
BasicHttpParams params = new BasicHttpParams();
params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
- params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,300000);
+ params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,300000);
params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
DefaultHttpClient localHttpClient = new
DefaultHttpClient(connectionManager,params);
+ // No retries
+ localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+
httpClient = localHttpClient;
loginToAPI();
