Author: kwright
Date: Sun Jan 13 17:34:12 2013
New Revision: 1432669
URL: http://svn.apache.org/viewvc?rev=1432669&view=rev
Log:
Fix for CONNECTORS-608. Increase default socket timeout to 15 minutes. Also
change handling of socket timeouts to skip the document rather than abort the
job.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1432669&r1=1432668&r2=1432669&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Jan 13 17:34:12 2013
@@ -3,6 +3,10 @@ $Id$
======================= 1.1-dev =====================
+CONNECTORS-608: Solr connector times out chronically on some
+documents.
+(David Morana, Karl Wright)
+
CONNECTORS-607: Handle multi-valued parameters properly in the
Solr connector.
(Minoru Osuka, Shinichiro Abe, Karl Wright)
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1432669&r1=1432668&r2=1432669&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Sun Jan 13 17:34:12 2013
@@ -370,19 +370,31 @@ public class HttpPoster
if ((e instanceof InterruptedIOException) && (!(e instanceof
java.net.SocketTimeoutException)))
throw new ManifoldCFException(e.getMessage(),
ManifoldCFException.INTERRUPTED);
- // Intercept "broken pipe" exception, since that seems to be what we get
if the ingestion API kills the socket right after a 400 goes out.
- // Basically, we have no choice but to interpret that in the same manner
as a 400, since no matter how we do it, it's a race and the 'broken pipe'
- // result is always possible. So we might as well expect it and treat it
properly.
- if (e.getClass().getName().equals("java.net.SocketException") &&
e.getMessage().toLowerCase().indexOf("broken pipe") != -1)
- {
- // We've seen what looks like the ingestion interface forcibly closing
the socket.
- // We *choose* to interpret this just like a 400 response. However, we
log in the history using a different code,
- // since we really don't know what happened for sure.
- return;
+ long currentTime = System.currentTimeMillis();
+
+ if (e.getClass().getName().equals("java.net.SocketException"))
+ {
+ // Intercept "broken pipe" exception, since that seems to be what we get
if the ingestion API kills the socket right after a 400 goes out.
+ // Basically, we have no choice but to interpret that in the same manner
as a 400, since no matter how we do it, it's a race and the 'broken pipe'
+ // result is always possible. So we might as well expect it and treat
it properly.
+ if (e.getMessage().toLowerCase().indexOf("broken pipe") != -1)
+ // We've seen what looks like the ingestion interface forcibly closing
the socket.
+ // We *choose* to interpret this just like a 400 response. However,
we log in the history using a different code,
+ // since we really don't know what happened for sure.
+ return;
+
+ // Other socket exceptions are service interruptions - but if we keep
getting them, it means
+ // that a socket timeout is probably set too low to accept this
particular document. So
+ // we retry for a while, then skip the document.
+ throw new ServiceInterruption("Socket timeout exception during
"+context+": "+e.getMessage(),
+ e,
+ currentTime + interruptionRetryTime,
+ currentTime + 1L * 60L * 60000L,
+ -1,
+ false);
}
- // Otherwise, presume that retries might fix it.
- long currentTime = System.currentTimeMillis();
+ // Otherwise, no idea what the trouble is, so presume that retries might
fix it.
throw new ServiceInterruption("IO exception during "+context+":
"+e.getMessage(),
e,
currentTime + interruptionRetryTime,
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1432669&r1=1432668&r2=1432669&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Sun Jan 13 17:34:12 2013
@@ -226,7 +226,7 @@ public class SolrConnector extends org.a
// Pick up timeouts
String socketTimeoutString =
params.getParameter(SolrConfig.PARAM_SOCKET_TIMEOUT);
if (socketTimeoutString == null)
- socketTimeoutString = "60";
+ socketTimeoutString = "900";
String connectTimeoutString =
params.getParameter(SolrConfig.PARAM_CONNECTION_TIMEOUT);
if (connectTimeoutString == null)
connectTimeoutString = "60";
@@ -967,7 +967,7 @@ public class SolrConnector extends org.a
String socketTimeout =
parameters.getParameter(SolrConfig.PARAM_SOCKET_TIMEOUT);
if (socketTimeout == null)
- socketTimeout = "60";
+ socketTimeout = "900";
String zkClientTimeout =
parameters.getParameter(SolrConfig.PARAM_ZOOKEEPER_CLIENT_TIMEOUT);
if (zkClientTimeout == null)
