Author: kwright
Date: Sat Jan 26 16:55:14 2013
New Revision: 1438915
URL: http://svn.apache.org/viewvc?rev=1438915&view=rev
Log:
Fix for CONNECTORS-624.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sat Jan 26 16:55:14 2013
@@ -3,6 +3,10 @@ $Id$
======================= 1.2-dev =====================
+CONNECTORS-624: Add indexing via subsidiary thread to OpenSearchServer
+connector.
+(Karl Wright)
+
CONNECTORS-606: Add indexing via subsidiary thread to ElasticSearch
connector.
(Karl Wright)
Modified:
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
(original)
+++
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
Sat Jan 26 16:55:14 2013
@@ -17,6 +17,7 @@
package org.apache.manifoldcf.agents.output.opensearchserver;
+import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
@@ -27,9 +28,9 @@ public class OpenSearchServerAction exte
optimize, reload;
}
- public OpenSearchServerAction(CommandEnum cmd, OpenSearchServerConfig config)
+ public OpenSearchServerAction(HttpClient client, CommandEnum cmd,
OpenSearchServerConfig config)
throws ManifoldCFException {
- super(config);
+ super(client, config);
StringBuffer url = getApiUrl("action");
url.append("&action=");
url.append(cmd.name());
Modified:
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
(original)
+++
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
Sat Jan 26 16:55:14 2013
@@ -25,6 +25,7 @@ import java.io.StringWriter;
import java.io.Reader;
import java.io.InputStreamReader;
import java.io.InputStream;
+import java.io.InterruptedIOException;
import java.net.URLEncoder;
import javax.xml.parsers.DocumentBuilder;
@@ -35,26 +36,11 @@ import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
-import org.apache.http.conn.ClientConnectionManager;
-import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.client.HttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpDelete;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
-import org.apache.http.Header;
-import org.apache.http.params.CoreProtocolPNames;
-import org.apache.http.params.HttpProtocolParams;
-import org.apache.http.ProtocolVersion;
import org.apache.http.util.EntityUtils;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.protocol.HttpContext;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.RedirectException;
@@ -62,7 +48,6 @@ import org.apache.http.client.CircularRe
import org.apache.http.NoHttpResponseException;
import org.apache.http.HttpException;
-
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
@@ -86,8 +71,6 @@ public class OpenSearchServerConnection
private Document xmlResponse;
- private ClientConnectionManager connectionManager;
-
private HttpClient httpClient;
protected String xPathStatus = "/response/entry[@key='Status']/text()";
@@ -99,7 +82,8 @@ public class OpenSearchServerConnection
private Result result;
- protected OpenSearchServerConnection(OpenSearchServerConfig config) {
+ protected OpenSearchServerConnection(HttpClient client,
OpenSearchServerConfig config) {
+ this.httpClient = client;
result = Result.UNKNOWN;
response = null;
xmlResponse = null;
@@ -109,24 +93,6 @@ public class OpenSearchServerConnection
indexName = config.getIndexName();
userName = config.getUserName();
apiKey = config.getApiKey();
- PoolingClientConnectionManager localConnectionManager = new
PoolingClientConnectionManager();
- localConnectionManager.setMaxTotal(1);
- connectionManager = localConnectionManager;
- DefaultHttpClient localHttpClient = new
DefaultHttpClient(connectionManager);
- // No retries
- localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
- {
- public boolean retryRequest(
- IOException exception,
- int executionCount,
- HttpContext context)
- {
- return false;
- }
-
- });
-
- httpClient = localHttpClient;
}
protected final String urlEncode(String t) throws ManifoldCFException {
@@ -155,21 +121,110 @@ public class OpenSearchServerConnection
return url;
}
- protected void call(HttpRequestBase method) throws ManifoldCFException {
- try {
- HttpResponse resp = httpClient.execute(method);
- if (!checkResultCode(resp.getStatusLine().getStatusCode()))
- throw new ManifoldCFException(getResultDescription());
- response = getResponseBodyAsString(resp.getEntity());
- } catch (HttpException e) {
+ protected static class CallThread extends Thread
+ {
+ protected final HttpClient client;
+ protected final HttpRequestBase method;
+ protected int resultCode = -1;
+ protected String response = null;
+ protected Throwable exception = null;
+
+ public CallThread(HttpClient client, HttpRequestBase method)
+ {
+ this.client = client;
+ this.method = method;
+ setDaemon(true);
+ }
+
+ @Override
+ public void run()
+ {
+ try
+ {
+ try
+ {
+ HttpResponse resp = client.execute(method);
+ resultCode = resp.getStatusLine().getStatusCode();
+ response = getResponseBodyAsString(resp.getEntity());
+ }
+ finally
+ {
+ method.abort();
+ }
+ }
+ catch (java.net.SocketTimeoutException e)
+ {
+ exception = e;
+ }
+ catch (InterruptedIOException e)
+ {
+ // Just exit
+ }
+ catch (Throwable e)
+ {
+ exception = e;
+ }
+ }
+
+ public int getResultCode()
+ {
+ return resultCode;
+ }
+
+ public String getResponse()
+ {
+ return response;
+ }
+
+ public Throwable getException()
+ {
+ return exception;
+ }
+ }
+
+ protected void call(HttpRequestBase method) throws ManifoldCFException
+ {
+ CallThread ct = new CallThread(httpClient, method);
+ try
+ {
+ ct.start();
+ try
+ {
+ ct.join();
+ Throwable t = ct.getException();
+ if (t != null)
+ {
+ if (t instanceof HttpException)
+ throw (HttpException)t;
+ else if (t instanceof IOException)
+ throw (IOException)t;
+ else if (t instanceof RuntimeException)
+ throw (RuntimeException)t;
+ else if (t instanceof Error)
+ throw (Error)t;
+ else
+ throw new RuntimeException("Unexpected exception thrown:
"+t.getMessage(),t);
+ }
+
+ if (!checkResultCode(ct.getResultCode()))
+ throw new ManifoldCFException(getResultDescription());
+ response = ct.getResponse();
+ }
+ catch (InterruptedException e)
+ {
+ ct.interrupt();
+ throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ }
+ catch (HttpException e)
+ {
setResult(Result.ERROR, e.getMessage());
throw new ManifoldCFException(e);
- } catch (IOException e) {
+ }
+ catch (IOException e)
+ {
setResult(Result.ERROR, e.getMessage());
throw new ManifoldCFException(e);
- } finally {
- if (method != null)
- method.abort();
}
}
Modified:
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
(original)
+++
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
Sat Jan 26 16:55:14 2013
@@ -28,6 +28,13 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.Locale;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
+
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
@@ -86,6 +93,13 @@ public class OpenSearchServerConnector e
/** Forward to the javascript to check the specification parameters for the
job */
private static final String EDIT_SPEC_HEADER_FORWARD =
"editSpecification.js";
+ /** Connection expiration interval */
+ private static final long EXPIRATION_INTERVAL = 60000L;
+
+ private ClientConnectionManager connectionManager = null;
+ private HttpClient client = null;
+ private long expirationTime = -1L;
+
// Private data
private String specsCacheOutpuDescription;
@@ -97,6 +111,73 @@ public class OpenSearchServerConnector e
}
@Override
+ public void connect(ConfigParams configParams)
+ {
+ super.connect(configParams);
+ }
+
+ protected HttpClient getSession()
+ throws ManifoldCFException
+ {
+ if (client == null)
+ {
+ PoolingClientConnectionManager localConnectionManager = new
PoolingClientConnectionManager();
+ localConnectionManager.setMaxTotal(1);
+ connectionManager = localConnectionManager;
+ DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+ // No retries
+ localClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+ {
+ public boolean retryRequest(
+ IOException exception,
+ int executionCount,
+ HttpContext context)
+ {
+ return false;
+ }
+
+ });
+ client = localClient;
+ }
+ expirationTime = System.currentTimeMillis() + EXPIRATION_INTERVAL;
+ return client;
+ }
+
+ protected void closeSession()
+ {
+ if (connectionManager != null)
+ {
+ connectionManager.shutdown();
+ connectionManager = null;
+ }
+ client = null;
+ expirationTime = -1L;
+ }
+
+ @Override
+ public void disconnect()
+ throws ManifoldCFException
+ {
+ super.disconnect();
+ closeSession();
+ }
+
+
+ @Override
+ public void poll()
+ throws ManifoldCFException
+ {
+ super.poll();
+ if (connectionManager != null)
+ {
+ if (System.currentTimeMillis() > expirationTime)
+ {
+ closeSession();
+ }
+ }
+ }
+
+ @Override
public String[] getActivitiesList() {
return OPENSEARCHSERVER_ACTIVITIES;
}
@@ -304,13 +385,16 @@ public class OpenSearchServerConnector e
RepositoryDocument document, String authorityNameString,
IOutputAddActivity activities) throws ManifoldCFException,
ServiceInterruption {
+ HttpClient client = getSession();
OpenSearchServerConfig config = getConfigParameters(null);
Integer count = addInstance(config);
synchronized (count) {
InputStream inputStream = document.getBinaryStream();
try {
long startTime = System.currentTimeMillis();
- OpenSearchServerIndex oi = new OpenSearchServerIndex(documentURI,
+ OpenSearchServerIndex oi = new OpenSearchServerIndex(
+ client,
+ documentURI,
inputStream, config);
activities.recordActivity(startTime,
OPENSEARCHSERVER_INDEXATION_ACTIVITY, document.getBinaryLength(),
@@ -328,8 +412,11 @@ public class OpenSearchServerConnector e
public void removeDocument(String documentURI, String outputDescription,
IOutputRemoveActivity activities) throws ManifoldCFException,
ServiceInterruption {
+ HttpClient client = getSession();
long startTime = System.currentTimeMillis();
- OpenSearchServerDelete od = new OpenSearchServerDelete(documentURI,
+ OpenSearchServerDelete od = new OpenSearchServerDelete(
+ client,
+ documentURI,
getConfigParameters(null));
activities.recordActivity(startTime, OPENSEARCHSERVER_DELETION_ACTIVITY,
null, documentURI, od.getResult().name(), od.getResultDescription());
@@ -337,7 +424,9 @@ public class OpenSearchServerConnector e
@Override
public String check() throws ManifoldCFException {
+ HttpClient client = getSession();
OpenSearchServerSchema oss = new OpenSearchServerSchema(
+ client,
getConfigParameters(null));
return oss.getResult().name() + " " + oss.getResultDescription();
}
@@ -345,8 +434,10 @@ public class OpenSearchServerConnector e
@Override
public void noteJobComplete(IOutputNotifyActivity activities)
throws ManifoldCFException, ServiceInterruption {
+ HttpClient client = getSession();
long startTime = System.currentTimeMillis();
OpenSearchServerAction oo = new OpenSearchServerAction(
+ client,
CommandEnum.optimize, getConfigParameters(null));
activities.recordActivity(startTime, OPENSEARCHSERVER_OPTIMIZE_ACTIVITY,
null, oo.getCallUrlSnippet(), oo.getResult().name(),
Modified:
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
(original)
+++
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
Sat Jan 26 16:55:14 2013
@@ -17,15 +17,16 @@
package org.apache.manifoldcf.agents.output.opensearchserver;
+import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class OpenSearchServerDelete extends OpenSearchServerConnection {
- public OpenSearchServerDelete(String documentURI,
+ public OpenSearchServerDelete(HttpClient client, String documentURI,
OpenSearchServerConfig config) throws ManifoldCFException {
- super(config);
+ super(client, config);
StringBuffer url = getApiUrl("delete");
url.append("&uniq=");
url.append(urlEncode(documentURI));
Modified:
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
(original)
+++
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
Sat Jan 26 16:55:14 2013
@@ -22,6 +22,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
+import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.HttpEntity;
import org.apache.http.message.BasicHeader;
@@ -118,9 +119,9 @@ public class OpenSearchServerIndex exten
}
- public OpenSearchServerIndex(String documentURI, InputStream inputStream,
+ public OpenSearchServerIndex(HttpClient client, String documentURI,
InputStream inputStream,
OpenSearchServerConfig config) throws ManifoldCFException {
- super(config);
+ super(client, config);
StringBuffer url = getApiUrl("update");
HttpPut put = new HttpPut(url.toString());
put.setEntity(new IndexRequestEntity(documentURI, inputStream));
Modified:
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
(original)
+++
manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
Sat Jan 26 16:55:14 2013
@@ -17,15 +17,16 @@
package org.apache.manifoldcf.agents.output.opensearchserver;
+import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
public class OpenSearchServerSchema extends OpenSearchServerConnection {
- public OpenSearchServerSchema(OpenSearchServerConfig config)
+ public OpenSearchServerSchema(HttpClient client, OpenSearchServerConfig
config)
throws ManifoldCFException {
- super(config);
+ super(client, config);
String indexName = config.getIndexName();
StringBuffer url = getApiUrl("schema");
url.append("&cmd=indexList");
