Author: kwright
Date: Wed Jan 30 18:51:11 2013
New Revision: 1440583
URL: http://svn.apache.org/viewvc?rev=1440583&view=rev
Log:
Fix for CONNECTORS-629.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Jan 30 18:51:11 2013
@@ -3,6 +3,9 @@ $Id$
======================= 1.2-dev =====================
+CONNECTORS-629: Revamp error handling in ElasticSearch connector.
+(Andrew Clegg, Karl Wright)
+
CONNECTORS-625: Finish internationalization work for parts of the
core crawler UI.
(Karl Wright)
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
Wed Jan 30 18:51:11 2013
@@ -23,6 +23,7 @@ import org.apache.http.client.methods.Ht
import org.apache.http.client.HttpClient;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
public class ElasticSearchAction extends ElasticSearchConnection
{
@@ -32,10 +33,15 @@ public class ElasticSearchAction extends
_optimize, _refresh, _status;
}
- public ElasticSearchAction(HttpClient client, CommandEnum cmd,
ElasticSearchConfig config, boolean checkConnection)
+ public ElasticSearchAction(HttpClient client, ElasticSearchConfig config)
throws ManifoldCFException
{
super(config, client);
+ }
+
+ public void execute(CommandEnum cmd, boolean checkConnection)
+ throws ManifoldCFException, ServiceInterruption
+ {
StringBuffer url = getApiUrl(cmd.toString(), checkConnection);
HttpGet method = new HttpGet(url.toString());
call(method);
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
Wed Jan 30 18:51:11 2013
@@ -56,9 +56,12 @@ import org.apache.http.HttpException;
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
public class ElasticSearchConnection
{
+ protected ElasticSearchConfig config;
+
private HttpClient client;
private String serverLocation;
@@ -83,6 +86,7 @@ public class ElasticSearchConnection
protected ElasticSearchConnection(ElasticSearchConfig config, HttpClient
client)
{
+ this.config = config;
this.client = client;
result = Result.UNKNOWN;
response = null;
@@ -176,7 +180,11 @@ public class ElasticSearchConnection
}
}
- protected void call(HttpRequestBase method) throws ManifoldCFException
+ /** Call ElasticSearch.
+ *@return false if there was a "rejection".
+ */
+ protected boolean call(HttpRequestBase method)
+ throws ManifoldCFException, ServiceInterruption
{
CallThread ct = new CallThread(client, method);
try
@@ -200,9 +208,8 @@ public class ElasticSearchConnection
throw new RuntimeException("Unexpected exception thrown:
"+t.getMessage(),t);
}
- if (!checkResultCode(ct.getResultCode()))
- throw new ManifoldCFException(getResultDescription());
response = ct.getResponse();
+ return handleResultCode(ct.getResultCode(), response);
}
catch (InterruptedException e)
{
@@ -212,16 +219,67 @@ public class ElasticSearchConnection
}
catch (HttpException e)
{
- setResult(Result.ERROR, e.getMessage());
- throw new ManifoldCFException(e);
+ handleHttpException(e);
+ return false;
}
catch (IOException e)
{
- setResult(Result.ERROR, e.getMessage());
- throw new ManifoldCFException(e);
+ handleIOException(e);
+ return false;
}
}
+ private boolean handleResultCode(int code, String response)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ if (code == 200)
+ {
+ setResult(Result.OK, null);
+ return true;
+ }
+ else if (code == 404)
+ {
+ setResult(Result.ERROR, response);
+ throw new ManifoldCFException("Server/page not found");
+ }
+ else if (code >= 400 && code < 500)
+ {
+ setResult(Result.ERROR, response);
+ return false;
+ }
+ else if (code >= 500 && code < 600)
+ {
+ setResult(Result.ERROR, "Server exception: "+response);
+ long currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Server exception: "+response,
+ new ManifoldCFException(response),
+ currentTime + 300000L,
+ currentTime + 20L * 60000L,
+ -1,
+ false);
+ }
+ setResult(Result.UNKNOWN, response);
+ throw new ManifoldCFException("Unexpected HTTP result code: "+code+":
"+response);
+ }
+
+ private void handleHttpException(HttpException e)
+ throws ManifoldCFException, ServiceInterruption {
+ setResult(Result.ERROR, e.getMessage());
+ throw new ManifoldCFException(e);
+ }
+
+ private void handleIOException(IOException e)
+ throws ManifoldCFException, ServiceInterruption {
+ setResult(Result.ERROR, e.getMessage());
+ long currentTime = System.currentTimeMillis();
+ // All IO exceptions are treated as service interruptions, retried for an
hour
+ throw new ServiceInterruption("IO exception: "+e.getMessage(),e,
+ currentTime + 60000L,
+ currentTime + 1L * 60L * 60000L,
+ -1,
+ true);
+ }
+
private static String getResponseBodyAsString(HttpEntity entity)
throws IOException, HttpException {
InputStream is = entity.getContent();
@@ -292,24 +350,6 @@ public class ElasticSearchConnection
return response;
}
- private boolean checkResultCode(int code)
- {
- switch (code)
- {
- case 0:
- setResult(Result.UNKNOWN, null);
- return false;
- case 200:
- setResult(Result.OK, null);
- return true;
- case 404:
- setResult(Result.ERROR, "Server/page not found");
- return false;
- default:
- setResult(Result.ERROR, null);
- return false;
- }
- }
public Result getResult()
{
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Wed Jan 30 18:51:11 2013
@@ -368,14 +368,19 @@ public class ElasticSearchConnector exte
ElasticSearchConfig config = getConfigParameters(null);
InputStream inputStream = document.getBinaryStream();
long startTime = System.currentTimeMillis();
- ElasticSearchIndex oi = new ElasticSearchIndex(client, documentURI,
- document, inputStream, config);
- activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
- document.getBinaryLength(), documentURI, oi.getResult().name(),
- oi.getResultDescription());
- if (oi.getResult() != Result.OK)
- return DOCUMENTSTATUS_REJECTED;
- return DOCUMENTSTATUS_ACCEPTED;
+ ElasticSearchIndex oi = new ElasticSearchIndex(client, config);
+ try
+ {
+ oi.execute(documentURI, document, inputStream);
+ if (oi.getResult() != Result.OK)
+ return DOCUMENTSTATUS_REJECTED;
+ return DOCUMENTSTATUS_ACCEPTED;
+ }
+ finally
+ {
+ activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
+ document.getBinaryLength(), documentURI, oi.getResult().name(),
oi.getResultDescription());
+ }
}
@Override
@@ -385,22 +390,35 @@ public class ElasticSearchConnector exte
{
HttpClient client = getSession();
long startTime = System.currentTimeMillis();
- ElasticSearchDelete od = new ElasticSearchDelete(client, documentURI,
- getConfigParameters(null));
- activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY, null,
- documentURI, od.getResult().name(), od.getResultDescription());
+ ElasticSearchDelete od = new ElasticSearchDelete(client,
getConfigParameters(null));
+ try
+ {
+ od.execute(documentURI);
+ }
+ finally
+ {
+ activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY,
null,
+ documentURI, od.getResult().name(), od.getResultDescription());
+ }
}
@Override
public String check() throws ManifoldCFException
{
HttpClient client = getSession();
- ElasticSearchAction oss = new ElasticSearchAction(client,
CommandEnum._status,
- getConfigParameters(null), true);
- String resultName = oss.getResult().name();
- if (resultName.equals("OK"))
- return super.check();
- return resultName + " " + oss.getResultDescription();
+ ElasticSearchAction oss = new ElasticSearchAction(client,
getConfigParameters(null));
+ try
+ {
+ oss.execute(CommandEnum._status, true);
+ String resultName = oss.getResult().name();
+ if (resultName.equals("OK"))
+ return super.check();
+ return resultName + " " + oss.getResultDescription();
+ }
+ catch (ServiceInterruption e)
+ {
+ return "Transient exception: "+e.getMessage();
+ }
}
@Override
@@ -409,11 +427,17 @@ public class ElasticSearchConnector exte
{
HttpClient client = getSession();
long startTime = System.currentTimeMillis();
- ElasticSearchAction oo = new ElasticSearchAction(client,
CommandEnum._optimize,
- getConfigParameters(null), false);
- activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
- oo.getCallUrlSnippet(), oo.getResult().name(),
- oo.getResultDescription());
+ ElasticSearchAction oo = new ElasticSearchAction(client,
getConfigParameters(null));
+ try
+ {
+ oo.execute(CommandEnum._optimize, false);
+ }
+ finally
+ {
+ activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY,
null,
+ oo.getCallUrlSnippet(), oo.getResult().name(),
+ oo.getResultDescription());
+ }
}
}
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
Wed Jan 30 18:51:11 2013
@@ -23,14 +23,19 @@ import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpDelete;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
public class ElasticSearchDelete extends ElasticSearchConnection
{
- public ElasticSearchDelete(HttpClient client, String documentURI,
ElasticSearchConfig config)
- throws ManifoldCFException
+ public ElasticSearchDelete(HttpClient client, ElasticSearchConfig config)
{
super(config, client);
+ }
+
+ public void execute(String documentURI)
+ throws ManifoldCFException, ServiceInterruption
+ {
try
{
String idField = java.net.URLEncoder.encode(documentURI,"utf-8");
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Wed Jan 30 18:51:11 2013
@@ -34,6 +34,7 @@ import org.apache.http.Header;
import org.apache.commons.io.IOUtils;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.common.Base64;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.crawler.system.Logging;
@@ -151,11 +152,17 @@ public class ElasticSearchIndex extends
}
- public ElasticSearchIndex(HttpClient client, String documentURI,
RepositoryDocument document,
- InputStream inputStream, ElasticSearchConfig config) throws
ManifoldCFException
+ public ElasticSearchIndex(HttpClient client, ElasticSearchConfig config)
{
super(config, client);
-
+ }
+
+ /** Do the indexing.
+ *@return false to indicate that the document was rejected.
+ */
+ public boolean execute(String documentURI, RepositoryDocument document,
+ InputStream inputStream) throws ManifoldCFException, ServiceInterruption
+ {
String idField;
try
{
@@ -169,12 +176,14 @@ public class ElasticSearchIndex extends
StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
HttpPut put = new HttpPut(url.toString());
put.setEntity(new IndexRequestEntity(document, inputStream));
- call(put);
+ if (call(put) == false)
+ return false;
if ("true".equals(checkJson(jsonStatus)))
- return;
+ return true;
String error = checkJson(jsonException);
setResult(Result.ERROR, error);
Logging.connectors.error(getResponse());
+ return true;
}
}
