Revision: 18954
http://sourceforge.net/p/gate/code/18954
Author: ian_roberts
Date: 2015-10-20 21:13:20 +0000 (Tue, 20 Oct 2015)
Log Message:
-----------
Added batching support to Mimir indexing PR. Also improved the test in
MimirConnector for checking whether there are any outstanding documents to
send, so we don't make un-necessary HTTP connections to Mimir.
Modified Paths:
--------------
mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java
mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java
Modified: mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java
===================================================================
--- mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java
2015-10-18 11:59:01 UTC (rev 18953)
+++ mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java
2015-10-20 21:13:20 UTC (rev 18954)
@@ -84,6 +84,11 @@
private int connectionInterval = -1;
/**
+ * How many documents have been buffered since the last connection.
+ */
+ private int docsSinceLastConnection = 0;
+
+ /**
* The URL of the mimir index that is to receive the document. This would
* typically be of the form
* <code>http://server:port/mimir/<index UUID>/</code>.
@@ -145,6 +150,7 @@
synchronized(this) {
if(doc != null){
objectOutputStream.writeUnshared(doc);
+ docsSinceLastConnection++;
}
if(byteBuffer.size() > BYTE_BUFFER_SIZE) {
writeBuffer(); // this will also empty (reset) the buffer
@@ -170,7 +176,7 @@
* @throws IOException
*/
protected synchronized void writeBuffer() throws IOException {
- if(byteBuffer.size() > 0) {
+ if(docsSinceLastConnection > 0) {
// first phase - call the indexUrl action to find out where to post the
// data
StringBuilder indexURLString = new
StringBuilder(indexURL.toExternalForm());
@@ -194,6 +200,7 @@
} finally {
byteBuffer.reset();
objectOutputStream = new ObjectOutputStream(byteBuffer);
+ docsSinceLastConnection = 0;
}
}
Modified: mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java
===================================================================
--- mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java
2015-10-18 11:59:01 UTC (rev 18953)
+++ mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java
2015-10-20 21:13:20 UTC (rev 18954)
@@ -23,6 +23,8 @@
import gate.mimir.tool.WebUtils;
import gate.util.GateRuntimeException;
+import org.apache.log4j.Logger;
+
import java.net.URL;
@@ -35,12 +37,16 @@
private static final long serialVersionUID = 3291873032301133998L;
+ private static final Logger log = Logger.getLogger(MimirIndexingPR.class);
+
private URL mimirIndexUrl;
private String mimirUsername;
private String mimirPassword;
+ private Integer connectionInterval;
+
protected MimirConnector mimirConnector;
@@ -64,8 +70,7 @@
@RunTime
public void setMimirUsername(String mimirUsername) {
this.mimirUsername = mimirUsername;
- // invalidate the connector
- this.mimirConnector = null;
+ closeConnector();
}
public String getMimirPassword() {
@@ -77,17 +82,36 @@
@RunTime
public void setMimirPassword(String mimirPassword) {
this.mimirPassword = mimirPassword;
- // invalidate the connector
- this.mimirConnector = null;
+ closeConnector();
}
+ public Integer getConnectionInterval() {
+ return connectionInterval;
+ }
+
+ @CreoleParameter(comment="Interval between connections to the Mímir server.
-1 causes each document "
+ + "to be sent immediately, positive values cause documents to be
buffered and sent to the server in "
+ + "batches, which is generally much more efficient, particularly when
the documents are short.",
+ defaultValue = "1000")
+ @Optional
+ @RunTime
+ public void setConnectionInterval(Integer connectionInterval) {
+ this.connectionInterval = connectionInterval;
+ closeConnector();
+ }
+
@Override
public void cleanup() {
- try {
- mimirConnector.close();
- } catch(Exception e) {
- throw new GateRuntimeException("Execption while closing Mímir
connector",
- e);
+ closeConnector();
+ }
+
+ protected void closeConnector() {
+ if(mimirConnector != null) {
+ try {
+ mimirConnector.close();
+ } catch(Exception e) {
+ log.warn("Execption while closing Mímir connector", e);
+ }
}
mimirConnector = null;
}
@@ -103,6 +127,9 @@
} else {
mimirConnector = new MimirConnector(mimirIndexUrl);
}
+ if(connectionInterval != null) {
+ mimirConnector.setConnectionInterval(connectionInterval.intValue());
+ }
}
mimirConnector.sendToMimir(getDocument(), null);
} catch(Exception e) {
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
_______________________________________________
GATE-cvs mailing list
GATE-cvs@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/gate-cvs
