Author: kwright
Date: Wed Sep 17 12:06:56 2014
New Revision: 1625539
URL: http://svn.apache.org/r1625539
Log:
Pull up output connector changes related to CONNECTORS-977 from trunk.
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/branches/dev_1x/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineConnector.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1621449
Modified:
manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1625539&r1=1625538&r2=1625539&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Wed Sep 17 12:06:56 2014
@@ -456,10 +456,10 @@ public class ElasticSearchConnector exte
}
@Override
- public boolean checkMimeTypeIndexable(String outputDescription,
- String mimeType) throws ManifoldCFException, ServiceInterruption
+ public boolean checkMimeTypeIndexable(VersionContext outputDescription,
String mimeType, IOutputCheckActivity activities)
+ throws ManifoldCFException, ServiceInterruption
{
- ElasticSearchSpecs specs = getSpecsCache(outputDescription);
+ ElasticSearchSpecs specs =
getSpecsCache(outputDescription.getVersionString());
return specs.checkMimeType(mimeType);
}
@@ -504,11 +504,24 @@ public class ElasticSearchConnector exte
return new String[0];
}
+ /** Add (or replace) a document in the output data store using the connector.
+ * This method presumes that the connector object has been configured, and it
is thus able to communicate with the output data store should that be
+ * necessary.
+ *@param documentURI is the URI of the document. The URI is presumed to be
the unique identifier which the output data store will use to process
+ * and serve the document. This URI is constructed by the repository
connector which fetches the document, and is thus universal across all output
connectors.
+ *@param pipelineDescription includes the description string that was
constructed for this document by the getOutputDescription() method.
+ *@param document is the document data to be processed (handed to the output
data store).
+ *@param authorityNameString is the name of the authority responsible for
authorizing any access tokens passed in with the repository document. May be
null.
+ *@param activities is the handle to an object that the implementer of a
pipeline connector may use to perform operations, such as logging processing
activity,
+ * or sending a modified document to the next stage in the pipeline.
+ *@return the document status (accepted or permanently rejected).
+ *@throws IOException only if there's a stream error reading the document
data.
+ */
@Override
- public int addOrReplaceDocument(String documentURI, String outputDescription,
+ public int addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription,
RepositoryDocument document, String authorityNameString,
IOutputAddActivity activities) throws ManifoldCFException,
- ServiceInterruption
+ ServiceInterruption, IOException
{
HttpClient client = getSession();
ElasticSearchConfig config = getConfigParameters(null);
Modified:
manifoldcf/branches/dev_1x/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java?rev=1625539&r1=1625538&r2=1625539&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/gts/connector/src/main/java/org/apache/manifoldcf/agents/output/gts/GTSConnector.java
Wed Sep 17 12:06:56 2014
@@ -200,7 +200,7 @@ public class GTSConnector extends org.ap
*@return true if the mime type is indexable by this connector.
*/
@Override
- public boolean checkMimeTypeIndexable(String mimeType)
+ public boolean checkMimeTypeIndexable(VersionContext outputDescription,
String mimeType, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
return (ingestableMimeTypeMap.get(mimeType) != null);
@@ -213,11 +213,9 @@ public class GTSConnector extends org.ap
*@return true if the file is indexable.
*/
@Override
- public boolean checkDocumentIndexable(File localFile)
+ public boolean checkDocumentIndexable(VersionContext outputDescription, File
localFile, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- if (!super.checkDocumentIndexable(localFile))
- return false;
int docType = fingerprint(localFile);
return (docType == DT_TEXT ||
docType == DT_MSWORD ||
@@ -288,21 +286,22 @@ public class GTSConnector extends org.ap
/** Add (or replace) a document in the output data store using the connector.
* This method presumes that the connector object has been configured, and it
is thus able to communicate with the output data store should that be
* necessary.
- * The OutputSpecification is *not* provided to this method, because the goal
is consistency, and if output is done it must be consistent with the
- * output description, since that was what was partly used to determine if
output should be taking place. So it may be necessary for this method to decode
- * an output description string in order to determine what should be done.
*@param documentURI is the URI of the document. The URI is presumed to be
the unique identifier which the output data store will use to process
* and serve the document. This URI is constructed by the repository
connector which fetches the document, and is thus universal across all output
connectors.
- *@param outputDescription is the description string that was constructed for
this document by the getOutputDescription() method.
+ *@param pipelineDescription includes the description string that was
constructed for this document by the getOutputDescription() method.
*@param document is the document data to be processed (handed to the output
data store).
*@param authorityNameString is the name of the authority responsible for
authorizing any access tokens passed in with the repository document. May be
null.
- *@param activities is the handle to an object that the implementer of an
output connector may use to perform operations, such as logging processing
activity.
+ *@param activities is the handle to an object that the implementer of a
pipeline connector may use to perform operations, such as logging processing
activity,
+ * or sending a modified document to the next stage in the pipeline.
*@return the document status (accepted or permanently rejected).
+ *@throws IOException only if there's a stream error reading the document
data.
*/
@Override
- public int addOrReplaceDocument(String documentURI, String
outputDescription, RepositoryDocument document, String authorityNameString,
IOutputAddActivity activities)
- throws ManifoldCFException, ServiceInterruption
+ public int addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription, RepositoryDocument document, String
authorityNameString, IOutputAddActivity activities)
+ throws ManifoldCFException, ServiceInterruption, IOException
{
+ String outputDescription = pipelineDescription.getVersionString();
+
// Establish a session
getSession();
Modified:
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1625539&r1=1625538&r2=1625539&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Wed Sep 17 12:06:56 2014
@@ -256,22 +256,22 @@ public class HDFSOutputConnector extends
}
/** Add (or replace) a document in the output data store using the connector.
- * This method presumes that the connector object has been configured, and
it is thus able to communicate with the output data store should that be
- * necessary.
- * The OutputSpecification is *not* provided to this method, because the
goal is consistency, and if output is done it must be consistent with the
- * output description, since that was what was partly used to determine if
output should be taking place. So it may be necessary for this method to decode
- * an output description string in order to determine what should be done.
- *@param documentURI is the URI of the document. The URI is presumed to be
the unique identifier which the output data store will use to process
- * and serve the document. This URI is constructed by the repository
connector which fetches the document, and is thus universal across all output
connectors.
- *@param outputDescription is the description string that was constructed
for this document by the getOutputDescription() method.
- *@param document is the document data to be processed (handed to the output
data store).
- *@param authorityNameString is the name of the authority responsible for
authorizing any access tokens passed in with the repository document. May be
null.
- *@param activities is the handle to an object that the implementer of an
output connector may use to perform operations, such as logging processing
activity.
- *@return the document status (accepted or permanently rejected).
- */
+ * This method presumes that the connector object has been configured, and it
is thus able to communicate with the output data store should that be
+ * necessary.
+ *@param documentURI is the URI of the document. The URI is presumed to be
the unique identifier which the output data store will use to process
+ * and serve the document. This URI is constructed by the repository
connector which fetches the document, and is thus universal across all output
connectors.
+ *@param pipelineDescription includes the description string that was
constructed for this document by the getOutputDescription() method.
+ *@param document is the document data to be processed (handed to the output
data store).
+ *@param authorityNameString is the name of the authority responsible for
authorizing any access tokens passed in with the repository document. May be
null.
+ *@param activities is the handle to an object that the implementer of a
pipeline connector may use to perform operations, such as logging processing
activity,
+ * or sending a modified document to the next stage in the pipeline.
+ *@return the document status (accepted or permanently rejected).
+ *@throws IOException only if there's a stream error reading the document
data.
+ */
@Override
- public int addOrReplaceDocument(String documentURI, String
outputDescription, RepositoryDocument document, String authorityNameString,
IOutputAddActivity activities) throws ManifoldCFException, ServiceInterruption {
-
+ public int addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription, RepositoryDocument document, String
authorityNameString, IOutputAddActivity activities)
+ throws ManifoldCFException, ServiceInterruption, IOException {
+ String outputDescription = pipelineDescription.getVersionString();
try {
HDFSOutputSpecs specs = new HDFSOutputSpecs(outputDescription);
Modified:
manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java?rev=1625539&r1=1625538&r2=1625539&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
Wed Sep 17 12:06:56 2014
@@ -37,6 +37,7 @@ import org.apache.http.protocol.HttpRequ
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.agents.output.BaseOutputConnector;
@@ -259,29 +260,24 @@ public class OpenSearchServerConnector e
null);
}
- /**
- * Obtain the name of the form check javascript method to call.
- *
- * @param connectionSequenceNumber is the unique number of this connection
within the job.
- * @return the name of the form check javascript method.
+ /** Obtain the name of the form check javascript method to call.
+ *@param connectionSequenceNumber is the unique number of this connection
within the job.
+ *@return the name of the form check javascript method.
*/
@Override
public String getFormCheckJavascriptMethodName(int connectionSequenceNumber)
{
- return "s" + connectionSequenceNumber + "_checkSpecification";
+ return "s"+connectionSequenceNumber+"_checkSpecification";
}
- /**
- * Obtain the name of the form presave check javascript method to call.
- *
- * @param connectionSequenceNumber is the unique number of this connection
within the job.
- * @return the name of the form presave check javascript method.
+ /** Obtain the name of the form presave check javascript method to call.
+ *@param connectionSequenceNumber is the unique number of this connection
within the job.
+ *@return the name of the form presave check javascript method.
*/
@Override
- public String getFormPresaveCheckJavascriptMethodName(
- int connectionSequenceNumber)
+ public String getFormPresaveCheckJavascriptMethodName(int
connectionSequenceNumber)
{
- return "s" + connectionSequenceNumber + "_checkSpecificationForSave";
+ return "s"+connectionSequenceNumber+"_checkSpecificationForSave";
}
/**
@@ -431,29 +427,21 @@ public class OpenSearchServerConnector e
}
@Override
- public boolean checkLengthIndexable(String outputDescription, long length)
+ public boolean checkLengthIndexable(VersionContext outputDescription, long
length, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- OpenSearchServerSpecs specs = getSpecsCache(outputDescription);
+ OpenSearchServerSpecs specs =
getSpecsCache(outputDescription.getVersionString());
long maxFileSize = specs.getMaxFileSize();
if (length > maxFileSize)
return false;
- return super.checkLengthIndexable(outputDescription, length);
- }
-
- @Override
- public boolean checkDocumentIndexable(String outputDescription, File
localFile)
- throws ManifoldCFException, ServiceInterruption
- {
return true;
}
@Override
- public boolean checkMimeTypeIndexable(String outputDescription,
- String mimeType)
+ public boolean checkMimeTypeIndexable(VersionContext outputDescription,
String mimeType, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- OpenSearchServerSpecs specs = getSpecsCache(outputDescription);
+ OpenSearchServerSpecs specs =
getSpecsCache(outputDescription.getVersionString());
return specs.checkMimeType(mimeType);
}
@@ -469,10 +457,10 @@ public class OpenSearchServerConnector e
* @return true if the file is indexable.
*/
@Override
- public boolean checkURLIndexable(String outputDescription, String url)
+ public boolean checkURLIndexable(VersionContext outputDescription, String
url, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- OpenSearchServerSpecs specs = getSpecsCache(outputDescription);
+ OpenSearchServerSpecs specs =
getSpecsCache(outputDescription.getVersionString());
return specs.checkExtension(FilenameUtils.getExtension(url));
}
@@ -550,11 +538,24 @@ public class OpenSearchServerConnector e
}
}
+ /** Add (or replace) a document in the output data store using the connector.
+ * This method presumes that the connector object has been configured, and it
is thus able to communicate with the output data store should that be
+ * necessary.
+ *@param documentURI is the URI of the document. The URI is presumed to be
the unique identifier which the output data store will use to process
+ * and serve the document. This URI is constructed by the repository
connector which fetches the document, and is thus universal across all output
connectors.
+ *@param pipelineDescription includes the description string that was
constructed for this document by the getOutputDescription() method.
+ *@param document is the document data to be processed (handed to the output
data store).
+ *@param authorityNameString is the name of the authority responsible for
authorizing any access tokens passed in with the repository document. May be
null.
+ *@param activities is the handle to an object that the implementer of a
pipeline connector may use to perform operations, such as logging processing
activity,
+ * or sending a modified document to the next stage in the pipeline.
+ *@return the document status (accepted or permanently rejected).
+ *@throws IOException only if there's a stream error reading the document
data.
+ */
@Override
- public int addOrReplaceDocument(String documentURI, String outputDescription,
+ public int addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription,
RepositoryDocument document, String authorityNameString,
IOutputAddActivity activities)
- throws ManifoldCFException, ServiceInterruption
+ throws ManifoldCFException, ServiceInterruption, IOException
{
HttpClient client = getSession();
OpenSearchServerConfig config = getConfigParameters(null);
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1625539&r1=1625538&r2=1625539&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Wed Sep 17 12:06:56 2014
@@ -31,6 +31,7 @@ import java.util.HashSet;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.interfaces.Specification;
@@ -516,7 +517,7 @@ public class SolrConnector extends org.a
*@return true if the mime type is indexable by this connector.
*/
@Override
- public boolean checkMimeTypeIndexable(String outputDescription, String
mimeType)
+ public boolean checkMimeTypeIndexable(VersionContext outputDescription,
String mimeType, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
getSession();
@@ -526,7 +527,7 @@ public class SolrConnector extends org.a
return false;
if (excludedMimeTypes != null && excludedMimeTypes.get(mimeType) != null)
return false;
- return super.checkMimeTypeIndexable(outputDescription,mimeType);
+ return true;
}
return acceptableMimeTypes.contains(mimeType.toLowerCase(Locale.ROOT));
}
@@ -538,34 +539,33 @@ public class SolrConnector extends org.a
*@return true if the file is indexable.
*/
@Override
- public boolean checkLengthIndexable(String outputDescription, long length)
+ public boolean checkLengthIndexable(VersionContext outputDescription, long
length, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption
{
getSession();
if (maxDocumentLength != null && length > maxDocumentLength.longValue())
return false;
- return super.checkLengthIndexable(outputDescription,length);
+ return true;
}
/** Add (or replace) a document in the output data store using the connector.
* This method presumes that the connector object has been configured, and it
is thus able to communicate with the output data store should that be
* necessary.
- * The OutputSpecification is *not* provided to this method, because the goal
is consistency, and if output is done it must be consistent with the
- * output description, since that was what was partly used to determine if
output should be taking place. So it may be necessary for this method to decode
- * an output description string in order to determine what should be done.
*@param documentURI is the URI of the document. The URI is presumed to be
the unique identifier which the output data store will use to process
* and serve the document. This URI is constructed by the repository
connector which fetches the document, and is thus universal across all output
connectors.
- *@param outputDescription is the description string that was constructed for
this document by the getOutputDescription() method.
+ *@param pipelineDescription includes the description string that was
constructed for this document by the getOutputDescription() method.
*@param document is the document data to be processed (handed to the output
data store).
*@param authorityNameString is the name of the authority responsible for
authorizing any access tokens passed in with the repository document. May be
null.
- *@param activities is the handle to an object that the implementer of an
output connector may use to perform operations, such as logging processing
activity.
+ *@param activities is the handle to an object that the implementer of a
pipeline connector may use to perform operations, such as logging processing
activity,
+ * or sending a modified document to the next stage in the pipeline.
*@return the document status (accepted or permanently rejected).
+ *@throws IOException only if there's a stream error reading the document
data.
*/
@Override
- public int addOrReplaceDocument(String documentURI, String
outputDescription, RepositoryDocument document, String authorityNameString,
IOutputAddActivity activities)
- throws ManifoldCFException, ServiceInterruption
+ public int addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription, RepositoryDocument document, String
authorityNameString, IOutputAddActivity activities)
+ throws ManifoldCFException, ServiceInterruption, IOException
{
- SpecPacker sp = new SpecPacker(outputDescription);
+ SpecPacker sp = new SpecPacker(pipelineDescription.getVersionString());
// Establish a session
getSession();
Modified:
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineConnector.java?rev=1625539&r1=1625538&r2=1625539&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineConnector.java
(original)
+++
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IPipelineConnector.java
Wed Sep 17 12:06:56 2014
@@ -101,12 +101,9 @@ public interface IPipelineConnector exte
/** Add (or replace) a document in the output data store using the connector.
* This method presumes that the connector object has been configured, and it
is thus able to communicate with the output data store should that be
* necessary.
- * The OutputSpecification is *not* provided to this method, because the goal
is consistency, and if output is done it must be consistent with the
- * output description, since that was what was partly used to determine if
output should be taking place. So it may be necessary for this method to decode
- * an output description string in order to determine what should be done.
*@param documentURI is the URI of the document. The URI is presumed to be
the unique identifier which the output data store will use to process
* and serve the document. This URI is constructed by the repository
connector which fetches the document, and is thus universal across all output
connectors.
- *@param outputDescription is the description string that was constructed for
this document by the getOutputDescription() method.
+ *@param pipelineDescription includes the description string that was
constructed for this document by the getOutputDescription() method.
*@param document is the document data to be processed (handed to the output
data store).
*@param authorityNameString is the name of the authority responsible for
authorizing any access tokens passed in with the repository document. May be
null.
*@param activities is the handle to an object that the implementer of a
pipeline connector may use to perform operations, such as logging processing
activity,
