Author: kwright
Date: Wed Jun 21 20:43:40 2017
New Revision: 1799511
URL: http://svn.apache.org/viewvc?rev=1799511&view=rev
Log:
Tentative fix for CONNECTORS-1433
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
Modified: manifoldcf/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Jun 21 20:43:40 2017
@@ -3,6 +3,9 @@ $Id$
======================= 2.8-dev =====================
+CONNECTORS-1433: Add pipeline specification to ES connector.
+(Steph van Schalkwyk, Karl Wright)
+
CONNECTORS-1432: Fix deleting stale check methods from window object
(Steph van Schalkwyk, Karl Wright, Kishore Kumar)
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
Wed Jun 21 20:43:40 2017
@@ -40,6 +40,7 @@ public class ElasticSearchConfig extends
ParameterEnum.INDEXNAME,
ParameterEnum.INDEXTYPE,
ParameterEnum.USEMAPPERATTACHMENTS,
+ ParameterEnum.PIPELINENAME,
ParameterEnum.CONTENTATTRIBUTENAME,
ParameterEnum.CREATEDDATEATTRIBUTENAME,
ParameterEnum.MODIFIEDDATEATTRIBUTENAME,
@@ -147,6 +148,11 @@ public class ElasticSearchConfig extends
return Boolean.valueOf(get(ParameterEnum.USEMAPPERATTACHMENTS));
}
+ final public String getPipelineName()
+ {
+ return get(ParameterEnum.PIPELINENAME);
+ }
+
final public String getContentAttributeName()
{
return get(ParameterEnum.CONTENTATTRIBUTENAME);
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Wed Jun 21 20:43:40 2017
@@ -392,11 +392,15 @@ public class ElasticSearchIndex extends
String[] acls, String[] denyAcls, String[] shareAcls, String[]
shareDenyAcls, String[] parentAcls, String[] parentDenyAcls)
throws ManifoldCFException, ServiceInterruption
{
- String idField;
- idField = URLEncoder.encode(documentURI);
- StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
+ final String idField = URLEncoder.encode(documentURI);
+ final String encodedPipelineName = (config.getPipelineName() == null ||
config.getPipelineName().length() ==
0)?null:URLEncoder.encode(config.getPipelineName());
+
+ final String command = config.getIndexType() + "/" + idField;
+ final String fullCommand = (encodedPipelineName == null)?command:(command
+ "?pipeline=" + encodedPipelineName);
+
+ StringBuffer url = getApiUrl(fullCommand, false);
HttpPut put = new HttpPut(url.toString());
put.setEntity(new IndexRequestEntity(document, inputStream,
acls, denyAcls, shareAcls, shareDenyAcls, parentAcls, parentDenyAcls,
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
Wed Jun 21 20:43:40 2017
@@ -42,6 +42,8 @@ public class ElasticSearchParam extends
INDEXTYPE("generictype"),
USEMAPPERATTACHMENTS("true"),
+
+ PIPELINENAME(""),
CONTENTATTRIBUTENAME(""),
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
ElasticSearchConnector.IndexNameColon=Index name:
ElasticSearchConnector.IndexTypeColon=Index type:
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
ElasticSearchConnector.ContentAttributeNameColon=Content field name:
ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
ElasticSearchConnector.IndexNameColon=El nombre de Ãndice:
ElasticSearchConnector.IndexTypeColon=tipo de Ãndice:
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
ElasticSearchConnector.ContentAttributeNameColon=Content field name:
ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
ElasticSearchConnector.IndexNameColon=ã¤ã³ããã¯ã¹åï¼
ElasticSearchConnector.IndexTypeColon=ã¿ã¤ãåï¼
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
ElasticSearchConnector.ContentAttributeNameColon=Content field name:
ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
ElasticSearchConnector.IndexNameColon=ç´¢å¼å:
ElasticSearchConnector.IndexTypeColon=ç´¢å¼ç±»å:
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
ElasticSearchConnector.ContentAttributeNameColon=Content field name:
ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
Wed Jun 21 20:43:40 2017
@@ -58,7 +58,12 @@
#end
</td>
</tr>
-
+ <tr>
+ <td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PipelineNameColon'))</nobr></td>
+ <td class="value"><input name="pipelinename" type="text"
value="$Encoder.attributeEscape($PIPELINENAME)"
+ size="24" /></td>
+ </tr>
+
<tr>
<td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ContentAttributeNameColon'))</nobr></td>
<td class="value"><input name="contentattributename" type="text"
value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)"
@@ -96,6 +101,7 @@
<input type="hidden" name="usemapperattachments_checkbox" value="false"/>
<input type="hidden" name="usemapperattachments_present" value="true"/>
<input type="hidden" name="usemapperattachments"
value="$Encoder.bodyEscape($USEMAPPERATTACHMENTS)" />
+<input type="hidden" name="pipelinename"
value="$Encoder.attributeEscape($PIPELINENAME)" />
<input type="hidden" name="contentattributename"
value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)" />
<input type="hidden" name="createddateattributename"
value="$Encoder.attributeEscape($CREATEDDATEATTRIBUTENAME)" />
<input type="hidden" name="modifieddateattributename"
value="$Encoder.attributeEscape($MODIFIEDDATEATTRIBUTENAME)" />
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
(original)
+++
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
Wed Jun 21 20:43:40 2017
@@ -38,6 +38,10 @@
<td
class="value"><nobr>$Encoder.bodyEscape($USEMAPPERATTACHMENTS)</nobr></td>
</tr>
<tr>
+ <td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PipelineNameColon'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($PIPELINENAME)</nobr></td>
+ </tr>
+ <tr>
<td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ContentAttributeNameColon'))</nobr></td>
<td class="value">$Encoder.bodyEscape($CONTENTATTRIBUTENAME)</td>
</tr>
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
---
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
(original)
+++
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
Wed Jun 21 20:43:40 2017
@@ -193,11 +193,11 @@ public class ModifiedHttpSolrClient exte
if (name == null) {
name = "";
}
- parts.add(new FormBodyPart(name,
+ parts.add(new FormBodyPart(encodeForHeader(name),
new InputStreamBody(
content.getStream(),
contentType,
- content.getName())));
+ encodeForHeader(content.getName()))));
}
}
@@ -293,4 +293,21 @@ public class ModifiedHttpSolrClient exte
return sb.toString();
}
+ // This is a hack added by KDW on 6/21/2017 because HttpClient doesn't do
any character
+ // escaping when it puts together header and file names
+ private static String encodeForHeader(final String headerName) {
+ if (headerName == null) {
+ return null;
+ }
+ final StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < headerName.length(); i++) {
+ final char x = headerName.charAt(i);
+ if (x == '"' || x == '\\') {
+ sb.append("\\");
+ }
+ sb.append(x);
+ }
+ return sb.toString();
+ }
+
}
