[
https://issues.apache.org/jira/browse/NIFI-6404?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
David Vassallo updated NIFI-6404:
---------------------------------
Description:
In ES 7.x and above, document "type" is no longer compulsory and in fact is
deprecated. When using the 1.9.2 version of PutElasticsearchHttp with ES v7.2,
it still works however you'll see the following HTTP in the response:
{{HTTP/1.1 200 OK}}
{{Warning: 299 Elasticsearch-7.2.0-508c38a "[types removal] Specifying types
in bulk requests is deprecated."}}
{{content-type: application/json; charset=UTF-8}}
The fix is relatively straightforward:
* In *PutElasticserachHttp.java*, remove the requirement of a compulsory
"Type" property:
{code:java}
public static final PropertyDescriptor TYPE = new PropertyDescriptor.Builder()
.name("put-es-type")
.displayName("Type")
.description("The type of this document (used by Elasticsearch < 7.0 for
indexing and searching). Leave empty for ES >= 7.0") // <-
.required(false) // <--------- CHANGE
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
.build();
{code}
* In *AbstractElasticsearchHttpProcessor.java*, check for the presence of
"docType". If not present, assume elasticsearch 7.x or above and omit from bulk
API URL:
{code:java}
protected void buildBulkCommand(StringBuilder sb, String index, String docType,
String indexOp, String id, String jsonString) {
if (indexOp.equalsIgnoreCase("index")) {
sb.append("{\"index\": { \"_index\": \"");
sb.append(StringEscapeUtils.escapeJson(index));
if (!(StringUtils.isEmpty(docType) | docType == null)){
sb.append("\", \"_type\": \"");
sb.append(StringEscapeUtils.escapeJson(docType));
sb.append("\"");
}
if (!StringUtils.isEmpty(id)) { // <--------- CHANGE START
sb.append(", \"_id\": \"");
sb.append(StringEscapeUtils.escapeJson(id));
sb.append("\"");
} // <--------- CHANGE END
sb.append("}}\n");
sb.append(jsonString);
sb.append("\n");
} else if (indexOp.equalsIgnoreCase("upsert") ||
indexOp.equalsIgnoreCase("update")) {
sb.append("{\"update\": { \"_index\": \"");
sb.append(StringEscapeUtils.escapeJson(index));
sb.append("\", \"_type\": \"");
sb.append(StringEscapeUtils.escapeJson(docType));
sb.append("\", \"_id\": \"");
sb.append(StringEscapeUtils.escapeJson(id));
sb.append("\" }\n");
sb.append("{\"doc\": ");
sb.append(jsonString);
sb.append(", \"doc_as_upsert\": ");
sb.append(indexOp.equalsIgnoreCase("upsert"));
sb.append(" }\n");
} else if (indexOp.equalsIgnoreCase("delete")) {
sb.append("{\"delete\": { \"_index\": \"");
sb.append(StringEscapeUtils.escapeJson(index));
sb.append("\", \"_type\": \"");
sb.append(StringEscapeUtils.escapeJson(docType));
sb.append("\", \"_id\": \"");
sb.append(StringEscapeUtils.escapeJson(id));
sb.append("\" }\n");
}
}
{code}
* The *TestPutElasticsearchHttp.java* test file needs to be updated to reflect
that now a requests without type is valid (it's currently marked as invalid)
was:
In ES 7.x and above, document "type" is no longer compulsory and in fact is
deprecated. When using the 1.9.2 version of PutElasticsearchHttp with ES v7.2,
it still works however you'll see the following HTTP in the response:
{{HTTP/1.1 200 OK}}
{{Warning: 299 Elasticsearch-7.2.0-508c38a "[types removal] Specifying types in
bulk requests is deprecated."}}
{{content-type: application/json; charset=UTF-8}}
The fix is relatively straightforward:
* In *PutElasticserachHttp.java*, remove the requirement of a compulsory
"Type" property:
{{public static final PropertyDescriptor TYPE = new
PropertyDescriptor.Builder()}}
{{ .name("put-es-type")}}
{{ .displayName("Type")}}
{{ .description(*"The type of this document (used by Elasticsearch < 7.0 for
indexing and searching). Leave empty for ES >= 7.0"*)}}
{{ *.required(false)*}}
{{ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)}}
{{ .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)}}
{{ .build();}}
* In *AbstractElasticsearchHttpProcessor.java*, check for the presence of
"docType". If not present, assume elasticsearch 7.x or above and omit from bulk
API URL:
{{protected void buildBulkCommand(StringBuilder sb, String index, String
docType, String indexOp, String id, String jsonString) {}}
{{ if (indexOp.equalsIgnoreCase("index")) {}}
{{ sb.append("{\"index\": { \"_index\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(index));}}
{{ *if (!(StringUtils.isEmpty(docType) | docType == null)){*}}
{{ *sb.append("\", \"_type\": \"");*}}
{{ *sb.append(StringEscapeUtils.escapeJson(docType));*}}
{{ *sb.append("\"");*}}
{{ *}*}}
{{ if (!StringUtils.isEmpty(id)) {}}
{{ sb.append(", \"_id\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(id));}}
{{ sb.append("\"");}}
{{ }}}
{{ sb.append("}}\n");}}
{{ sb.append(jsonString);}}
{{ sb.append("\n");}}
{{ } else if (indexOp.equalsIgnoreCase("upsert") ||
indexOp.equalsIgnoreCase("update")) {}}
{{ sb.append("{\"update\": { \"_index\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(index));}}
{{ sb.append("\", \"_type\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(docType));}}
{{ sb.append("\", \"_id\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(id));}}
{{ sb.append("\" }\n");}}
{{ sb.append("{\"doc\": ");}}
{{ sb.append(jsonString);}}
{{ sb.append(", \"doc_as_upsert\": ");}}
{{ sb.append(indexOp.equalsIgnoreCase("upsert"));}}
{{ sb.append(" }\n");}}
{{ } else if (indexOp.equalsIgnoreCase("delete")) {}}
{{ sb.append("{\"delete\": { \"_index\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(index));}}
{{ sb.append("\", \"_type\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(docType));}}
{{ sb.append("\", \"_id\": \"");}}
{{ sb.append(StringEscapeUtils.escapeJson(id));}}
{{ sb.append("\" }\n");}}
{{ }}}
{{}}}
* The *TestPutElasticsearchHttp.java* test file needs to be updated to reflect
that now a requests without type is valid (it's currently marked as invalid)
> PutElasticsearchHttp: Remove _type as being compulsory
> ------------------------------------------------------
>
> Key: NIFI-6404
> URL: https://issues.apache.org/jira/browse/NIFI-6404
> Project: Apache NiFi
> Issue Type: Improvement
> Components: Extensions
> Affects Versions: 1.9.2
> Environment: Elasticsearch 7.x
> Reporter: David Vassallo
> Priority: Major
>
> In ES 7.x and above, document "type" is no longer compulsory and in fact is
> deprecated. When using the 1.9.2 version of PutElasticsearchHttp with ES
> v7.2, it still works however you'll see the following HTTP in the response:
>
> {{HTTP/1.1 200 OK}}
> {{Warning: 299 Elasticsearch-7.2.0-508c38a "[types removal] Specifying types
> in bulk requests is deprecated."}}
> {{content-type: application/json; charset=UTF-8}}
>
> The fix is relatively straightforward:
> * In *PutElasticserachHttp.java*, remove the requirement of a compulsory
> "Type" property:
> {code:java}
> public static final PropertyDescriptor TYPE = new PropertyDescriptor.Builder()
> .name("put-es-type")
> .displayName("Type")
> .description("The type of this document (used by Elasticsearch < 7.0 for
> indexing and searching). Leave empty for ES >= 7.0") // <-
> .required(false) // <--------- CHANGE
> .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
> .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
> .build();
> {code}
>
> * In *AbstractElasticsearchHttpProcessor.java*, check for the presence of
> "docType". If not present, assume elasticsearch 7.x or above and omit from
> bulk API URL:
>
> {code:java}
> protected void buildBulkCommand(StringBuilder sb, String index, String
> docType, String indexOp, String id, String jsonString) {
> if (indexOp.equalsIgnoreCase("index")) {
> sb.append("{\"index\": { \"_index\": \"");
> sb.append(StringEscapeUtils.escapeJson(index));
> if (!(StringUtils.isEmpty(docType) | docType == null)){
> sb.append("\", \"_type\": \"");
> sb.append(StringEscapeUtils.escapeJson(docType));
> sb.append("\"");
> }
> if (!StringUtils.isEmpty(id)) { // <--------- CHANGE START
> sb.append(", \"_id\": \"");
> sb.append(StringEscapeUtils.escapeJson(id));
> sb.append("\"");
> } // <--------- CHANGE END
> sb.append("}}\n");
> sb.append(jsonString);
> sb.append("\n");
> } else if (indexOp.equalsIgnoreCase("upsert") ||
> indexOp.equalsIgnoreCase("update")) {
> sb.append("{\"update\": { \"_index\": \"");
> sb.append(StringEscapeUtils.escapeJson(index));
> sb.append("\", \"_type\": \"");
> sb.append(StringEscapeUtils.escapeJson(docType));
> sb.append("\", \"_id\": \"");
> sb.append(StringEscapeUtils.escapeJson(id));
> sb.append("\" }\n");
> sb.append("{\"doc\": ");
> sb.append(jsonString);
> sb.append(", \"doc_as_upsert\": ");
> sb.append(indexOp.equalsIgnoreCase("upsert"));
> sb.append(" }\n");
> } else if (indexOp.equalsIgnoreCase("delete")) {
> sb.append("{\"delete\": { \"_index\": \"");
> sb.append(StringEscapeUtils.escapeJson(index));
> sb.append("\", \"_type\": \"");
> sb.append(StringEscapeUtils.escapeJson(docType));
> sb.append("\", \"_id\": \"");
> sb.append(StringEscapeUtils.escapeJson(id));
> sb.append("\" }\n");
> }
> }
> {code}
>
> * The *TestPutElasticsearchHttp.java* test file needs to be updated to
> reflect that now a requests without type is valid (it's currently marked as
> invalid)
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
