This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch backport/29734-to-camel-4-22-x in repository https://gitbox.apache.org/repos/asf/camel.git
commit 81a6760d6456187bed4b21fb66ceae884bd131d8 Author: Claus Ibsen <[email protected]> AuthorDate: Thu Nov 12 21:22:02 2026 +0100 CAMEL-26283: camel-elasticsearch-rest-client - send the document id and the index name as one encoded path segment (#29734) The producer put the index name and the document id into the request path as they were, and the low-level client parses that path as a URI. A '/', '?' or '#' in the value changed the path of the request, and a space or '%' made it fail. Both values are now percent-encoded as one path segment each. A document id of '.' or '..' is rejected, and an index name is checked against the characters Elasticsearch does not allow in index names; a comma separated list of names and patterns still works. GET_BY_ID returns no document when the response has no _source, instead of failing with a NullPointerException. (cherry picked from commit ddeaef6b5b075f51835f7441f6e843ceb8b1c874) Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../docs/elasticsearch-rest-client-component.adoc | 6 + .../docs/elasticsearch-rest-client-component.adoc | 6 + .../client/ElasticsearchRestClientProducer.java | 61 +++++- .../ElasticsearchRestClientRequestPathTest.java | 212 +++++++++++++++++++++ .../ElasticsearchRestClientComponentIT.java | 37 ++++ .../ElasticsearchRestClientITSupport.java | 11 +- 6 files changed, 321 insertions(+), 12 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/elasticsearch-rest-client-component.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/elasticsearch-rest-client-component.adoc index 18b388918d13..3efc4489dd5d 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/elasticsearch-rest-client-component.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/elasticsearch-rest-client-component.adoc @@ -73,6 +73,12 @@ You can set the message exchange body to a `Map` of `String` keys & values for t |=== +The document id and the index name are each sent as one path segment of the request, with reserved characters such as +`/`, `?`, `#`, `%` and spaces percent-encoded, so Elasticsearch takes them literally. A document id of `.` or `..` is +rejected, as is an index name that contains a character Elasticsearch does not allow in an index name (`\`, `/`, `?`, +`"`, `<`, `>`, `|`, `#` or a space). A comma separated list of index names and patterns, such as `logs-*,my_index`, can +be used to search several indexes. + == Examples === Index Content Example diff --git a/components/camel-elasticsearch-rest-client/src/main/docs/elasticsearch-rest-client-component.adoc b/components/camel-elasticsearch-rest-client/src/main/docs/elasticsearch-rest-client-component.adoc index 18b388918d13..3efc4489dd5d 100644 --- a/components/camel-elasticsearch-rest-client/src/main/docs/elasticsearch-rest-client-component.adoc +++ b/components/camel-elasticsearch-rest-client/src/main/docs/elasticsearch-rest-client-component.adoc @@ -73,6 +73,12 @@ You can set the message exchange body to a `Map` of `String` keys & values for t |=== +The document id and the index name are each sent as one path segment of the request, with reserved characters such as +`/`, `?`, `#`, `%` and spaces percent-encoded, so Elasticsearch takes them literally. A document id of `.` or `..` is +rejected, as is an index name that contains a character Elasticsearch does not allow in an index name (`\`, `/`, `?`, +`"`, `<`, `>`, `|`, `#` or a space). A comma separated list of index names and patterns, such as `logs-*,my_index`, can +be used to search several indexes. + == Examples === Index Content Example diff --git a/components/camel-elasticsearch-rest-client/src/main/java/org/apache/camel/component/elasticsearch/rest/client/ElasticsearchRestClientProducer.java b/components/camel-elasticsearch-rest-client/src/main/java/org/apache/camel/component/elasticsearch/rest/client/ElasticsearchRestClientProducer.java index 401e78593f90..d3cf72a4d57a 100644 --- a/components/camel-elasticsearch-rest-client/src/main/java/org/apache/camel/component/elasticsearch/rest/client/ElasticsearchRestClientProducer.java +++ b/components/camel-elasticsearch-rest-client/src/main/java/org/apache/camel/component/elasticsearch/rest/client/ElasticsearchRestClientProducer.java @@ -18,6 +18,8 @@ package org.apache.camel.component.elasticsearch.rest.client; import java.io.IOException; import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.security.KeyStore; import java.security.cert.Certificate; import java.security.cert.CertificateFactory; @@ -68,6 +70,9 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { public static final String POST = "POST"; public static final String GET = "GET"; + // the characters that an index name must not contain, besides the comma and the wildcard of a list or pattern + private static final String INVALID_INDEX_NAME_CHARS = "\\/?\"<>|# "; + private ElasticsearchRestClientEndpoint endpoint; private RestClient restClient; private boolean createdRestClient; @@ -205,7 +210,7 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { * Generate the Request for operation CREATE_INDEX */ private Request createIndexRequest(String indexName, Exchange exchange) { - var endpoint = String.format("/%s", indexName); + var endpoint = "/" + indexPath(indexName); var request = new Request(PUT, endpoint); var additionalParameters = exchange.getMessage().getHeader(ElasticSearchRestClientConstant.INDEX_SETTINGS, String.class); @@ -219,7 +224,7 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { * Generate the Request for operation DELETE_INDEX */ private Request deleteIndexRequest(String indexName) { - var endpoint = String.format("/%s", indexName); + var endpoint = "/" + indexPath(indexName); return new Request(DELETE, endpoint); } @@ -229,12 +234,12 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { private Request indexRequest(String indexName, Exchange exchange) throws InvalidPayloadException { var jsonBody = exchange.getMessage().getMandatoryBody(String.class); - var endpoint = String.format("/%s/_doc", indexName); + var endpoint = "/" + indexPath(indexName) + "/_doc"; // Gets the id of the document if specified - var id = exchange.getMessage().getHeader(ElasticSearchRestClientConstant.ID); + var id = exchange.getMessage().getHeader(ElasticSearchRestClientConstant.ID, String.class); if (ObjectHelper.isNotEmpty(id)) { - endpoint = String.format("%s/%s", endpoint, id); + endpoint = endpoint + "/" + documentIdPath(id); } Request request = new Request(POST, endpoint); @@ -255,7 +260,7 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { "id value is mandatory when performing GET_BY_ID operation"); } } - var endpoint = String.format("/%s/_doc/%s", indexName, id); + var endpoint = "/" + indexPath(indexName) + "/_doc/" + documentIdPath(id); return new Request(GET, endpoint); } @@ -271,7 +276,7 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { "id value is mandatory when performing DELETE operation"); } } - var endpoint = String.format("/%s/_doc/%s", indexName, id); + var endpoint = "/" + indexPath(indexName) + "/_doc/" + documentIdPath(id); return new Request(DELETE, endpoint); } @@ -279,7 +284,7 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { * Generate the Request for operation SEARCH */ private Request search(String indexName, Exchange exchange) { - var endpoint = String.format("/%s/_search", indexName); + var endpoint = "/" + indexPath(indexName) + "/_search"; // Create Request : at this stage this one get all results Request request = new Request(GET, endpoint); @@ -299,6 +304,40 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { return request; } + /** + * The index name, or the comma separated list of index names, as one path segment of the request. Each name is + * checked against the characters Elasticsearch does not allow in an index name, except the wildcard ({@code *}) of + * an index pattern. + */ + private static String indexPath(String indexName) { + for (String name : indexName.split(",", -1)) { + if (name.isEmpty() || ".".equals(name) || "..".equals(name) + || name.chars().anyMatch(c -> INVALID_INDEX_NAME_CHARS.indexOf(c) >= 0)) { + throw new IllegalArgumentException( + "Invalid index name: " + indexName + " (an index name must not be empty, . or .., and must not contain" + + " \\ / ? \" < > | # or a space)"); + } + } + return pathSegment(indexName); + } + + /** + * The document id as one path segment of the request + */ + private static String documentIdPath(String id) { + if (".".equals(id) || "..".equals(id)) { + throw new IllegalArgumentException("Invalid document id: " + id); + } + return pathSegment(id); + } + + /** + * Encodes the value, so it is sent as one path segment, with the reserved characters percent-encoded + */ + private static String pathSegment(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8).replace("+", "%20"); + } + /** * creates an Elasticsearch Json Query based on matches */ @@ -349,11 +388,15 @@ public class ElasticsearchRestClientProducer extends DefaultAsyncProducer { String extractDocument(JsonObject doc) { // check if document exists Boolean found = doc.getBoolean("found"); - if (!found) { + if (!Boolean.TRUE.equals(found)) { return null; } // if it exists extract document Map<String, Object> map = doc.getMap("_source"); + if (map == null) { + // a response without the source of the document + return null; + } JsonObject jsonObject = new JsonObject(map); return jsonObject.toJson(); diff --git a/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/ElasticsearchRestClientRequestPathTest.java b/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/ElasticsearchRestClientRequestPathTest.java new file mode 100644 index 000000000000..6faf19e3bf65 --- /dev/null +++ b/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/ElasticsearchRestClientRequestPathTest.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.elasticsearch.rest.client; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Stream; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.apache.camel.impl.DefaultCamelContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The path of the request the producer sends: the document id and the index name are each one path segment, checked + * against a node that records the raw request and answers it as Elasticsearch does. + */ +class ElasticsearchRestClientRequestPathTest { + + private static final String DOCUMENT_ID = "a/b?c#d e%"; + private static final String ENCODED_DOCUMENT_ID = "a%2Fb%3Fc%23d%20e%25"; + + private final List<Recorded> requests = new CopyOnWriteArrayList<>(); + private HttpServer node; + private CamelContext context; + private ProducerTemplate template; + private String uri; + + @BeforeEach + void start() throws Exception { + node = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + node.createContext("/", this::answer); + node.start(); + uri = "elasticsearch-rest-client:camel?hostAddressesList=127.0.0.1:" + node.getAddress().getPort(); + context = new DefaultCamelContext(); + context.start(); + template = context.createProducerTemplate(); + } + + @AfterEach + void stop() { + context.stop(); + node.stop(0); + } + + static Stream<Arguments> documentIds() { + return Stream.of( + Arguments.of("a/b", "a%2Fb"), + Arguments.of("1?x=y", "1%3Fx%3Dy"), + Arguments.of("1#2", "1%232"), + Arguments.of("a b", "a%20b"), + Arguments.of("100%", "100%25"), + Arguments.of("café", "caf%C3%A9"), + Arguments.of(DOCUMENT_ID, ENCODED_DOCUMENT_ID)); + } + + @ParameterizedTest + @MethodSource("documentIds") + void getByIdSendsTheIdAsOnePathSegment(String id, String encoded) { + String document = template.requestBodyAndHeader(uri + "&operation=GET_BY_ID&indexName=my_index", null, + ElasticSearchRestClientConstant.ID, id, String.class); + + assertEquals("{\"title\":\"camel\"}", document); + assertEquals("/my_index/_doc/" + encoded, single().rawPath()); + assertNull(single().rawQuery()); + } + + @Test + void getByIdSendsTheIdOfTheBodyAsOnePathSegment() { + template.requestBody(uri + "&operation=GET_BY_ID&indexName=my_index", DOCUMENT_ID, String.class); + + assertEquals("/my_index/_doc/" + ENCODED_DOCUMENT_ID, single().rawPath()); + assertNull(single().rawQuery()); + } + + @Test + void indexSendsTheIdAsOnePathSegment() { + String id = template.requestBodyAndHeader(uri + "&operation=INDEX_OR_UPDATE&indexName=my_index", + "{\"title\":\"camel\"}", ElasticSearchRestClientConstant.ID, DOCUMENT_ID, String.class); + + assertEquals("1", id); + assertEquals("POST", single().method()); + assertEquals("/my_index/_doc/" + ENCODED_DOCUMENT_ID, single().rawPath()); + assertNull(single().rawQuery()); + } + + @Test + void indexWithoutIdStillLetsTheServerChooseTheId() { + template.requestBody(uri + "&operation=INDEX_OR_UPDATE&indexName=my_index", "{\"title\":\"camel\"}", String.class); + + assertEquals("/my_index/_doc", single().rawPath()); + } + + @Test + void deleteSendsTheIdAsOnePathSegment() { + Object deleted = template.requestBodyAndHeader(uri + "&operation=DELETE&indexName=my_index", null, + ElasticSearchRestClientConstant.ID, DOCUMENT_ID); + + assertEquals(Boolean.TRUE, deleted); + assertEquals("DELETE", single().method()); + assertEquals("/my_index/_doc/" + ENCODED_DOCUMENT_ID, single().rawPath()); + assertNull(single().rawQuery()); + } + + @ParameterizedTest + @ValueSource(strings = { ".", ".." }) + void dotSegmentDocumentIdIsRejected(String id) { + Exchange exchange = template.request(uri + "&operation=GET_BY_ID&indexName=my_index", + e -> e.getMessage().setHeader(ElasticSearchRestClientConstant.ID, id)); + + assertInstanceOf(IllegalArgumentException.class, exchange.getException()); + assertTrue(requests.isEmpty(), "requests sent: " + requests); + } + + @Test + void indexNamesAndPatternsOfASearchAreSentAsOnePathSegment() { + template.requestBodyAndHeader(uri + "&operation=SEARCH", null, ElasticSearchRestClientConstant.INDEX_NAME, + "logs-*,my_index", String.class); + + assertEquals("/logs-*%2Cmy_index/_search", single().rawPath()); + // the node decodes the segment back to the list of index names and patterns + assertEquals("/logs-*,my_index/_search", single().path()); + } + + @ParameterizedTest + @ValueSource(strings = { "my index", "a/b", "a?b", "a#b", "a\\b", "a|b", "a\"b", "<a>", ".", "..", "a,,b", "a," }) + void indexNameWithCharactersElasticsearchDoesNotAllowIsRejected(String indexName) { + Exchange exchange = template.request(uri + "&operation=GET_BY_ID", e -> { + e.getMessage().setHeader(ElasticSearchRestClientConstant.INDEX_NAME, indexName); + e.getMessage().setHeader(ElasticSearchRestClientConstant.ID, "1"); + }); + + assertInstanceOf(IllegalArgumentException.class, exchange.getException()); + assertTrue(requests.isEmpty(), "requests sent: " + requests); + } + + @Test + void getByIdResponseWithoutSourceReturnsNoDocument() { + Exchange exchange = template.request(uri + "&operation=GET_BY_ID&indexName=my_index", + e -> e.getMessage().setHeader(ElasticSearchRestClientConstant.ID, "nosource")); + + assertNull(exchange.getException()); + assertNull(exchange.getMessage().getBody()); + } + + private Recorded single() { + assertEquals(1, requests.size(), "requests sent: " + requests); + return requests.get(0); + } + + private void answer(HttpExchange exchange) throws IOException { + exchange.getRequestBody().readAllBytes(); + String method = exchange.getRequestMethod(); + String rawPath = exchange.getRequestURI().getRawPath(); + requests.add(new Recorded( + method, rawPath, exchange.getRequestURI().getPath(), + exchange.getRequestURI().getRawQuery())); + String response; + if (rawPath.endsWith("/_search")) { + response = "{\"hits\":{\"hits\":[{\"_source\":{\"title\":\"camel\"}}]}}"; + } else if (rawPath.endsWith("/_doc/nosource")) { + response = "{\"_index\":\"my_index\",\"_id\":\"nosource\",\"found\":true}"; + } else if ("DELETE".equals(method)) { + response = "{\"_index\":\"my_index\",\"_id\":\"1\",\"result\":\"deleted\"}"; + } else if ("POST".equals(method)) { + response = "{\"_index\":\"my_index\",\"_id\":\"1\",\"result\":\"created\"}"; + } else { + response = "{\"_index\":\"my_index\",\"_id\":\"1\",\"found\":true,\"_source\":{\"title\":\"camel\"}}"; + } + byte[] bytes = response.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + } + + private record Recorded(String method, String rawPath, String path, String rawQuery) { + } +} diff --git a/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientComponentIT.java b/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientComponentIT.java index 6b49b5b14395..dbb1c37ca3bd 100644 --- a/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientComponentIT.java +++ b/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientComponentIT.java @@ -74,6 +74,18 @@ public class ElasticsearchRestClientComponentIT extends ElasticsearchRestClientI .setHeader(ElasticSearchRestClientConstant.OPERATION).constant(ElasticsearchRestClientOperation.DELETE) .to("elasticsearch-rest-client:my-cluster?restClient=#restClient&indexName=my_index"); + from("direct:index-path-segment") + .to("elasticsearch-rest-client:my-cluster?operation=INDEX_OR_UPDATE&restClient=#restClient&indexName=path_segment_index"); + + from("direct:get-by-id-path-segment") + .to("elasticsearch-rest-client:my-cluster?operation=GET_BY_ID&restClient=#restClient&indexName=path_segment_index"); + + from("direct:search-path-segment") + .to("elasticsearch-rest-client:my-cluster?operation=SEARCH&restClient=#restClient&indexName=path_segment_*,path_segment_index"); + + from("direct:delete-path-segment") + .to("elasticsearch-rest-client:my-cluster?operation=DELETE&restClient=#restClient&indexName=path_segment_index"); + from("direct:errorHandler") .log("Handling exception: ${exception.message}") .to("mock:errorHandler"); @@ -265,6 +277,31 @@ public class ElasticsearchRestClientComponentIT extends ElasticsearchRestClientI assertTrue(ack.get()); } + @Test + void documentWhoseIdContainsReservedCharacters() throws Exception { + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + Response health = restClient.performRequest(new Request("GET", "/_cluster/health")); + assertEquals(200, health.getStatusLine().getStatusCode()); + }); + + var id = "a/b?c#d e%"; + var document = "{\"title\": \"Elastic with reserved characters in the id\"}"; + + assertEquals(id, template.requestBodyAndHeader("direct:index-path-segment", document, + ElasticSearchRestClientConstant.ID, id, String.class)); + + assertEquals("{\"title\":\"Elastic with reserved characters in the id\"}", + template.requestBody("direct:get-by-id-path-segment", id, String.class)); + + // the comma separated list of an index pattern and an index name still searches both + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> assertEquals( + "[{\"title\":\"Elastic with reserved characters in the id\"}]", + template.requestBody("direct:search-path-segment", null, String.class))); + + assertTrue(template.requestBodyAndHeader("direct:delete-path-segment", null, ElasticSearchRestClientConstant.ID, id, + Boolean.class)); + } + @Test void operationUndefined() { Exchange exchange = template.request("elasticsearch-rest-client:my-cluster?restClient=#restClient&indexName=my_index", diff --git a/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientITSupport.java b/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientITSupport.java index 246ac6ed2414..9017a168d08a 100644 --- a/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientITSupport.java +++ b/components/camel-elasticsearch-rest-client/src/test/java/org/apache/camel/component/elasticsearch/rest/client/integration/ElasticsearchRestClientITSupport.java @@ -26,6 +26,7 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; @@ -44,6 +45,10 @@ public class ElasticsearchRestClientITSupport extends CamelTestSupport { @Override protected void setupResources() throws Exception { super.setupResources(); + if (restClient != null) { + // one client for all the tests of the class, as the routes of the class are bound to it + return; + } String scheme = service.getSslContext().isPresent() ? "https" : "http"; HttpHost host = new HttpHost(service.getElasticSearchHost(), service.getPort(), scheme); @@ -62,11 +67,11 @@ public class ElasticsearchRestClientITSupport extends CamelTestSupport { restClient = builder.build(); } - @Override - protected void cleanupResources() throws Exception { - super.cleanupResources(); + @AfterAll + void closeRestClient() throws Exception { if (restClient != null) { restClient.close(); + restClient = null; } }
