Repository: beam Updated Branches: refs/heads/master bd0facc46 -> 5ea2537ec
[BEAM-2957] Fix flaky ElasticsearchIOTest.testSplit in beam-sdks-java-io-elasticsearch-tests-5 ES Slices are based on document id. For them to be deterministic (no empty slice), ids need to be always the same. Provide document id at insertion time rather than letting Elasticsearch generate one. Fix assert message in testSplit Unrelated: Update ES rest client from 5.0 to 5.4 Remove unneeded jna version property Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0bb0bb54 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0bb0bb54 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0bb0bb54 Branch: refs/heads/master Commit: 0bb0bb54f882264e5aa3aac026d18f460d1058ec Parents: 307b036 Author: Etienne Chauchot <[email protected]> Authored: Fri Sep 15 15:36:18 2017 +0200 Committer: Etienne Chauchot <[email protected]> Committed: Mon Sep 18 11:10:23 2017 +0200 ---------------------------------------------------------------------- .../apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java | 2 +- .../apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java | 2 +- .../io/elasticsearch-tests/elasticsearch-tests-common/pom.xml | 1 - .../beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java | 5 ++++- sdks/java/io/elasticsearch-tests/pom.xml | 2 +- sdks/java/io/elasticsearch/pom.xml | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/0bb0bb54/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java index 28ffa02..06298cd 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java @@ -180,6 +180,6 @@ public class ElasticsearchIOTest implements Serializable { nonEmptySplits += 1; } } - assertEquals("Wrong number of empty splits", expectedNumSplits, nonEmptySplits); + assertEquals("Wrong number of non empty splits", expectedNumSplits, nonEmptySplits); } } http://git-wip-us.apache.org/repos/asf/beam/blob/0bb0bb54/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java index 362f0a8..9a7eb07 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java @@ -179,6 +179,6 @@ public class ElasticsearchIOTest extends ESIntegTestCase implements Serializable nonEmptySplits += 1; } } - assertEquals("Wrong number of empty splits", expectedNumSources, nonEmptySplits); + assertEquals("Wrong number of non empty splits", expectedNumSources, nonEmptySplits); } } http://git-wip-us.apache.org/repos/asf/beam/blob/0bb0bb54/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml index 4381eeb..1732529 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml @@ -33,7 +33,6 @@ <properties> <commons-io.version>1.3.2</commons-io.version> - <jna.version>4.1.0</jna.version> <log4j.version>2.6.2</log4j.version> <elasticsearch.client.rest.version>5.0.0</elasticsearch.client.rest.version> <httpcomponents.core.version>4.4.5</httpcomponents.core.version> http://git-wip-us.apache.org/repos/asf/beam/blob/0bb0bb54/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java index 2f6ac3e..142789b 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java @@ -61,8 +61,11 @@ class ElasticSearchIOTestUtils { ElasticSearchIOTestUtils.createDocuments( numDocs, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS); StringBuilder bulkRequest = new StringBuilder(); + int i = 0; for (String document : data) { - bulkRequest.append(String.format("{ \"index\" : {} }%n%s%n", document)); + bulkRequest.append(String.format( + "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%d\" } }%n%s%n", + connectionConfiguration.getIndex(), connectionConfiguration.getType(), i++, document)); } String endPoint = String.format("/%s/%s/_bulk", connectionConfiguration.getIndex(), connectionConfiguration.getType()); http://git-wip-us.apache.org/repos/asf/beam/blob/0bb0bb54/sdks/java/io/elasticsearch-tests/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/elasticsearch-tests/pom.xml b/sdks/java/io/elasticsearch-tests/pom.xml index 4d947be..a4a1e2a 100644 --- a/sdks/java/io/elasticsearch-tests/pom.xml +++ b/sdks/java/io/elasticsearch-tests/pom.xml @@ -35,7 +35,7 @@ <commons-io.version>1.3.2</commons-io.version> <jna.version>4.1.0</jna.version> <log4j.version>2.6.2</log4j.version> - <elasticsearch.client.rest.version>5.0.0</elasticsearch.client.rest.version> + <elasticsearch.client.rest.version>5.4.0</elasticsearch.client.rest.version> </properties> <dependencies> http://git-wip-us.apache.org/repos/asf/beam/blob/0bb0bb54/sdks/java/io/elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/elasticsearch/pom.xml b/sdks/java/io/elasticsearch/pom.xml index 6624c46..4942d4f 100644 --- a/sdks/java/io/elasticsearch/pom.xml +++ b/sdks/java/io/elasticsearch/pom.xml @@ -33,7 +33,7 @@ <description>IO to read and write on Elasticsearch</description> <properties> - <elasticsearch.client.rest.version>5.0.0</elasticsearch.client.rest.version> + <elasticsearch.client.rest.version>5.4.0</elasticsearch.client.rest.version> <httpcomponents.core.version>4.4.5</httpcomponents.core.version> <httpcomponents.httpasyncclient.version>4.1.2</httpcomponents.httpasyncclient.version> <httpcomponents.httpclient.version>4.5.2</httpcomponents.httpclient.version>
