This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-2.9 by this push:
new 9ad4ef2 [branch-2.9] [tests] Pulsar IO - Elasticsearch - reduce
elastic container memory usage and clean up test cases (#14631)
9ad4ef2 is described below
commit 9ad4ef2d3145346818f8398b5bb4500e13e51dd8
Author: Nicolò Boschi <[email protected]>
AuthorDate: Thu Mar 10 07:09:56 2022 +0100
[branch-2.9] [tests] Pulsar IO - Elasticsearch - reduce elastic container
memory usage and clean up test cases (#14631)
---
.../io/elasticsearch/ElasticSearchBWCTests.java | 2 +-
.../elasticsearch/ElasticSearchClientSslTests.java | 17 ++++------
.../io/elasticsearch/ElasticSearchClientTests.java | 19 +++++------
.../elasticsearch/ElasticSearchExtractTests.java | 39 ++++++++--------------
.../ElasticSearchSinkRawDataTests.java | 13 +++-----
.../io/elasticsearch/ElasticSearchSinkTests.java | 9 ++---
.../io/elasticsearch/ElasticSearchTestBase.java | 36 ++++++++++++++++++++
7 files changed, 72 insertions(+), 63 deletions(-)
diff --git
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java
index 7b24a7c..86b9df3 100644
---
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java
+++
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java
@@ -23,7 +23,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.client.api.schema.GenericObject;
import org.apache.pulsar.common.schema.SchemaType;
import org.apache.pulsar.functions.api.Record;
-import org.junit.Test;
+import org.testng.annotations.Test;
import java.nio.charset.StandardCharsets;
diff --git
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java
index 7cbd249..325f69c 100644
---
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java
+++
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java
@@ -25,17 +25,12 @@ import org.testng.annotations.Test;
import java.io.IOException;
import java.time.Duration;
-import java.util.Optional;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
// see
https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#ssl-tls-settings
-public class ElasticSearchClientSslTests {
-
- public static final String ELASTICSEARCH_IMAGE =
Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
-
.orElse("docker.elastic.co/elasticsearch/elasticsearch:7.10.2-amd64");
+public class ElasticSearchClientSslTests extends ElasticSearchTestBase {
final static String INDEX = "myindex";
@@ -44,7 +39,7 @@ public class ElasticSearchClientSslTests {
@Test
public void testSslBasic() throws IOException {
- try(ElasticsearchContainer container = new
ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ try (ElasticsearchContainer container = createElasticsearchContainer()
.withCreateContainerCmdModifier(c ->
c.withName("elasticsearch"))
.withFileSystemBind(sslResourceDir, configDir + "/ssl")
.withEnv("ELASTIC_PASSWORD","elastic") // boostrap password
@@ -80,7 +75,7 @@ public class ElasticSearchClientSslTests {
@Test
public void testSslWithHostnameVerification() throws IOException {
- try(ElasticsearchContainer container = new
ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ try (ElasticsearchContainer container = createElasticsearchContainer()
.withCreateContainerCmdModifier(c ->
c.withName("elasticsearch"))
.withFileSystemBind(sslResourceDir, configDir + "/ssl")
.withEnv("ELASTIC_PASSWORD","elastic") // boostrap password
@@ -119,7 +114,7 @@ public class ElasticSearchClientSslTests {
@Test
public void testSslWithClientAuth() throws IOException {
- try(ElasticsearchContainer container = new
ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ try(ElasticsearchContainer container = createElasticsearchContainer()
.withCreateContainerCmdModifier(c ->
c.withName("elasticsearch"))
.withFileSystemBind(sslResourceDir, configDir + "/ssl")
.withEnv("ELASTIC_PASSWORD","elastic") // boostrap password
diff --git
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java
index a831d5c..1a505da 100644
---
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java
+++
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java
@@ -26,9 +26,9 @@ import
org.apache.pulsar.io.elasticsearch.testcontainers.ChaosContainer;
import org.awaitility.Awaitility;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
-import org.junit.AfterClass;
import org.mockito.Mockito;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -36,29 +36,28 @@ import java.io.IOException;
import java.util.Optional;
import java.util.UUID;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.verify;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.*;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
@Slf4j
-public class ElasticSearchClientTests {
-
- public static final String ELASTICSEARCH_IMAGE =
Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
-
.orElse("docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2-amd64");
+public class ElasticSearchClientTests extends ElasticSearchTestBase {
static ElasticsearchContainer container;
@BeforeClass
public static final void initBeforeClass() throws IOException {
- container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE);
+ container = createElasticsearchContainer();
container.start();
}
- @AfterClass
+ @AfterClass(alwaysRun = true)
public static void closeAfterClass() {
container.close();
}
diff --git
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
index 89d686c..b0d26d9 100644
---
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
+++
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
@@ -18,8 +18,10 @@
*/
package org.apache.pulsar.io.elasticsearch;
-import com.google.common.collect.ImmutableList;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
import com.google.common.collect.ImmutableMap;
+import java.util.Optional;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.schema.GenericObject;
@@ -31,33 +33,18 @@ import org.apache.pulsar.common.schema.KeyValue;
import org.apache.pulsar.common.schema.KeyValueEncodingType;
import org.apache.pulsar.common.schema.SchemaType;
import org.apache.pulsar.functions.api.Record;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
-import java.util.Collection;
-import java.util.Optional;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNull;
-
-@RunWith(Parameterized.class)
public class ElasticSearchExtractTests {
- SchemaType schemaType;
-
- @Parameters
- public static Collection schemaTypes() {
- return ImmutableList.of(SchemaType.JSON, SchemaType.AVRO);
- }
-
- public ElasticSearchExtractTests(SchemaType schemaType) {
- this.schemaType = schemaType;
+ @DataProvider(name = "schemaType")
+ public Object[] schemaType() {
+ return new Object[]{SchemaType.JSON, SchemaType.AVRO};
}
- @Test
- public void testGenericRecord() throws Exception {
+ @Test(dataProvider = "schemaType")
+ public void testGenericRecord(SchemaType schemaType) throws Exception {
RecordSchemaBuilder valueSchemaBuilder =
org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
@@ -154,8 +141,8 @@ public class ElasticSearchExtractTests {
assertNull(pair4.getRight());
}
- @Test
- public void testKeyValueGenericRecord() throws Exception {
+ @Test(dataProvider = "schemaType")
+ public void testKeyValueGenericRecord(SchemaType schemaType) throws
Exception {
RecordSchemaBuilder keySchemaBuilder =
org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null);
@@ -273,4 +260,4 @@ public class ElasticSearchExtractTests {
assertEquals(pair3.getLeft(), "[\"1\",1]");
assertNull(pair3.getRight());
}
-}
+}
\ No newline at end of file
diff --git
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java
index f195a38..018a11e 100644
---
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java
+++
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java
@@ -23,11 +23,11 @@ import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.schema.GenericObject;
import org.apache.pulsar.functions.api.Record;
import org.apache.pulsar.io.core.SinkContext;
-import org.junit.AfterClass;
import org.mockito.Mock;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
@@ -42,13 +42,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNull;
-public class ElasticSearchSinkRawDataTests {
-
- public static final String ELASTICSEARCH_IMAGE =
Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
-
.orElse("docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2-amd64");
+public class ElasticSearchSinkRawDataTests extends ElasticSearchTestBase {
private static ElasticsearchContainer container;
@@ -67,11 +62,11 @@ public class ElasticSearchSinkRawDataTests {
@BeforeClass
public static final void initBeforeClass() {
- container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE);
+ container = createElasticsearchContainer();
schema = Schema.BYTES;
}
- @AfterClass
+ @AfterClass(alwaysRun = true)
public static void closeAfterClass() {
container.close();
}
diff --git
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java
index c752354..c8c500a 100644
---
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java
+++
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java
@@ -54,10 +54,7 @@ import org.testng.annotations.Test;
import java.util.Locale;
import static org.testng.Assert.assertNull;
-public class ElasticSearchSinkTests {
-
- public static final String ELASTICSEARCH_IMAGE =
Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
-
.orElse("docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2-amd64");
+public class ElasticSearchSinkTests extends ElasticSearchTestBase {
private static ElasticsearchContainer container;
@@ -76,7 +73,7 @@ public class ElasticSearchSinkTests {
@BeforeClass
public static final void initBeforeClass() {
- container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE);
+ container = createElasticsearchContainer();
valueSchema = Schema.JSON(UserProfile.class);
genericSchema = Schema.generic(valueSchema.getSchemaInfo());
@@ -89,7 +86,7 @@ public class ElasticSearchSinkTests {
}
- @AfterClass
+ @AfterClass(alwaysRun = true)
public static void closeAfterClass() {
container.close();
}
diff --git
a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java
new file mode 100644
index 0000000..34e3fc2
--- /dev/null
+++
b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.elasticsearch;
+
+import org.testcontainers.elasticsearch.ElasticsearchContainer;
+
+import java.util.Optional;
+
+public class ElasticSearchTestBase {
+
+ private static final String ELASTICSEARCH_IMAGE =
Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
+
.orElse("docker.elastic.co/elasticsearch/elasticsearch:7.16.3-amd64");
+
+ protected static ElasticsearchContainer createElasticsearchContainer() {
+ return new ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m");
+
+ }
+
+}
