This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push: new 5c21b33 Compaction CLI integration test (#1572) 5c21b33 is described below commit 5c21b338b27b043a27d221ad9b252f8d346e786e Author: Ivan Kelly <iv...@apache.org> AuthorDate: Fri Apr 13 20:27:45 2018 +0200 Compaction CLI integration test (#1572) Integration test to exercise CLI based compaction from end to end. --- .../single-cluster-3-bookie-2-broker.yaml | 179 +++++++++++++++++++++ tests/integration/{ => compaction}/pom.xml | 29 ++-- .../pulsar/tests/integration/TestCompaction.java | 103 ++++++++++++ .../src/test/resources/arquillian.xml} | 28 ++-- tests/integration/pom.xml | 1 + 5 files changed, 313 insertions(+), 27 deletions(-) diff --git a/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml b/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml new file mode 100644 index 0000000..dd12abd --- /dev/null +++ b/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml @@ -0,0 +1,179 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +networks: + pulsarnet*: + driver: bridge + +zookeeper*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + env: [ZOOKEEPER_SERVERS=zookeeper] + labels: + cluster: test + service: zookeeper + entryPoint: [bin/run-local-zk.sh] + aliases: + - zookeeper + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + +global-zookeeper*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + env: [ZOOKEEPER_SERVERS=zookeeper] + labels: + cluster: test + service: global-zookeeper + entryPoint: [bin/run-global-zk.sh] + aliases: + - global-zookeeper + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + +init*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + env: + - cluster=test + - zkServers=zookeeper + - globalZkServers=global-zookeeper:2184 + - pulsarNode=pulsar-broker1 + labels: + cluster: test + service: init + entryPoint: [bin/init-cluster.sh] + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.LogToTargetDirStopAction + networkMode: pulsarnet* + +bookkeeper1*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + env: + - zkServers=zookeeper + - useHostNameAsBookieID=true + labels: + cluster: test + service: bookie + entryPoint: [bin/run-bookie.sh] + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + +bookkeeper2*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + env: + - zkServers=zookeeper + - useHostNameAsBookieID=true + labels: + cluster: test + service: bookie + entryPoint: [bin/run-bookie.sh] + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + +bookkeeper3*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + env: + - zkServers=zookeeper + - useHostNameAsBookieID=true + labels: + cluster: test + service: bookie + entryPoint: [bin/run-bookie.sh] + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + +pulsar-broker1*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + aliases: + - pulsar-broker1 + env: + - zookeeperServers=zookeeper + - globalZookeeperServers=global-zookeeper:2184 + - clusterName=test + labels: + cluster: test + service: pulsar-broker + entryPoint: [bin/run-broker.sh] + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + +pulsar-broker2*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + aliases: + - pulsar-broker2 + env: + - zookeeperServers=zookeeper + - globalZookeeperServers=global-zookeeper:2184 + - clusterName=test + labels: + cluster: test + service: pulsar-broker + entryPoint: [bin/run-broker.sh] + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + +pulsar-proxy*: + image: apachepulsar/pulsar-test-latest-version:latest + await: + strategy: org.apache.pulsar.tests.NoopAwaitStrategy + aliases: + - pulsar-broker2 + env: + - zookeeperServers=zookeeper + - globalZookeeperServers=global-zookeeper:2184 + - clusterName=test + labels: + cluster: test + service: pulsar-proxy + entryPoint: [bin/run-proxy.sh] + beforeStop: + - customBeforeStopAction: + strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction + networkMode: pulsarnet* + + diff --git a/tests/integration/pom.xml b/tests/integration/compaction/pom.xml similarity index 57% copy from tests/integration/pom.xml copy to tests/integration/compaction/pom.xml index a2e36d4..1eee545 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/compaction/pom.xml @@ -19,20 +19,29 @@ under the License. --> -<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> - <packaging>pom</packaging> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation=" + http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.pulsar.tests</groupId> - <artifactId>tests-parent</artifactId> + <artifactId>integration-tests-base</artifactId> <version>2.0.0-incubating-SNAPSHOT</version> + <relativePath>../../integration-tests-base</relativePath> </parent> - <groupId>org.apache.pulsar.tests</groupId> - <artifactId>integration</artifactId> - <name>Apache Pulsar :: Tests :: Integration</name> - <modules> - <module>smoke</module> - </modules> + <groupId>org.apache.pulsar.tests.integration</groupId> + <artifactId>compaction</artifactId> + <packaging>jar</packaging> + <name>Apache Pulsar :: Tests :: Integration Tests :: Topic Compaction</name> + + <dependencies> + <dependency> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar-client</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> </project> diff --git a/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java new file mode 100644 index 0000000..6a6130f --- /dev/null +++ b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration; + +import com.github.dockerjava.api.DockerClient; + +import java.util.concurrent.TimeUnit; + +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageBuilder; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.tests.DockerUtils; +import org.apache.pulsar.tests.PulsarClusterUtils; + +import org.jboss.arquillian.testng.Arquillian; +import org.jboss.arquillian.test.api.ArquillianResource; + +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestCompaction extends Arquillian { + private static final Logger LOG = LoggerFactory.getLogger(TestCompaction.class); + private static byte[] PASSWD = "foobar".getBytes(); + private static String clusterName = "test"; + + @ArquillianResource + DockerClient docker; + + @BeforeMethod + public void waitServicesUp() throws Exception { + Assert.assertTrue(PulsarClusterUtils.waitZooKeeperUp(docker, clusterName, 30, TimeUnit.SECONDS)); + Assert.assertTrue(PulsarClusterUtils.waitAllBrokersUp(docker, clusterName)); + } + + @Test + public void testPublishCompactAndConsumeCLI() throws Exception { + PulsarClusterUtils.runOnAnyBroker(docker, clusterName, + "/pulsar/bin/pulsar-admin", "properties", + "create", "compaction-test-cli", "--allowed-clusters", clusterName, + "--admin-roles", "admin"); + PulsarClusterUtils.runOnAnyBroker(docker, clusterName, + "/pulsar/bin/pulsar-admin", "namespaces", + "create", "compaction-test-cli/test/ns1"); + + String brokerIp = DockerUtils.getContainerIP( + docker, PulsarClusterUtils.proxySet(docker, clusterName).stream().findAny().get()); + String serviceUrl = "pulsar://" + brokerIp + ":6650"; + String topic = "persistent://compaction-test-cli/test/ns1/topic1"; + + try (PulsarClient client = PulsarClient.create(serviceUrl)) { + client.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close(); + + try(Producer<byte[]> producer = client.newProducer().topic(topic).create()) { + producer.send(MessageBuilder.create().setKey("key0").setContent("content0".getBytes()).build()); + producer.send(MessageBuilder.create().setKey("key0").setContent("content1".getBytes()).build()); + } + + try (Consumer<byte[]> consumer = client.newConsumer().topic(topic) + .readCompacted(true).subscriptionName("sub1").subscribe()) { + Message<byte[]> m = consumer.receive(); + Assert.assertEquals(m.getKey(), "key0"); + Assert.assertEquals(m.getData(), "content0".getBytes()); + + m = consumer.receive(); + Assert.assertEquals(m.getKey(), "key0"); + Assert.assertEquals(m.getData(), "content1".getBytes()); + } + + PulsarClusterUtils.runOnAnyBroker(docker, clusterName, + "/pulsar/bin/pulsar", "compact-topic", + "-t", topic); + + try (Consumer<byte[]> consumer = client.newConsumer().topic(topic) + .readCompacted(true).subscriptionName("sub1").subscribe()) { + Message<byte[]> m = consumer.receive(); + Assert.assertEquals(m.getKey(), "key0"); + Assert.assertEquals(m.getData(), "content1".getBytes()); + } + } + } +} diff --git a/tests/integration/pom.xml b/tests/integration/compaction/src/test/resources/arquillian.xml similarity index 54% copy from tests/integration/pom.xml copy to tests/integration/compaction/src/test/resources/arquillian.xml index a2e36d4..821d6e6 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/compaction/src/test/resources/arquillian.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="UTF-8"?> +<?xml version="1.0"?> <!-- Licensed to the Apache Software Foundation (ASF) under one @@ -19,20 +19,14 @@ under the License. --> -<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> - <packaging>pom</packaging> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.pulsar.tests</groupId> - <artifactId>tests-parent</artifactId> - <version>2.0.0-incubating-SNAPSHOT</version> - </parent> +<arquillian xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns="http://jboss.org/schema/arquillian" + xsi:schemaLocation="http://jboss.org/schema/arquillian + http://jboss.org/schema/arquillian/arquillian_1_0.xsd"> - <groupId>org.apache.pulsar.tests</groupId> - <artifactId>integration</artifactId> - <name>Apache Pulsar :: Tests :: Integration</name> - <modules> - <module>smoke</module> - </modules> -</project> + <extension qualifier="docker"> + <property name="definitionFormat">CUBE</property> + <property name="dockerContainersResource">cube-definitions/single-cluster-3-bookie-2-broker.yaml</property> + </extension> + +</arquillian> diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index a2e36d4..4b25ac4 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -34,5 +34,6 @@ <name>Apache Pulsar :: Tests :: Integration</name> <modules> <module>smoke</module> + <module>compaction</module> </modules> </project> -- To stop receiving notification emails like this one, please contact mme...@apache.org.