This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 35a121b Add a new docker image to include all pulsar io connectors
(#2158)
35a121b is described below
commit 35a121b55569bb5b7f1cbe001bc8410a6f246589
Author: Sijie Guo <[email protected]>
AuthorDate: Sat Jul 14 10:57:47 2018 -0700
Add a new docker image to include all pulsar io connectors (#2158)
*Motivation*
We shipped two distributions : server & io-connectors. For people who
run pulsar in docker, we need to put all the built-in connectors into
the docker, so that all built-in connectors are available to be used.
*Solution*
Introduce a `pulsar-all` image to include built-in connectors.
Named it as `pulsar-all` to allow include more pluggable components later,
for example, offloaders.
---
distribution/io/pom.xml | 2 +-
docker/pom.xml | 1 +
.../pulsar-all}/Dockerfile | 20 +-----
.../pulsar-all}/pom.xml | 80 +++++++++-------------
pom.xml | 5 +-
.../docker-images/latest-version-image/Dockerfile | 2 +-
tests/docker-images/latest-version-image/pom.xml | 2 +-
7 files changed, 41 insertions(+), 71 deletions(-)
diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
index c00a5e6..dcb5095 100644
--- a/distribution/io/pom.xml
+++ b/distribution/io/pom.xml
@@ -53,7 +53,7 @@
<goal>single</goal>
</goals>
<configuration>
- <attach>false</attach>
+ <attach>true</attach>
<tarLongFileMode>posix</tarLongFileMode>
<finalName>apache-pulsar-io-connectors-${project.version}</finalName>
<descriptors>
diff --git a/docker/pom.xml b/docker/pom.xml
index 8878b2b..675656a 100644
--- a/docker/pom.xml
+++ b/docker/pom.xml
@@ -37,5 +37,6 @@
<modules>
<module>pulsar</module>
<module>grafana</module>
+ <module>pulsar-all</module>
</modules>
</project>
diff --git a/tests/docker-images/latest-version-image/Dockerfile
b/docker/pulsar-all/Dockerfile
similarity index 50%
copy from tests/docker-images/latest-version-image/Dockerfile
copy to docker/pulsar-all/Dockerfile
index bcc2b53..46f7b9f 100644
--- a/tests/docker-images/latest-version-image/Dockerfile
+++ b/docker/pulsar-all/Dockerfile
@@ -19,21 +19,7 @@
FROM apachepulsar/pulsar:latest
-RUN apt-get update && apt-get install -y supervisor python-pip
-
-RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/ && mkdir -p
/pulsar/ssl
-
-COPY conf/supervisord.conf /etc/supervisord.conf
-COPY conf/global-zk.conf conf/local-zk.conf conf/bookie.conf conf/broker.conf
conf/functions_worker.conf \
- conf/proxy.conf /etc/supervisord/conf.d/
-
-COPY ssl/ca.cert.pem ssl/broker.key-pk8.pem ssl/broker.cert.pem \
- ssl/admin.key-pk8.pem ssl/admin.cert.pem /pulsar/ssl/
-
-COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \
- scripts/run-bookie.sh scripts/run-broker.sh
scripts/run-functions-worker.sh scripts/run-proxy.sh \
- scripts/install-pulsar-client.sh /pulsar/bin/
-
-ADD target/python-client/ /pulsar/pulsar-client
-RUN /pulsar/bin/install-pulsar-client.sh
+ARG PULSAR_IO_TARBALL
+ADD ${PULSAR_IO_TARBALL} /
+RUN mv /apache-pulsar-io-connectors-*/connectors /pulsar/connectors
diff --git a/tests/docker-images/latest-version-image/pom.xml
b/docker/pulsar-all/pom.xml
similarity index 56%
copy from tests/docker-images/latest-version-image/pom.xml
copy to docker/pulsar-all/pom.xml
index 5094f89..5149999 100644
--- a/tests/docker-images/latest-version-image/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -21,24 +21,31 @@
<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<parent>
- <groupId>org.apache.pulsar.tests</groupId>
+ <groupId>org.apache.pulsar</groupId>
<artifactId>docker-images</artifactId>
<version>2.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.pulsar.tests</groupId>
- <artifactId>latest-version-image</artifactId>
- <name>Apache Pulsar :: Tests :: Docker Images :: Latest Version
Testing</name>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-all-docker-image</artifactId>
+ <name>Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All
Components)</name>
<packaging>pom</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-io-distribution</artifactId>
+ <version>${project.parent.version}</version>
+ <classifier>bin</classifier>
+ <type>tar.gz</type>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
<profiles>
<profile>
<id>docker</id>
- <activation>
- <property>
- <name>integrationTests</name>
- </property>
- </activation>
+ <!-- include the docker image only when docker profile is active -->
<dependencies>
<dependency>
<groupId>org.apache.pulsar</groupId>
@@ -48,47 +55,21 @@
</dependency>
</dependencies>
<build>
- <!-- build cpp client, copy the wheel file and then build docker image
-->
<plugins>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>${exec-maven-plugin.version}</version>
- <executions>
- <execution>
- <id>build-pulsar-clients</id>
- <phase>compile</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <skip>${skipBuildPythonClient}</skip>
-
<workingDirectory>${project.basedir}/target</workingDirectory>
-
<executable>${project.basedir}/../../../pulsar-client-cpp/docker/build-wheels.sh</executable>
- <arguments>
- <!-- build python 2.7 -->
- <argument>2.7 cp27-cp27mu</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
+ <artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <phase>compile</phase>
+ <id>copy-io-tarball</id>
<goals>
- <goal>run</goal>
+ <goal>copy-dependencies</goal>
</goals>
+ <phase>generate-resources</phase>
<configuration>
- <skip>${skipBuildPythonClient}</skip>
- <tasks>
- <echo>copy python wheel file</echo>
- <mkdir dir="${basedir}/target/python-client"/>
- <copydir
src="${basedir}/../../../pulsar-client-cpp/python/wheelhouse"
dest="${basedir}/target/python-client"/>
- </tasks>
+
<outputDirectory>${project.build.directory}/</outputDirectory>
+
<includeArtifactIds>pulsar-io-distribution</includeArtifactIds>
+ <excludeTransitive>true</excludeTransitive>
</configuration>
</execution>
</executions>
@@ -100,28 +81,29 @@
<executions>
<execution>
<id>default</id>
- <phase>package</phase>
<goals>
<goal>build</goal>
</goals>
</execution>
<execution>
- <id>add-latest-tag</id>
- <phase>package</phase>
+ <id>tag-and-push-latest</id>
<goals>
<goal>tag</goal>
+ <goal>push</goal>
</goals>
<configuration>
-
<repository>apachepulsar/pulsar-test-latest-version</repository>
+ <repository>${docker.organization}/pulsar-all</repository>
<tag>latest</tag>
</configuration>
</execution>
</executions>
<configuration>
- <repository>apachepulsar/pulsar-test-latest-version</repository>
- <tag>${project.version}</tag>
+ <repository>${docker.organization}/pulsar-all</repository>
<pullNewerImage>false</pullNewerImage>
- <noCache>true</noCache>
+ <tag>${project.version}</tag>
+ <buildArgs>
+
<PULSAR_IO_TARBALL>target/pulsar-io-distribution-${project.version}-bin.tar.gz</PULSAR_IO_TARBALL>
+ </buildArgs>
</configuration>
</plugin>
</plugins>
diff --git a/pom.xml b/pom.xml
index 8a0da98..7b59f99 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,8 +97,6 @@ flexible messaging model and an intuitive client
API.</description>
<module>pulsar-client-auth-athenz</module>
<module>pulsar-client-kafka-compat</module>
<module>pulsar-zookeeper</module>
- <module>docker</module>
- <module>tests</module>
<module>pulsar-log4j2-appender</module>
<module>protobuf-shaded</module>
@@ -108,7 +106,10 @@ flexible messaging model and an intuitive client
API.</description>
<!-- connector-related modules -->
<module>pulsar-io</module>
+ <!-- all these 3 modules should be put at the end in this exact sequence
-->
<module>distribution</module>
+ <module>docker</module>
+ <module>tests</module>
</modules>
<issueManagement>
diff --git a/tests/docker-images/latest-version-image/Dockerfile
b/tests/docker-images/latest-version-image/Dockerfile
index bcc2b53..9c93de9 100644
--- a/tests/docker-images/latest-version-image/Dockerfile
+++ b/tests/docker-images/latest-version-image/Dockerfile
@@ -17,7 +17,7 @@
# under the License.
#
-FROM apachepulsar/pulsar:latest
+FROM apachepulsar/pulsar-all:latest
RUN apt-get update && apt-get install -y supervisor python-pip
diff --git a/tests/docker-images/latest-version-image/pom.xml
b/tests/docker-images/latest-version-image/pom.xml
index 5094f89..8e47055 100644
--- a/tests/docker-images/latest-version-image/pom.xml
+++ b/tests/docker-images/latest-version-image/pom.xml
@@ -42,7 +42,7 @@
<dependencies>
<dependency>
<groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-docker-image</artifactId>
+ <artifactId>pulsar-all-docker-image</artifactId>
<version>${project.parent.version}</version>
<classifier>docker-info</classifier>
</dependency>
