This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 5614cc5 Integration test coverage on python functions (#2072)
5614cc5 is described below
commit 5614cc5ef6e5eec01213baabe6cedbb6b65d8f53
Author: Sijie Guo <[email protected]>
AuthorDate: Tue Jul 3 00:24:09 2018 -0700
Integration test coverage on python functions (#2072)
*Motivation*
- Make sure python wheel file is built and can be installed/used correctly
- Make sure pulsar python functions work as expected
*Changes*
- copy `python-examples` to the distribution
- install python client in test image for pulsar python function test
- add PYTHON runtime to functions runtime test
---
distribution/server/src/assemble/bin.xml | 4 ++
pom.xml | 1 +
pulsar-client-cpp/docker/build-wheels.sh | 28 +++++++++++++
.../docker-images/latest-version-image/Dockerfile | 8 +++-
tests/docker-images/latest-version-image/pom.xml | 46 ++++++++++++++++++++++
.../scripts/install-pulsar-client.sh | 32 ++-------------
.../functions/PulsarFunctionsTestBase.java | 20 +++++++++-
.../runtime/PulsarFunctionsProcessRuntimeTest.java | 3 ++
.../runtime/PulsarFunctionsRuntimeTest.java | 34 ++++++++++++++--
.../runtime/PulsarFunctionsThreadRuntimeTest.java | 4 ++
.../functions/utils/CommandGenerator.java | 2 +-
11 files changed, 145 insertions(+), 37 deletions(-)
diff --git a/distribution/server/src/assemble/bin.xml
b/distribution/server/src/assemble/bin.xml
index ff7c1a3..61178bf 100644
--- a/distribution/server/src/assemble/bin.xml
+++ b/distribution/server/src/assemble/bin.xml
@@ -47,6 +47,10 @@
<directory>${basedir}/../../pulsar-functions/runtime/target/python-instance</directory>
<outputDirectory>instances/python-instance</outputDirectory>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/../../pulsar-functions/python-examples</directory>
+ <outputDirectory>examples/python-examples</outputDirectory>
+ </fileSet>
</fileSets>
<files>
<file>
diff --git a/pom.xml b/pom.xml
index 1c7f899..862a29d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,6 +166,7 @@ flexible messaging model and an intuitive client
API.</description>
<!-- Plugin dependencies -->
<protobuf-maven-plugin.version>0.5.0</protobuf-maven-plugin.version>
+ <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
</properties>
<dependencyManagement>
diff --git a/pulsar-client-cpp/docker/build-wheels.sh
b/pulsar-client-cpp/docker/build-wheels.sh
index 2146869..ebbb441 100755
--- a/pulsar-client-cpp/docker/build-wheels.sh
+++ b/pulsar-client-cpp/docker/build-wheels.sh
@@ -35,6 +35,34 @@ PYTHON_VERSIONS=(
'3.6 cp36-cp36m'
)
+function contains() {
+ local n=$#
+ local value=${!n}
+ for ((i=1;i < $#;i++)) {
+ if [ "${!i}" == "${value}" ]; then
+ echo "y"
+ return 0
+ fi
+ }
+ echo "n"
+ return 1
+}
+
+
+if [ $# -ge 1 ]; then
+ build_version=$@
+ if [ $(contains "${PYTHON_VERSIONS[@]}" "${build_version}") == "y" ]; then
+ PYTHON_VERSIONS=(
+ "${build_version}"
+ )
+ else
+ echo "Unknown build version : ${build_version}"
+ echo "Supported python build versions are :"
+ echo ${PYTHON_VERSIONS[@]}
+ exit 1
+ fi
+fi
+
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
diff --git a/tests/docker-images/latest-version-image/Dockerfile
b/tests/docker-images/latest-version-image/Dockerfile
index e638ba0..bcc2b53 100644
--- a/tests/docker-images/latest-version-image/Dockerfile
+++ b/tests/docker-images/latest-version-image/Dockerfile
@@ -19,7 +19,7 @@
FROM apachepulsar/pulsar:latest
-RUN apt-get update && apt-get install -y supervisor
+RUN apt-get update && apt-get install -y supervisor python-pip
RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/ && mkdir -p
/pulsar/ssl
@@ -31,5 +31,9 @@ COPY ssl/ca.cert.pem ssl/broker.key-pk8.pem
ssl/broker.cert.pem \
ssl/admin.key-pk8.pem ssl/admin.cert.pem /pulsar/ssl/
COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \
- scripts/run-bookie.sh scripts/run-broker.sh
scripts/run-functions-worker.sh scripts/run-proxy.sh /pulsar/bin/
+ scripts/run-bookie.sh scripts/run-broker.sh
scripts/run-functions-worker.sh scripts/run-proxy.sh \
+ scripts/install-pulsar-client.sh /pulsar/bin/
+
+ADD target/python-client/ /pulsar/pulsar-client
+RUN /pulsar/bin/install-pulsar-client.sh
diff --git a/tests/docker-images/latest-version-image/pom.xml
b/tests/docker-images/latest-version-image/pom.xml
index 232312f..5094f89 100644
--- a/tests/docker-images/latest-version-image/pom.xml
+++ b/tests/docker-images/latest-version-image/pom.xml
@@ -48,20 +48,66 @@
</dependency>
</dependencies>
<build>
+ <!-- build cpp client, copy the wheel file and then build docker image
-->
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${exec-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>build-pulsar-clients</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <skip>${skipBuildPythonClient}</skip>
+
<workingDirectory>${project.basedir}/target</workingDirectory>
+
<executable>${project.basedir}/../../../pulsar-client-cpp/docker/build-wheels.sh</executable>
+ <arguments>
+ <!-- build python 2.7 -->
+ <argument>2.7 cp27-cp27mu</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>compile</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <skip>${skipBuildPythonClient}</skip>
+ <tasks>
+ <echo>copy python wheel file</echo>
+ <mkdir dir="${basedir}/target/python-client"/>
+ <copydir
src="${basedir}/../../../pulsar-client-cpp/python/wheelhouse"
dest="${basedir}/target/python-client"/>
+ </tasks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>com.spotify</groupId>
<artifactId>dockerfile-maven-plugin</artifactId>
<version>${dockerfile-maven.version}</version>
<executions>
<execution>
<id>default</id>
+ <phase>package</phase>
<goals>
<goal>build</goal>
</goals>
</execution>
<execution>
<id>add-latest-tag</id>
+ <phase>package</phase>
<goals>
<goal>tag</goal>
</goals>
diff --git a/pulsar-client-cpp/docker/build-wheels.sh
b/tests/docker-images/latest-version-image/scripts/install-pulsar-client.sh
similarity index 53%
copy from pulsar-client-cpp/docker/build-wheels.sh
copy to
tests/docker-images/latest-version-image/scripts/install-pulsar-client.sh
index 2146869..2e048c1 100755
--- a/pulsar-client-cpp/docker/build-wheels.sh
+++ b/tests/docker-images/latest-version-image/scripts/install-pulsar-client.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,31 +18,7 @@
# under the License.
#
+set -x
-set -e
-
-BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
-
-ROOT_DIR=$(git rev-parse --show-toplevel)
-cd $ROOT_DIR
-
-PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu'
- '2.7 cp27-cp27m'
- '3.3 cp33-cp33m'
- '3.4 cp34-cp34m'
- '3.5 cp35-cp35m'
- '3.6 cp36-cp36m'
-)
-
-for line in "${PYTHON_VERSIONS[@]}"; do
- read -r -a PY <<< "$line"
- PYTHON_VERSION=${PY[0]}
- PYTHON_SPEC=${PY[1]}
- echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC"
-
- IMAGE_NAME=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
-
- echo "Using image: $IMAGE_NAME"
- docker run -i -v $PWD:/pulsar $IMAGE_NAME
/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
-done
+WHEEL_FILE=`ls /pulsar/pulsar-client`
+pip install /pulsar/pulsar-client/${WHEEL_FILE}
diff --git
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
index dc2e290..b8ffc7c 100644
---
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
+++
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java
@@ -31,9 +31,24 @@ import org.testng.annotations.DataProvider;
@Slf4j
public abstract class PulsarFunctionsTestBase extends PulsarClusterTestBase {
- public static final String EXCLAMATION_FUNC_CLASS =
+ public static final String EXCLAMATION_JAVA_CLASS =
"org.apache.pulsar.functions.api.examples.ExclamationFunction";
+ public static final String EXCLAMATION_PYTHON_CLASS =
+ "exclamation.Exclamation";
+
+ public static final String EXCLAMATION_PYTHON_FILE = "exclamation.py";
+
+ protected static String getExclamationClass(Runtime runtime) {
+ if (Runtime.JAVA == runtime) {
+ return EXCLAMATION_JAVA_CLASS;
+ } else if (Runtime.PYTHON == runtime) {
+ return EXCLAMATION_PYTHON_CLASS;
+ } else {
+ throw new IllegalArgumentException("Unsupported runtime : " +
runtime);
+ }
+ }
+
@BeforeClass
public static void setupCluster() throws Exception {
PulsarClusterTestBase.setupCluster();
@@ -47,7 +62,8 @@ public abstract class PulsarFunctionsTestBase extends
PulsarClusterTestBase {
@DataProvider(name = "FunctionRuntimes")
public static Object[][] functionRuntimes() {
return new Object[][] {
- new Object[] { Runtime.JAVA }
+ new Object[] { Runtime.JAVA },
+ new Object[] { Runtime.PYTHON }
};
}
diff --git
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsProcessRuntimeTest.java
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsProcessRuntimeTest.java
index 7c4033f..83edac8 100644
---
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsProcessRuntimeTest.java
+++
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsProcessRuntimeTest.java
@@ -22,4 +22,7 @@ package org.apache.pulsar.tests.integration.functions.runtime;
* Run runtime tests in process mode.
*/
public class PulsarFunctionsProcessRuntimeTest extends
PulsarFunctionsRuntimeTest {
+ public PulsarFunctionsProcessRuntimeTest() {
+ super(ContainerFactory.PROCESS);
+ }
}
diff --git
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsRuntimeTest.java
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsRuntimeTest.java
index e816f70..41be67f 100644
---
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsRuntimeTest.java
+++
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsRuntimeTest.java
@@ -40,12 +40,28 @@ import org.testng.annotations.Test;
*/
public abstract class PulsarFunctionsRuntimeTest extends
PulsarFunctionsTestBase {
+ protected enum ContainerFactory {
+ PROCESS,
+ THREAD
+ }
+
+ private final ContainerFactory containerFactory;
+
+ public PulsarFunctionsRuntimeTest(ContainerFactory containerFactory) {
+ this.containerFactory = containerFactory;
+ }
+
//
// Test CRUD functions on different runtimes.
//
@Test(dataProvider = "FunctionRuntimes")
public void testExclamationFunction(Runtime runtime) throws Exception {
+ if (ContainerFactory.THREAD == containerFactory && Runtime.PYTHON ==
runtime) {
+ // python can only run on process mode
+ return;
+ }
+
String inputTopicName = "test-exclamation-" + runtime + "-input-" +
randomName(8);
String outputTopicName = "test-exclamation-" + runtime + "-output-" +
randomName(8);
String functionName = "test-exclamation-fn-" + randomName(8);
@@ -53,7 +69,7 @@ public abstract class PulsarFunctionsRuntimeTest extends
PulsarFunctionsTestBase
// submit the exclamation function
submitExclamationFunction(
- inputTopicName, outputTopicName, functionName);
+ runtime, inputTopicName, outputTopicName, functionName);
// get function info
getFunctionInfoSuccess(functionName);
@@ -71,13 +87,23 @@ public abstract class PulsarFunctionsRuntimeTest extends
PulsarFunctionsTestBase
getFunctionInfoNotFound(functionName);
}
- private static void submitExclamationFunction(String inputTopicName,
+ private static void submitExclamationFunction(Runtime runtime,
+ String inputTopicName,
String outputTopicName,
String functionName) throws
Exception {
- CommandGenerator generator =
CommandGenerator.createDefaultGenerator(inputTopicName, EXCLAMATION_FUNC_CLASS);
+ CommandGenerator generator;
+ generator = CommandGenerator.createDefaultGenerator(inputTopicName,
getExclamationClass(runtime));
generator.setSinkTopic(outputTopicName);
generator.setFunctionName(functionName);
- String command = generator.generateCreateFunctionCommand();
+ String command;
+ if (Runtime.JAVA == runtime) {
+ command = generator.generateCreateFunctionCommand();
+ } else if (Runtime.PYTHON == runtime) {
+ generator.setRuntime(runtime);
+ command =
generator.generateCreateFunctionCommand(EXCLAMATION_PYTHON_FILE);
+ } else {
+ throw new IllegalArgumentException("Unsupported runtime : " +
runtime);
+ }
String[] commands = {
"sh", "-c", command
};
diff --git
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsThreadRuntimeTest.java
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsThreadRuntimeTest.java
index 61ddba3..7ca9931 100644
---
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsThreadRuntimeTest.java
+++
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/runtime/PulsarFunctionsThreadRuntimeTest.java
@@ -29,6 +29,10 @@ import org.testng.annotations.BeforeClass;
@Slf4j
public class PulsarFunctionsThreadRuntimeTest extends
PulsarFunctionsRuntimeTest {
+ public PulsarFunctionsThreadRuntimeTest() {
+ super(ContainerFactory.THREAD);
+ }
+
@BeforeClass
public static void setupCluster() throws Exception {
PulsarClusterTestBase.setupCluster();
diff --git
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java
index 12225fe..7146616 100644
---
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java
+++
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java
@@ -54,7 +54,7 @@ public class CommandGenerator {
private Map<String, String> userConfig = new HashMap<>();
private static final String JAVAJAR = "/pulsar/examples/api-examples.jar";
- private static final String PYTHONBASE = "/pulsar/examples/python";
+ private static final String PYTHONBASE =
"/pulsar/examples/python-examples/";
public static CommandGenerator createDefaultGenerator(String sourceTopic,
String functionClassName) {
CommandGenerator generator = new CommandGenerator();
