This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git
The following commit(s) were added to refs/heads/main by this push:
new dc8eaeb Enhance developer experience for debugging tests and add
related README (#16)
dc8eaeb is described below
commit dc8eaeb39a03dc2c8611aa53e3a1bb962033f175
Author: Yunze Xu <[email protected]>
AuthorDate: Fri Oct 14 04:28:44 2022 +0800
Enhance developer experience for debugging tests and add related README
(#16)
### Motivation
When I ran `python3 ./tests/pulsar_tests.py`, I found some tests failed.
However, when I went into the container that was started by
`./build-support/pulsar-test-service-start.sh`, the logs are incomplete
because the immediate flush is disabled by default.
Finally I found the error is caused by the relative path in tests. The
certs are under `tests/test-conf` directory, but `pulsar_tests.py` uses
the `test-conf/` relative path. Since `tests/run-unit-tests.sh` first
goes to the `tests` directory before running the tests, CI works well.
But a developer might just want to run a specific test from any possible
directory, e.g.
```bash
python3 ./tests/pulsar_tests.py PulsarTest.test_tls_auth
```
Then it could fail and he cannot see complete logs.
### Modifications
- Modify `immediateFlush` to true
- Use absolute path to represent `test-conf/` in `pulsar_tests.py`
- Add an initial README for quick start
---
README.md | 84 ++++++++++++++++++++++
.../start-test-service-inside-container.sh | 3 +
tests/pulsar_test.py | 23 +++---
3 files changed, 97 insertions(+), 13 deletions(-)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0d403cd
--- /dev/null
+++ b/README.md
@@ -0,0 +1,84 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# Pulsar Python client library
+
+## Requirements
+
+- Python >= 3.7
+- A C++ compiler that supports C++11
+- CMake >= 3.18
+- [Pulsar C++ client library](https://github.com/apache/pulsar-client-cpp)
+- [Boost.Python](https://github.com/boostorg/python)
+
+## Install the Python wheel
+
+```bash
+cmake -B build
+cmake --build build -j8
+cp build/_pulsar.so .
+./setup.py bdist_wheel
+pip3 install dist/pulsar_client-*.whl --force-reinstall
+rm _pulsar.so
+```
+
+> **NOTE**
+>
+> 1. Here a separate `build` directory is created to store all CMake temporary
files. However, the `setup.py` requires the `_pulsar.so` is under the project
directory.
+> 2. Add the `--force-reinstall` option to overwrite the existing Python wheel
in case your system has already installed a wheel before.
+
+You can run `python3 -c 'import pulsar'` to see whether the wheel has been
installed successfully. If it failed, check whether dependencies (e.g.
`libpulsar.so`) are in the system path. If not, make sure the dependencies are
in `LD_LIBRARY_PATH` (on Linux) or `DYLD_LIBRARY_PATH` (on macOS).
+
+Then you can run examples as a simple end-to-end test.
+
+```bash
+# In terminal 1
+python3 ./examples/consumer.py
+```
+
+```bash
+# In terminal 2
+python3 ./examples/producer.py
+```
+
+Before executing the commands above, you must ensure the Pulsar service is
running. See [here](https://pulsar.apache.org/docs/getting-started-standalone)
for quick start.
+
+## Unit tests
+
+Before running the unit tests, you must run a Pulsar service with all things
set up:
+
+```bash
+./build-support/pulsar-test-service-start.sh
+```
+
+The command above runs a Pulsar standalone in a Docker container. You can run
`./build-support/pulsar-test-service-stop.sh` to stop it.
+
+Run all unit tests:
+
+```bash
+./tests/run-unit-tests.sh
+```
+
+Run a single unit test (e.g. `PulsarTest.test_tls_auth`):
+
+```bash
+python3 ./tests/pulsar_test.py 'PulsarTest.test_tls_auth'
+```
diff --git a/build-support/start-test-service-inside-container.sh
b/build-support/start-test-service-inside-container.sh
index 0517f77..658be03 100755
--- a/build-support/start-test-service-inside-container.sh
+++ b/build-support/start-test-service-inside-container.sh
@@ -22,6 +22,9 @@ set -e -x
export PULSAR_EXTRA_OPTS=-Dpulsar.auth.basic.conf=test-conf/.htpasswd
+# Show logs immediately for debugging
+sed -i.bak 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml
+
# Generate secret key and token
mkdir -p data/tokens
bin/pulsar tokens create-secret-key --output data/tokens/secret.key
diff --git a/tests/pulsar_test.py b/tests/pulsar_test.py
index 314f6f6..71af279 100755
--- a/tests/pulsar_test.py
+++ b/tests/pulsar_test.py
@@ -49,6 +49,7 @@ from schema_test import *
from urllib.request import urlopen, Request
TM = 10000 # Do not wait forever in tests
+CERTS_DIR = os.path.dirname(__file__) + "/test-conf/"
def doHttpPost(url, data):
@@ -319,12 +320,11 @@ class PulsarTest(TestCase):
client.close()
def test_tls_auth(self):
- certs_dir = "test-conf/"
client = Client(
self.serviceUrlTls,
- tls_trust_certs_file_path=certs_dir + "cacert.pem",
+ tls_trust_certs_file_path=CERTS_DIR + "cacert.pem",
tls_allow_insecure_connection=False,
- authentication=AuthenticationTLS(certs_dir + "client-cert.pem",
certs_dir + "client-key.pem"),
+ authentication=AuthenticationTLS(CERTS_DIR + "client-cert.pem",
CERTS_DIR + "client-key.pem"),
)
topic = "my-python-topic-tls-auth-" + str(time.time())
@@ -342,13 +342,12 @@ class PulsarTest(TestCase):
client.close()
def test_tls_auth2(self):
- certs_dir = "test-conf/"
authPlugin = "org.apache.pulsar.client.impl.auth.AuthenticationTls"
- authParams =
"tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir,
certs_dir)
+ authParams =
"tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (CERTS_DIR,
CERTS_DIR)
client = Client(
self.serviceUrlTls,
- tls_trust_certs_file_path=certs_dir + "cacert.pem",
+ tls_trust_certs_file_path=CERTS_DIR + "cacert.pem",
tls_allow_insecure_connection=False,
authentication=Authentication(authPlugin, authParams),
)
@@ -368,8 +367,8 @@ class PulsarTest(TestCase):
client.close()
def test_encryption(self):
- publicKeyPath = "test-conf/public-key.client-rsa.pem"
- privateKeyPath = "test-conf/private-key.client-rsa.pem"
+ publicKeyPath = CERTS_DIR + "public-key.client-rsa.pem"
+ privateKeyPath = CERTS_DIR + "private-key.client-rsa.pem"
crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath)
client = Client(self.serviceUrl)
topic = "my-python-test-end-to-end-encryption"
@@ -400,13 +399,12 @@ class PulsarTest(TestCase):
client.close()
def test_tls_auth3(self):
- certs_dir = "test-conf/"
authPlugin = "tls"
- authParams =
"tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir,
certs_dir)
+ authParams =
"tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (CERTS_DIR,
CERTS_DIR)
client = Client(
self.serviceUrlTls,
- tls_trust_certs_file_path=certs_dir + "cacert.pem",
+ tls_trust_certs_file_path=CERTS_DIR + "cacert.pem",
tls_allow_insecure_connection=False,
authentication=Authentication(authPlugin, authParams),
)
@@ -426,12 +424,11 @@ class PulsarTest(TestCase):
client.close()
def test_auth_junk_params(self):
- certs_dir = "test-conf/"
authPlugin = "someoldjunk.so"
authParams = "blah"
client = Client(
self.serviceUrlTls,
- tls_trust_certs_file_path=certs_dir + "cacert.pem",
+ tls_trust_certs_file_path=CERTS_DIR + "cacert.pem",
tls_allow_insecure_connection=False,
authentication=Authentication(authPlugin, authParams),
)
