This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 8cd83abb6 chore(example): test python examples in CI (#2174)
8cd83abb6 is described below
commit 8cd83abb6cbb0f6cb5da157c38be714a848782d5
Author: Huan-Cheng Chang <[email protected]>
AuthorDate: Tue Nov 18 12:12:08 2025 +0000
chore(example): test python examples in CI (#2174)
* Added one script `scripts/run-python-examples-from-readme.sh` which
runs all python examples and returns an error when any of the examples
fails. For consistency, the script basically aligns with other scripts
for different SDKs.
* Added one github action that builds the python SDK package and runs
example tests. Since checking python examples requires building and
installing the SDK, which means a few more setup steps, the check is
turned into a github action.
* Added one task that checks python examples using the new action above.
* Updated a few things regarding the examples, e.g. fixing the default
argument in the basic example, so that they work with the test script.
---------
Co-authored-by: Hubert Gruszecki <[email protected]>
---
.github/config/components.yml | 4 +-
.github/workflows/_test_examples.yml | 22 +++-
examples/python/README.md | 6 +-
examples/python/basic/consumer.py | 7 +-
examples/python/basic/producer.py | 9 +-
examples/python/getting-started/consumer.py | 6 +-
examples/python/getting-started/producer.py | 8 +-
examples/python/requirements.txt | 2 +-
scripts/run-python-examples-from-readme.sh | 168 ++++++++++++++++++++++++++++
9 files changed, 210 insertions(+), 22 deletions(-)
diff --git a/.github/config/components.yml b/.github/config/components.yml
index 1d775d85f..e77b3d9b7 100644
--- a/.github/config/components.yml
+++ b/.github/config/components.yml
@@ -237,13 +237,15 @@ components:
- "rust-server"
- "sdk-go"
- "sdk-csharp"
+ - "sdk-python"
- "ci-infrastructure" # CI changes trigger full regression
paths:
- "examples/**"
- "scripts/run-rust-examples-from-readme.sh"
- "scripts/run-go-examples-from-readme.sh"
- "scripts/run-csharp-examples-from-readme.sh"
- tasks: ["examples-rust", "examples-go", "examples-csharp"]
+ - "scripts/run-python-examples-from-readme.sh"
+ tasks: ["examples-rust", "examples-go", "examples-csharp",
"examples-python"]
web-ui:
paths:
diff --git a/.github/workflows/_test_examples.yml
b/.github/workflows/_test_examples.yml
index 89e847141..38e03a83a 100644
--- a/.github/workflows/_test_examples.yml
+++ b/.github/workflows/_test_examples.yml
@@ -52,6 +52,19 @@ jobs:
with:
cache-targets: false # Only cache registry and git deps, not target
dir (sccache handles that)
+ - name: Setup Python
+ if: inputs.component == 'examples-suite'
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Cache pip
+ if: inputs.component == 'examples-suite'
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/pip
+ key: pip-${{ runner.os }}-${{
hashFiles('foreign/python/pyproject.toml') }}
+
- name: Build common binaries for all examples
if: inputs.component == 'examples-suite'
run: |
@@ -95,23 +108,26 @@ jobs:
if: inputs.component == 'examples-suite' && inputs.task ==
'examples-rust'
run: |
echo "Running Rust examples tests..."
- # Run the examples script which will use the prebuilt binaries
./scripts/run-rust-examples-from-readme.sh
- name: Run Go examples
if: inputs.component == 'examples-suite' && inputs.task ==
'examples-go'
run: |
echo "Running Go examples tests..."
- # Run the examples script which will use the prebuilt server binary
./scripts/run-go-examples-from-readme.sh
- name: Run Csharp examples
if: inputs.component == 'examples-suite' && inputs.task ==
'examples-csharp'
run: |
echo "Running Csharp examples tests..."
- # Run the examples script which will use the prebuilt server binary
./scripts/run-csharp-examples-from-readme.sh
+ - name: Run Python examples
+ if: inputs.component == 'examples-suite' && inputs.task ==
'examples-python'
+ run: |
+ echo "Running Python examples tests..."
+ ./scripts/run-python-examples-from-readme.sh
+
- name: Upload reports
if: always()
uses: actions/upload-artifact@v4
diff --git a/examples/python/README.md b/examples/python/README.md
index eab95b923..1bcee58eb 100644
--- a/examples/python/README.md
+++ b/examples/python/README.md
@@ -30,7 +30,7 @@ IGGY_HTTP_ENABLED=true IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo
run --bin iggy-server
and then install Python dependencies:
```bash
-pip -r requirements.txt
+pip install -r requirements.txt
```
## Basic Examples
@@ -49,8 +49,8 @@ python getting-started/consumer.py
Core functionality with detailed configuration options:
```bash
-python basic/producer.py <connection_string>
-python basic/consumer.py <connection_string>
+python basic/producer.py
+python basic/consumer.py
```
Demonstrates fundamental client connection, authentication, batch message
sending, and polling with support for TCP/QUIC/HTTP protocols.
diff --git a/examples/python/basic/consumer.py
b/examples/python/basic/consumer.py
index 8523bf8cf..2097f98de 100644
--- a/examples/python/basic/consumer.py
+++ b/examples/python/basic/consumer.py
@@ -24,9 +24,9 @@ from loguru import logger
STREAM_NAME = "sample-stream"
TOPIC_NAME = "sample-topic"
-STREAM_ID = 1
-TOPIC_ID = 1
-PARTITION_ID = 1
+STREAM_ID = 0
+TOPIC_ID = 0
+PARTITION_ID = 0
BATCHES_LIMIT = 5
ArgNamespace = namedtuple("ArgNamespace", ["connection_string"])
@@ -40,6 +40,7 @@ def parse_args() -> argparse.Namespace:
"Connection string for Iggy client, e.g.
'iggy+tcp://iggy:[email protected]:8090'"
),
default="iggy+tcp://iggy:[email protected]:8090",
+ nargs="?",
type=str,
)
return parser.parse_args()
diff --git a/examples/python/basic/producer.py
b/examples/python/basic/producer.py
index b3548f70d..e2c6d524b 100644
--- a/examples/python/basic/producer.py
+++ b/examples/python/basic/producer.py
@@ -25,9 +25,9 @@ from loguru import logger
STREAM_NAME = "sample-stream"
TOPIC_NAME = "sample-topic"
-STREAM_ID = 1
-TOPIC_ID = 1
-PARTITION_ID = 1
+STREAM_ID = 0
+TOPIC_ID = 0
+PARTITION_ID = 0
BATCHES_LIMIT = 5
ArgNamespace = namedtuple("ArgNamespace", ["connection_string"])
@@ -41,6 +41,7 @@ def parse_args() -> argparse.Namespace:
"Connection string for Iggy client, e.g.
'iggy+tcp://iggy:[email protected]:8090'"
),
default="iggy+tcp://iggy:[email protected]:8090",
+ nargs="?",
type=str,
)
return parser.parse_args()
@@ -61,7 +62,7 @@ async def init_system(client: IggyClient):
logger.info(f"Creating stream with name {STREAM_NAME}...")
stream: StreamDetails = await client.get_stream(STREAM_NAME)
if stream is None:
- await client.create_stream(name=STREAM_NAME, stream_id=STREAM_ID)
+ await client.create_stream(name=STREAM_NAME)
logger.info("Stream was created successfully.")
else:
logger.warning(f"Stream {stream.name} already exists with ID
{stream.id}")
diff --git a/examples/python/getting-started/consumer.py
b/examples/python/getting-started/consumer.py
index 37cb51e69..30a78d5ba 100644
--- a/examples/python/getting-started/consumer.py
+++ b/examples/python/getting-started/consumer.py
@@ -27,9 +27,9 @@ from loguru import logger
STREAM_NAME = "sample-stream"
TOPIC_NAME = "sample-topic"
-STREAM_ID = 1
-TOPIC_ID = 1
-PARTITION_ID = 1
+STREAM_ID = 0
+TOPIC_ID = 0
+PARTITION_ID = 0
BATCHES_LIMIT = 5
ArgNamespace = namedtuple("ArgNamespace", ["tcp_server_address"])
diff --git a/examples/python/getting-started/producer.py
b/examples/python/getting-started/producer.py
index 71ccb2232..36445bfac 100644
--- a/examples/python/getting-started/producer.py
+++ b/examples/python/getting-started/producer.py
@@ -28,9 +28,9 @@ from loguru import logger
STREAM_NAME = "sample-stream"
TOPIC_NAME = "sample-topic"
-STREAM_ID = 1
-TOPIC_ID = 1
-PARTITION_ID = 1
+STREAM_ID = 0
+TOPIC_ID = 0
+PARTITION_ID = 0
BATCHES_LIMIT = 5
ArgNamespace = namedtuple("ArgNamespace", ["tcp_server_address"])
@@ -78,7 +78,7 @@ async def init_system(client: IggyClient):
logger.info(f"Creating stream with name {STREAM_NAME}...")
stream: StreamDetails = await client.get_stream(STREAM_NAME)
if stream is None:
- await client.create_stream(name=STREAM_NAME, stream_id=STREAM_ID)
+ await client.create_stream(name=STREAM_NAME)
logger.info("Stream was created successfully.")
else:
logger.warning(f"Stream {stream.name} already exists with ID
{stream.id}")
diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt
index 11231bec6..e46e8562d 100644
--- a/examples/python/requirements.txt
+++ b/examples/python/requirements.txt
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-apache-iggy==0.5.0
+apache-iggy
loguru==0.7.3
argparse==1.4.0
diff --git a/scripts/run-python-examples-from-readme.sh
b/scripts/run-python-examples-from-readme.sh
new file mode 100755
index 000000000..43abb0648
--- /dev/null
+++ b/scripts/run-python-examples-from-readme.sh
@@ -0,0 +1,168 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Script to run Python examples from README.md and examples/python/README.md
files
+# Usage: ./scripts/run-python-examples-from-readme.sh [TARGET]
+#
+# TARGET - Optional target architecture (e.g., x86_64-unknown-linux-musl)
+# If not provided, uses the default target
+#
+# This script will run all the commands from both README.md and
examples/python/README.md files
+# and check if they pass or fail.
+# If any command fails, it will print the command and exit with non-zero
status.
+# If all commands pass, it will remove the log file and exit with zero status.
+#
+# Note: This script assumes that the iggy-server is not running and will start
it in the background.
+# It will wait until the server is started before running the commands.
+# It will also terminate the server after running all the commands.
+# Script executes every command in README files which is enclosed in
backticks (`) and starts
+# with `python `. Other commands are ignored.
+# Order of commands in README files is important as script will execute
them from top to bottom.
+
+readonly LOG_FILE="iggy-server.log"
+readonly PID_FILE="iggy-server.pid"
+readonly TIMEOUT=300
+
+TARGET="${1:-}" # Iggy server target architecture
+
+if [ -n "${TARGET}" ]; then
+ echo "Using target architecture: ${TARGET}"
+else
+ echo "Using default target architecture"
+fi
+
+# Remove old server data if present
+test -d local_data && rm -fr local_data
+test -e ${LOG_FILE} && rm ${LOG_FILE}
+test -e ${PID_FILE} && rm ${PID_FILE}
+
+# Check if server binary exists
+SERVER_BIN=""
+if [ -n "${TARGET}" ]; then
+ SERVER_BIN="target/${TARGET}/debug/iggy-server"
+else
+ SERVER_BIN="target/debug/iggy-server"
+fi
+
+if [ ! -f "${SERVER_BIN}" ]; then
+ echo "Error: Server binary not found at ${SERVER_BIN}"
+ echo "Please build the server binary before running this script:"
+ if [ -n "${TARGET}" ]; then
+ echo " cargo build --target ${TARGET} --bin iggy-server"
+ else
+ echo " cargo build --bin iggy-server"
+ fi
+ exit 1
+fi
+
+echo "Using server binary at ${SERVER_BIN}"
+
+# Run iggy server using the prebuilt binary
+echo "Starting server from ${SERVER_BIN}..."
+IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy ${SERVER_BIN} --fresh
&>${LOG_FILE} &
+echo $! >${PID_FILE}
+
+# Wait until "Iggy server has started" string is present inside iggy-server.log
+SERVER_START_TIME=0
+while ! grep -q "has started" ${LOG_FILE}; do
+ if [ ${SERVER_START_TIME} -gt ${TIMEOUT} ]; then
+ echo "Server did not start within ${TIMEOUT} seconds."
+ ps fx
+ cat ${LOG_FILE}
+ exit 1
+ fi
+ echo "Waiting for Iggy server to start... ${SERVER_START_TIME}"
+ sleep 1
+ ((SERVER_START_TIME += 1))
+done
+
+cd examples/python || exit 1
+
+# Set up Python virtual environment and install dependencies
+echo "Setting up Python virtual environment..."
+python3 -m venv .venv
+# shellcheck disable=SC1091
+source .venv/bin/activate
+
+# Build and install Python SDK from repository
+echo "Building Python SDK from repository..."
+pip install -q maturin patchelf
+(cd ../../foreign/python && maturin develop -q)
+
+# Install other example dependencies (excluding apache-iggy which we built
locally)
+echo "Installing example dependencies..."
+pip install -q loguru argparse
+
+# Execute all example commands from examples/python/README.md and check if
they pass or fail
+exit_code=0
+if [ -f "README.md" ]; then
+ while IFS= read -r command; do
+ # Remove backticks and comments from command
+ command=$(echo "${command}" | tr -d '`' | sed 's/^#.*//')
+ # Skip empty lines
+ if [ -z "${command}" ]; then
+ continue
+ fi
+
+ echo -e "\e[33mChecking example command from
examples/python/README.md:\e[0m ${command}"
+ echo ""
+
+ set +e
+ eval "timeout 10 ${command}"
+ test_exit_code=$?
+ set -e
+
+ # Stop at first failure
+ # Since examples might last longer, timeout error 124 is ignored here
+ if [[ $test_exit_code -ne 0 && $test_exit_code -ne 124 ]]; then
+ echo ""
+ echo -e "\e[31mExample command failed:\e[0m ${command}"
+ echo ""
+ exit_code=$test_exit_code
+ break
+ fi
+ # Add a small delay between examples to avoid potential race conditions
+ sleep 2
+
+ done < <(grep -E "^python " "README.md")
+fi
+
+# Deactivate and clean up virtual environment
+deactivate 2>/dev/null || true
+rm -rf .venv
+
+cd ../..
+
+# Terminate server
+kill -TERM "$(cat ${PID_FILE})"
+test -e ${PID_FILE} && rm ${PID_FILE}
+
+# If everything is ok remove log and pid files otherwise cat server log
+if [ "${exit_code}" -eq 0 ]; then
+ echo "Test passed"
+else
+ echo "Test failed, see log file:"
+ test -e ${LOG_FILE} && cat ${LOG_FILE}
+fi
+
+test -e ${LOG_FILE} && rm ${LOG_FILE}
+test -e ${PID_FILE} && rm ${PID_FILE}
+
+exit "${exit_code}"
