This is an automated email from the ASF dual-hosted git repository. mgreber pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 8e7f51cd77877027f529440e43ef5e1aa835550e Author: Marton Greber <[email protected]> AuthorDate: Wed Oct 8 11:07:32 2025 +0200 KUDU-3478 Add Python client examples test script This patch adds test-python-examples.sh to verify that Python client examples run correctly against a local Kudu cluster. The script: - Tests examples with both Python 2 and Python 3 - Creates isolated virtualenvs for each Python version - Builds and installs the Python client from source - Starts a local Kudu cluster using shared test-cluster-common.sh - Runs example scripts and verifies they complete successfully - Supports testing all examples or individual scripts The script is integrated into build-and-test.sh to run during CI for both BUILD_PYTHON (Python2) and BUILD_PYTHON3 builds. Change-Id: Ic2ea271a8d33a285e4bf222afa17ec77583a5b8b Reviewed-on: http://gerrit.cloudera.org:8080/23509 Tested-by: Marton Greber <[email protected]> Reviewed-by: Alexey Serbin <[email protected]> Reviewed-by: Zoltan Martonka <[email protected]> --- build-support/jenkins/build-and-test.sh | 12 +++ python/Makefile | 18 ++++ python/test-python-examples.sh | 168 ++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+) diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index 0a27462fa..3c572ef1e 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -674,6 +674,12 @@ if [ "$BUILD_PYTHON" == "1" ]; then FAILURES="$FAILURES"$'Python tests failed\n' fi + # Run the Python examples tests + if ! ./test-python-examples.sh python2 ; then + TESTS_FAILED=1 + FAILURES="$FAILURES"$'Python examples tests failed\n' + fi + deactivate popd fi @@ -723,6 +729,12 @@ if [ "$BUILD_PYTHON3" == "1" ]; then FAILURES="$FAILURES"$'Python 3 tests failed\n' fi + # Run the Python examples tests + if ! ./test-python-examples.sh python3 ; then + TESTS_FAILED=1 + FAILURES="$FAILURES"$'Python 3 examples tests failed\n' + fi + deactivate popd fi diff --git a/python/Makefile b/python/Makefile index 7d3336fbf..775f09dc4 100644 --- a/python/Makefile +++ b/python/Makefile @@ -10,6 +10,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +.PHONY: all build requirements test install clean test-examples + build: python setup.py build_ext --inplace @@ -26,6 +28,19 @@ else python setup.py test endif +# Run Python client examples against a local Kudu cluster. +# Optional: PYTHON=python2|python3 (default: python3) +# Optional: EXAMPLE=<example_name.py> (default: all examples) +test-examples: +ifndef PYTHON + $(eval PYTHON := python3) +endif +ifdef EXAMPLE + ./test-python-examples.sh $(PYTHON) $(EXAMPLE) +else + ./test-python-examples.sh $(PYTHON) +endif + clean: find . -name "*.pyc" -exec rm -rf {} \; find . -name "*.so" -exec rm -rf {} \; @@ -36,3 +51,6 @@ clean: rm -f kudu/schema.cpp rm -f kudu/config.pxi rm -f kudu/version.py + +install: + python setup.py install \ No newline at end of file diff --git a/python/test-python-examples.sh b/python/test-python-examples.sh new file mode 100755 index 000000000..c5ec2b80e --- /dev/null +++ b/python/test-python-examples.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This script verifies that the Python client examples can run correctly +# against a local Kudu cluster. It is similar to client_examples-test.sh +# but for Python examples. +# +# Usage: +# test-python-examples.sh <python2|python3> [example-script] +# +# If no example script is provided, tests all basic Python examples. +# +# To add a new example: +# 1. Add the script name to the VALID_EXAMPLES array below +# 2. Add a case in the test_example() function with the command to run it + +# List of valid example scripts +VALID_EXAMPLES=( + "basic_example.py" + "non_unique_primary_key.py" +) + +PYTHON_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd) + +if [[ -z "$KUDU_HOME" ]]; then + # Try to infer it if not set + KUDU_HOME=$(cd "$PYTHON_DIR/.."; pwd) + echo "KUDU_HOME not set, inferring from script location: $KUDU_HOME" +fi + +if [[ ! -d "$KUDU_HOME" ]]; then + exit_error "KUDU_HOME directory does not exist: $KUDU_HOME" +fi + +EXAMPLES_DIR="$KUDU_HOME/examples/python" + +# Source the common cluster management functions +source "$KUDU_HOME/build-support/test-cluster-common.sh" + +BUILD_DIR="$KUDU_HOME/build/latest" +BIN_DIR="$BUILD_DIR/bin" + +# Track virtualenv directories for cleanup +VENV_DIRS=() + +# Clean up after the test. Must be idempotent. +cleanup() { + cleanup_cluster + for venv_dir in "${VENV_DIRS[@]}"; do + if [[ -n "$venv_dir" ]] && [[ -d "$venv_dir" ]]; then + rm -rf "$venv_dir" + fi + done +} +trap cleanup EXIT + +set -e +set -o pipefail +set -x + +test_example() { + local example_script=$1 + local python_version=$2 + + case "$example_script" in + basic_example.py) + echo "Testing basic_example.py..." + if ! "$python_version" "$EXAMPLES_DIR/basic-python-example/basic_example.py" \ + --masters $LOCALHOST_IP \ + --ports $MASTER_RPC_PORT ; then + exit_error "basic_example.py failed with $python_version" + fi + ;; + non_unique_primary_key.py) + echo "Testing non_unique_primary_key.py..." + if ! "$python_version" "$EXAMPLES_DIR/basic-python-example/non_unique_primary_key.py" \ + --masters $LOCALHOST_IP \ + --ports $MASTER_RPC_PORT ; then + exit_error "non_unique_primary_key.py failed with $python_version" + fi + ;; + *) + exit_error "Unknown example script: $example_script" + ;; + esac +} + +test_with_python_version() { + local python_version=$1 + local example_name=$2 + + if ! command -v "$python_version" &> /dev/null; then + echo "WARNING: $python_version not found, skipping $python_version tests" + return 0 + fi + + # Create a temporary virtualenv for this Python version + local venv_dir=$(mktemp -d -t kudu-python-examples-test-${python_version}.XXXXXXXXXXXXX) + VENV_DIRS+=("$venv_dir") + echo "Creating virtualenv for $python_version in $venv_dir" + virtualenv -p "$python_version" "$venv_dir" + source "$venv_dir/bin/activate" + + pip install -r "$PYTHON_DIR/requirements.txt" + make clean + make install + + echo "Running Python examples with $python_version..." + + if [[ -z "$example_name" ]]; then + for example in "${VALID_EXAMPLES[@]}"; do + test_example "$example" "$python_version" + done + else + test_example "$example_name" "$python_version" + fi + + echo "All $python_version examples completed successfully!" + + deactivate +} + +# Parse command-line arguments +PYTHON_VERSION="$1" +EXAMPLE_NAME="$2" + +# Validate Python version (required) +if [[ "$PYTHON_VERSION" != "python2" ]] && [[ "$PYTHON_VERSION" != "python3" ]]; then + exit_error "Usage: test-python-examples.sh <python2|python3> [example-script] +Valid example scripts: ${VALID_EXAMPLES[*]}" +fi + +# Validate example script name (optional) +if [[ -n "$EXAMPLE_NAME" ]]; then + valid=false + for example in "${VALID_EXAMPLES[@]}"; do + if [[ "$EXAMPLE_NAME" == "$example" ]]; then + valid=true + break + fi + done + if [[ "$valid" != "true" ]]; then + exit_error "Invalid example script: $EXAMPLE_NAME. Must be one of: ${VALID_EXAMPLES[*]}" + fi +fi + +start_test_cluster "$BIN_DIR" "python_examples-test" + +test_with_python_version "$PYTHON_VERSION" "$EXAMPLE_NAME" + +echo "All Python examples tests completed successfully!" \ No newline at end of file
