This is an automated email from the ASF dual-hosted git repository.
honahx pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 3b06a8034 Update Makefile for python client with auto setup (#1995)
3b06a8034 is described below
commit 3b06a803458aa61ad691d2edd132157dafbfb2d3
Author: Yong Zheng <yongzheng0...@gmail.com>
AuthorDate: Thu Jul 10 15:44:45 2025 -0500
Update Makefile for python client with auto setup (#1995)
Automate python client setup and use a virtual env instead to avoid change
an end-users' OS python
---
.github/workflows/python-client.yml | 18 +--
client/python/.pre-commit-config.yaml | 7 +-
client/python/Makefile | 82 +++++++++++++-
client/python/README.md | 14 +--
client/python/cli/constants.py | 205 +++++++++++++++++++---------------
client/python/cli/polaris_cli.py | 8 +-
client/python/pyproject.toml | 2 +-
7 files changed, 212 insertions(+), 124 deletions(-)
diff --git a/.github/workflows/python-client.yml
b/.github/workflows/python-client.yml
index 37ead61ed..7586e3b18 100644
--- a/.github/workflows/python-client.yml
+++ b/.github/workflows/python-client.yml
@@ -58,15 +58,6 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- - name: Install Poetry
- run: |
- pip install --user --upgrade -r regtests/requirements.txt
-
- # TODO: add cache for poetry dependencies once we have poetry.lock in
the repo
- - name: Install dependencies
- working-directory: client/python
- run: poetry install --all-extras
-
- name: Lint
working-directory: client/python
run: |
@@ -75,15 +66,14 @@ jobs:
- name: Generated Client Tests
working-directory: client/python
run: |
- export SCRIPT_DIR="non-existing-mock-directory"
- poetry run pytest test/
+ make test-client
- name: Image build
run: |
./gradlew \
- :polaris-server:assemble \
- :polaris-server:quarkusAppPartsBuild --rerun \
- -Dquarkus.container-image.build=true
+ :polaris-server:assemble \
+ :polaris-server:quarkusAppPartsBuild --rerun \
+ -Dquarkus.container-image.build=true
- name: Integration Tests
working-directory: client/python
diff --git a/client/python/.pre-commit-config.yaml
b/client/python/.pre-commit-config.yaml
index 84b1ab95e..5ffb2bfc6 100644
--- a/client/python/.pre-commit-config.yaml
+++ b/client/python/.pre-commit-config.yaml
@@ -23,10 +23,12 @@ repos:
- id: end-of-file-fixer
- id: debug-statements
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.11.13
+ rev: v0.12.1
hooks:
- - id: ruff
+ # Run the linter.
+ - id: ruff-check
args: [ --fix, --exit-non-zero-on-fix ]
+ # Run the formatter.
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.16.0
@@ -34,3 +36,4 @@ repos:
- id: mypy
args:
[--disallow-untyped-defs, --ignore-missing-imports, --install-types,
--non-interactive]
+ files: 'integration_tests/.*\.py'
diff --git a/client/python/Makefile b/client/python/Makefile
index 7d65922ab..9d222fa39 100644
--- a/client/python/Makefile
+++ b/client/python/Makefile
@@ -15,10 +15,65 @@
# specific language governing permissions and limitations
# under the License.
-regenerate-client:
+# .SILENT:
+
+# Configures the shell for recipes to use bash, enabling bash commands and
ensuring
+# that recipes exit on any command failure (including within pipes).
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+# Version information
+VERSION ?= $(shell cat pyproject.toml | grep version | sed 's/version *=
*"\(.*\)"/\1/')
+BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%S%:z")
+GIT_COMMIT := $(shell git rev-parse HEAD)
+POETRY_VERSION := $(shell cat pyproject.toml | grep requires-poetry | sed
's/requires-poetry *= *"\(.*\)"/\1/')
+
+# Variables
+VENV_DIR := .venv
+
+.PHONY: help
+help: ## Display this help.
+ @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make
\033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "
\033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n",
substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+.PHONY: version
+version: ## Print version information.
+ @echo "Apache Polaris version: ${VERSION}"
+ @echo "Build date: ${BUILD_DATE}"
+ @echo "Git commit: ${GIT_COMMIT}"
+ @echo "Poetry version: ${POETRY_VERSION}"
+
+# Target to create the virtual environment directory
+$(VENV_DIR):
+ @echo "Setting up Python virtual environment at $(VENV_DIR)..."
+ python3 -m venv $(VENV_DIR)
+ @echo "Virtual environment created."
+
+.PHONY: setup-env
+setup-env: $(VENV_DIR) install-poetry-deps
+
+.PHONY: install-poetry-deps
+install-poetry-deps:
+ @echo "Installing Poetry and project dependencies into $(VENV_DIR)..."
+ # Ensure pip is up-to-date within the venv
+ $(VENV_DIR)/bin/pip install --upgrade pip
+ # Install poetry if not already present
+ @if [ ! -f "$(VENV_DIR)/bin/poetry" ]; then \
+ $(VENV_DIR)/bin/pip install --upgrade
"poetry${POETRY_VERSION}"; \
+ fi
+ # Install needed dependencies using poetry
+ $(VENV_DIR)/bin/poetry install --all-extras
+ @echo "Poetry and dependencies installed."
+
+.PHONY: regenerate-client
+regenerate-client: ## Regenerate the client code
../templates/regenerate.sh
-test-integration:
+.PHONY: test-client
+test-client: setup-env ## Run client tests
+ SCRIPT_DIR="non-existing-mock-directory" $(VENV_DIR)/bin/poetry run
pytest test/
+
+.PHONY: test-integration
+test-integration: setup-env ## Run integration tests
docker compose -f docker-compose.yml kill
docker compose -f docker-compose.yml rm -f
docker compose -f docker-compose.yml up -d
@@ -28,8 +83,25 @@ test-integration:
echo "Still waiting for HTTP 200 from /q/health..."; \
done
@echo "Polaris is healthy. Starting integration tests..."
- poetry run pytest integration_tests/ ${PYTEST_ARGS}
+ $(VENV_DIR)/bin/poetry run pytest integration_tests/ ${PYTEST_ARGS}
+
+.PHONY: lint
+lint: setup-env ## Run linting checks
+ $(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/*
cli/*
+.PHONY: clean-venv
+clean-venv:
+ @echo "Attempting to remove virtual environment directory:
$(VENV_DIR)..."
+ # SAFETY CHECK: Ensure VENV_DIR is not empty and exists before
attempting to remove
+ @if [ -n "$(VENV_DIR)" ] && [ -d "$(VENV_DIR)" ]; then \
+ rm -rf "$(VENV_DIR)"; \
+ echo "Virtual environment removed."; \
+ else \
+ echo "Virtual environment directory '$(VENV_DIR)' not found or
VENV_DIR is empty. No action taken."; \
+ fi
-lint:
- poetry run pre-commit run --files integration_tests/*
+.PHONY: clean
+clean: clean-venv ## Cleanup
+ @echo "Cleaning up Python cache files..."
+ find . -type f -name "*.pyc" -delete
+ find . -type d -name "__pycache__" -delete
diff --git a/client/python/README.md b/client/python/README.md
index cec952681..4489452a1 100644
--- a/client/python/README.md
+++ b/client/python/README.md
@@ -6,9 +6,9 @@
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,17 +24,13 @@ The Apache Polaris Python package provides a client for
interacting with the Apa
### Prerequisites
- Python 3.9 or later
-- poetry >= 2.0
+- poetry >= 2.1
### Installation
-First we need to generate the OpenAPI client code from the OpenAPI
specification.
+First we need to generate the OpenAPI client code from the OpenAPI
specification.
```
make regenerate-client
```
-Install the project with test dependencies:
-```
-poetry install --all-extras
-```
### Auto-formatting and Linting
```
@@ -44,4 +40,4 @@ make lint
### Running Integration Tests
```
make test-integration
-```
\ No newline at end of file
+```
diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py
index f82ff2caa..93b36d998 100644
--- a/client/python/cli/constants.py
+++ b/client/python/cli/constants.py
@@ -53,8 +53,8 @@ class CatalogConnectionType(Enum):
Represents a ConnectionType for an EXTERNAL catalog -- see
ConnectionConfigInfo in the spec
"""
- HADOOP = 'hadoop'
- ICEBERG = 'iceberg-rest'
+ HADOOP = "hadoop"
+ ICEBERG = "iceberg-rest"
class AuthenticationType(Enum):
@@ -62,9 +62,9 @@ class AuthenticationType(Enum):
Represents a AuthenticationType for an EXTERNAL catalog -- see
AuthenticationParameters in the spec
"""
- OAUTH = 'oauth'
- BEARER = 'bearer'
- SIGV4 = 'sigv4'
+ OAUTH = "oauth"
+ BEARER = "bearer"
+ SIGV4 = "sigv4"
class ServiceIdentityType(Enum):
@@ -72,7 +72,7 @@ class ServiceIdentityType(Enum):
Represents a Service Identity Type for an EXTERNAL catalog -- see
ServiceIdentityInfo in the spec
"""
- AWS_IAM = 'aws_iam'
+ AWS_IAM = "aws_iam"
class Commands:
@@ -129,57 +129,57 @@ class Arguments:
These values should be snake_case, but they will get mapped to kebab-case
in `Parser.parse`
"""
- TYPE = 'type'
- DEFAULT_BASE_LOCATION = 'default_base_location'
- STORAGE_TYPE = 'storage_type'
- ALLOWED_LOCATION = 'allowed_location'
- ROLE_ARN = 'role_arn'
- EXTERNAL_ID = 'external_id'
- USER_ARN = 'user_arn'
- TENANT_ID = 'tenant_id'
- MULTI_TENANT_APP_NAME = 'multi_tenant_app_name'
- CONSENT_URL = 'consent_url'
- SERVICE_ACCOUNT = 'service_account'
- CATALOG_ROLE = 'catalog_role'
- CATALOG = 'catalog'
- PRINCIPAL = 'principal'
- CLIENT_ID = 'client_id'
- PRINCIPAL_ROLE = 'principal_role'
- PROPERTY = 'property'
- SET_PROPERTY = 'set_property'
- REMOVE_PROPERTY = 'remove_property'
- PRIVILEGE = 'privilege'
- NAMESPACE = 'namespace'
- TABLE = 'table'
- VIEW = 'view'
- CASCADE = 'cascade'
- CLIENT_SECRET = 'client_secret'
- ACCESS_TOKEN = 'access_token'
- HOST = 'host'
- PORT = 'port'
- BASE_URL = 'base_url'
- PARENT = 'parent'
- LOCATION = 'location'
- REGION = 'region'
- PROFILE = 'profile'
- PROXY = 'proxy'
- HADOOP_WAREHOUSE = 'hadoop_warehouse'
- ICEBERG_REMOTE_CATALOG_NAME = 'iceberg_remote_catalog_name'
- CATALOG_CONNECTION_TYPE = 'catalog_connection_type'
- CATALOG_AUTHENTICATION_TYPE = 'catalog_authentication_type'
- CATALOG_SERVICE_IDENTITY_TYPE = 'catalog_service_identity_type'
- CATALOG_SERVICE_IDENTITY_IAM_ARN = 'catalog_service_identity_iam_arn'
- CATALOG_URI = 'catalog_uri'
- CATALOG_TOKEN_URI = 'catalog_token_uri'
- CATALOG_CLIENT_ID = 'catalog_client_id'
- CATALOG_CLIENT_SECRET = 'catalog_client_secret'
- CATALOG_CLIENT_SCOPE = 'catalog_client_scope'
- CATALOG_BEARER_TOKEN = 'catalog_bearer_token'
- CATALOG_ROLE_ARN = 'catalog_role_arn'
- CATALOG_ROLE_SESSION_NAME = 'catalog_role_session_name'
- CATALOG_EXTERNAL_ID = 'catalog_external_id'
- CATALOG_SIGNING_REGION = 'catalog_signing_region'
- CATALOG_SIGNING_NAME = 'catalog_signing_name'
+ TYPE = "type"
+ DEFAULT_BASE_LOCATION = "default_base_location"
+ STORAGE_TYPE = "storage_type"
+ ALLOWED_LOCATION = "allowed_location"
+ ROLE_ARN = "role_arn"
+ EXTERNAL_ID = "external_id"
+ USER_ARN = "user_arn"
+ TENANT_ID = "tenant_id"
+ MULTI_TENANT_APP_NAME = "multi_tenant_app_name"
+ CONSENT_URL = "consent_url"
+ SERVICE_ACCOUNT = "service_account"
+ CATALOG_ROLE = "catalog_role"
+ CATALOG = "catalog"
+ PRINCIPAL = "principal"
+ CLIENT_ID = "client_id"
+ PRINCIPAL_ROLE = "principal_role"
+ PROPERTY = "property"
+ SET_PROPERTY = "set_property"
+ REMOVE_PROPERTY = "remove_property"
+ PRIVILEGE = "privilege"
+ NAMESPACE = "namespace"
+ TABLE = "table"
+ VIEW = "view"
+ CASCADE = "cascade"
+ CLIENT_SECRET = "client_secret"
+ ACCESS_TOKEN = "access_token"
+ HOST = "host"
+ PORT = "port"
+ BASE_URL = "base_url"
+ PARENT = "parent"
+ LOCATION = "location"
+ REGION = "region"
+ PROFILE = "profile"
+ PROXY = "proxy"
+ HADOOP_WAREHOUSE = "hadoop_warehouse"
+ ICEBERG_REMOTE_CATALOG_NAME = "iceberg_remote_catalog_name"
+ CATALOG_CONNECTION_TYPE = "catalog_connection_type"
+ CATALOG_AUTHENTICATION_TYPE = "catalog_authentication_type"
+ CATALOG_SERVICE_IDENTITY_TYPE = "catalog_service_identity_type"
+ CATALOG_SERVICE_IDENTITY_IAM_ARN = "catalog_service_identity_iam_arn"
+ CATALOG_URI = "catalog_uri"
+ CATALOG_TOKEN_URI = "catalog_token_uri"
+ CATALOG_CLIENT_ID = "catalog_client_id"
+ CATALOG_CLIENT_SECRET = "catalog_client_secret"
+ CATALOG_CLIENT_SCOPE = "catalog_client_scope"
+ CATALOG_BEARER_TOKEN = "catalog_bearer_token"
+ CATALOG_ROLE_ARN = "catalog_role_arn"
+ CATALOG_ROLE_SESSION_NAME = "catalog_role_session_name"
+ CATALOG_EXTERNAL_ID = "catalog_external_id"
+ CATALOG_SIGNING_REGION = "catalog_signing_region"
+ CATALOG_SIGNING_NAME = "catalog_signing_name"
class Hints:
@@ -237,39 +237,64 @@ class Hints:
DEFAULT_BASE_LOCATION = "A new default base location for the
catalog"
class External:
- CATALOG_CONNECTION_TYPE = 'The type of external catalog in
[ICEBERG, HADOOP].'
- CATALOG_AUTHENTICATION_TYPE = 'The type of authentication in
[OAUTH, BEARER, SIGV4]'
- CATALOG_SERVICE_IDENTITY_TYPE = 'The type of service identity in
[AWS_IAM]'
-
- CATALOG_SERVICE_IDENTITY_IAM_ARN = ('When using the AWS_IAM
service identity type, this is the ARN '
- 'of the IAM user or IAM role
Polaris uses to assume roles and '
- 'then access external
resources.')
-
- CATALOG_URI = 'The URI of the external catalog'
- HADOOP_WAREHOUSE = 'The warehouse to use when federating to a
HADOOP catalog'
- ICEBERG_REMOTE_CATALOG_NAME = 'The remote catalog name when
federating to an Iceberg REST catalog'
-
-
- CATALOG_TOKEN_URI = '(For authentication type OAUTH) Token server
URI'
- CATALOG_CLIENT_ID = '(For authentication type OAUTH) oauth client
id'
- CATALOG_CLIENT_SECRET = '(For authentication type OAUTH) oauth
client secret (input-only)'
- CATALOG_CLIENT_SCOPE = ('(For authentication type OAUTH) oauth
scopes to specify when exchanging '
- 'for a short-lived access token. Multiple
can be provided by specifying'
- ' this option more than once')
-
- CATALOG_BEARER_TOKEN = '(For authentication type BEARER) Bearer
token (input-only)'
-
- CATALOG_ROLE_ARN = ('(For authentication type SIGV4) The aws IAM
role arn assumed by polaris '
- 'userArn when signing requests')
- CATALOG_ROLE_SESSION_NAME = ('(For authentication type SIGV4) The
role session name to be used '
- 'by the SigV4 protocol for signing
requests')
- CATALOG_EXTERNAL_ID = ('(For authentication type SIGV4) An
optional external id used to establish '
- 'a trust relationship with AWS in the trust
policy')
- CATALOG_SIGNING_REGION = ('(For authentication type SIGV4) Region
to be used by the SigV4 protocol '
- 'for signing requests')
- CATALOG_SIGNING_NAME = ('(For authentication type SIGV4) The
service name to be used by the SigV4 '
- 'protocol for signing requests, the
default signing name is "execute-api" '
- 'is if not provided')
+ CATALOG_CONNECTION_TYPE = (
+ "The type of external catalog in [ICEBERG, HADOOP]."
+ )
+ CATALOG_AUTHENTICATION_TYPE = (
+ "The type of authentication in [OAUTH, BEARER, SIGV4]"
+ )
+ CATALOG_SERVICE_IDENTITY_TYPE = "The type of service identity in
[AWS_IAM]"
+
+ CATALOG_SERVICE_IDENTITY_IAM_ARN = (
+ "When using the AWS_IAM service identity type, this is the ARN
"
+ "of the IAM user or IAM role Polaris uses to assume roles and "
+ "then access external resources."
+ )
+
+ CATALOG_URI = "The URI of the external catalog"
+ HADOOP_WAREHOUSE = (
+ "The warehouse to use when federating to a HADOOP catalog"
+ )
+ ICEBERG_REMOTE_CATALOG_NAME = (
+ "The remote catalog name when federating to an Iceberg REST
catalog"
+ )
+
+ CATALOG_TOKEN_URI = "(For authentication type OAUTH) Token server
URI"
+ CATALOG_CLIENT_ID = "(For authentication type OAUTH) oauth client
id"
+ CATALOG_CLIENT_SECRET = (
+ "(For authentication type OAUTH) oauth client secret
(input-only)"
+ )
+ CATALOG_CLIENT_SCOPE = (
+ "(For authentication type OAUTH) oauth scopes to specify when
exchanging "
+ "for a short-lived access token. Multiple can be provided by
specifying"
+ " this option more than once"
+ )
+
+ CATALOG_BEARER_TOKEN = (
+ "(For authentication type BEARER) Bearer token (input-only)"
+ )
+
+ CATALOG_ROLE_ARN = (
+ "(For authentication type SIGV4) The aws IAM role arn assumed
by polaris "
+ "userArn when signing requests"
+ )
+ CATALOG_ROLE_SESSION_NAME = (
+ "(For authentication type SIGV4) The role session name to be
used "
+ "by the SigV4 protocol for signing requests"
+ )
+ CATALOG_EXTERNAL_ID = (
+ "(For authentication type SIGV4) An optional external id used
to establish "
+ "a trust relationship with AWS in the trust policy"
+ )
+ CATALOG_SIGNING_REGION = (
+ "(For authentication type SIGV4) Region to be used by the
SigV4 protocol "
+ "for signing requests"
+ )
+ CATALOG_SIGNING_NAME = (
+ "(For authentication type SIGV4) The service name to be used
by the SigV4 "
+ 'protocol for signing requests, the default signing name is
"execute-api" '
+ "is if not provided"
+ )
class Principals:
class Create:
diff --git a/client/python/cli/polaris_cli.py b/client/python/cli/polaris_cli.py
index 47e803865..83341ada4 100644
--- a/client/python/cli/polaris_cli.py
+++ b/client/python/cli/polaris_cli.py
@@ -158,9 +158,11 @@ class PolarisCli:
# Authenticate accordingly
if options.base_url:
if options.host is not None or options.port is not None:
- raise Exception(f'Please provide either
{Argument.to_flag_name(Arguments.BASE_URL)} or'
- f' {Argument.to_flag_name(Arguments.HOST)} &'
- f' {Argument.to_flag_name(Arguments.PORT)},
but not both')
+ raise Exception(
+ f"Please provide either
{Argument.to_flag_name(Arguments.BASE_URL)} or"
+ f" {Argument.to_flag_name(Arguments.HOST)} &"
+ f" {Argument.to_flag_name(Arguments.PORT)}, but not both"
+ )
polaris_management_url = f"{options.base_url}/api/management/v1"
polaris_catalog_url = f"{options.base_url}/api/catalog/v1"
diff --git a/client/python/pyproject.toml b/client/python/pyproject.toml
index a5e20d97f..9740437bf 100644
--- a/client/python/pyproject.toml
+++ b/client/python/pyproject.toml
@@ -42,7 +42,7 @@ homepage = "https://polaris.apache.org/";
repository = "https://github.com/apache/polaris/";
[tool.poetry]
-requires-poetry = ">=2.1"
+requires-poetry = "==2.1.3"
packages = [{ include = "polaris" }]
[tool.poetry.group.test.dependencies]
