This is an automated email from the ASF dual-hosted git repository. kumaab pushed a commit to branch ranger_5653 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 1b06fbdb3d0b89eaa82c1df1c51f2d52740ef100 Author: Abhishek Kumar <[email protected]> AuthorDate: Fri Jun 26 11:43:16 2026 -0700 RANGER-5653: Updates to apache-ranger Python client with Python 3.13+ support only --- .cursor/rules/ranger-python.mdc | 88 +++ intg/scripts/setup.py | 30 - intg/src/main/python/README.md | 679 +++++---------------- intg/src/main/python/setup.py | 16 +- intg/src/test/python/test_ranger_client.py | 225 +++++-- .../sample-client/scripts/run-pyclient.sh | 50 -- ranger-examples/sample-client/scripts/setup.py | 30 - 7 files changed, 450 insertions(+), 668 deletions(-) diff --git a/.cursor/rules/ranger-python.mdc b/.cursor/rules/ranger-python.mdc new file mode 100644 index 000000000..f309344f3 --- /dev/null +++ b/.cursor/rules/ranger-python.mdc @@ -0,0 +1,88 @@ +--- +description: Apache Ranger Python client and test conventions (apache-ranger on PyPI) +globs: intg/**/*.py,ranger-examples/sample-client/**/*.py +alwaysApply: false +--- + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Ranger Python client (`apache-ranger`) + +**Authoritative package root:** `intg/src/main/python/` — packaging via `setup.py` there only. Do not add duplicate `setup.py` scripts elsewhere. + +**Python version:** `apache-ranger` targets **Python 3.13+** (`python_requires` in `setup.py`). + +## Style and structure + +- Preserve the ASF license header on new files (match neighboring modules). +- Match existing layout: `apache_ranger/client/` for HTTP clients, `apache_ranger/model/` for typed models, `utils.py` for coercion helpers. +- Reuse `type_coerce` / `type_coerce_attrs()` — accept dict-like input and coerce to model types; do not duplicate parsing logic. +- Keep changes minimal; follow naming, imports, and spacing of the file you edit. + +## Vertical alignment + +Within a **consecutive block** of related lines, align `=` (and dict values after `:` where keys share one dict): + +```python +self.status_code = status_code +self.response = response +self.content = content + +from apache_ranger.exceptions import RangerServiceException +from apache_ranger.model.ranger_service import RangerService + +RangerAuthzRequest({ + "requestId": "req-1", + "user": RangerUserInfo({"name": "alice"}), + "access": RangerAccessInfo({...}), + "context": RangerAccessContext({...}), +}) +``` + +Apply alignment only within the block; do not pad unrelated single assignments. Reference points: `exceptions.py`, `sample_pdp_client.py`, `test_ranger_client.py`. + +## Model and sample data + +Use **compact dict literals** — one line per top-level field when still readable (`sample_pdp_client.py`). Avoid verbose nesting when a flat line is clear. + +## Unit tests + +Primary test file: `intg/src/test/python/test_ranger_client.py` + +Run from `intg/`: + +```bash +PYTHONPATH=src/main/python python -B src/test/python/test_ranger_client.py +``` + +Split tests **by client** — one `unittest.TestCase` per area (`TestRangerClient`, `TestGDSClient`, `TestPDPClient`). + +Fixtures: shared data as **class-level constants** (`URL`, `AUTH`, `AUTHZ_REQUEST`); single-use data inline in the test method. + +Assertions and mocking: + +- Prefer `assertRaises`, `assertIsNone`, `assertIsInstance` over bare `try/except`. +- Assert **current** client behavior — not obsolete error text or exceptions. +- Mock at the boundary: `Session` for Admin HTTP, `client_http.call_api` for GDS/PDP. +- Only add tests for real coercion, exports, or API wiring. + +## References + +- `intg/src/main/python/README.md` — published API surface and quickstarts +- `ranger-examples/sample-client/src/main/python/` — Admin, GDS, PDP, KMS usage examples diff --git a/intg/scripts/setup.py b/intg/scripts/setup.py deleted file mode 100644 index 9ce977301..000000000 --- a/intg/scripts/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup, find_packages - -# External dependencies -requirements = ['requests>=2.24'] - -setup( - name='pyclient', - packages=['pylib', 'pylib/ranger/client', 'pylib/ranger/model'], - data_files=[('../../lib/resources', ['./../src/main/resources/logging.conf'])], - install_requires=requirements, - zip_safe=False -) diff --git a/intg/src/main/python/README.md b/intg/src/main/python/README.md index fd6b2e2c0..e9caec873 100644 --- a/intg/src/main/python/README.md +++ b/intg/src/main/python/README.md @@ -17,69 +17,62 @@ specific language governing permissions and limitations under the License. --> -# Apache Ranger - Python client +# Apache Ranger Python Client -Python library for Apache Ranger. +`apache-ranger` is the official Python client package for Apache Ranger. +It provides typed helpers for Ranger Admin, user/group management, KMS, +Governed Data Sharing (GDS), and Policy Decision Point (PDP) APIs. + +## Requirements + +- Python 3.13 or later +- Apache Ranger Admin, KMS, or PDP service reachable from your Python process ## Installation -Use the package manager [pip](https://pip.pypa.io/en/stable/) to install python client for Apache Ranger. +Install the client from PyPI: ```bash -> pip install apache-ranger -> pip install requests_kerberos (If using kerberos for authentication) +pip install apache-ranger ``` -Verify if apache-ranger client is installed: -```bash -> pip list +For Kerberos/SPNEGO authentication, install the optional Kerberos dependency: -Package Version ------------- --------- -apache-ranger 0.0.12 +```bash +pip install requests-kerberos ``` -## Python API clients - -### Ranger Admin (`RangerClient`) +Verify the installed package: -Base URL: `http(s)://<ranger-admin-host>:<port>` - -`RangerClient` wraps APIs under `service/public/v2/api`, including: +```bash +python -m pip show apache-ranger +``` -- service-def CRUD -- service CRUD -- policy CRUD/apply/search -- roles -- security-zones -- plugin-info and policy-delta maintenance +## Supported Clients -Authentication options: +- `RangerClient`: Ranger Admin APIs for service definitions, services, policies, + roles, security zones, plugin information, and policy delta maintenance. +- `RangerUserMgmtClient`: user, group, and user-group management APIs. +- `RangerKMSClient`: Ranger KMS key management APIs. +- `RangerGdsClient`: Governed Data Sharing APIs for datasets, projects, + datashares, shared resources, and GDS policies. +- `RangerPDPClient`: PDP authorization APIs for single authorization, batch + authorization, and effective resource permissions. -- Basic auth tuple, for example `("admin", "password")` -- Kerberos/SPNEGO (`requests-kerberos`) -- custom headers/query params via constructor +## Quick Start -Example: +### Ranger Admin ```python from apache_ranger.client.ranger_client import RangerClient ranger = RangerClient("http://localhost:6080";, ("admin", "rangerR0cks!")) + services = ranger.find_services() -print(len(services.list)) +print(f"{len(services.list)} services found") ``` -### Ranger User Management (`RangerUserMgmtClient`) - -`RangerUserMgmtClient` builds on `RangerClient` and covers user/group/group-user operations, including: - -- create/get/update/delete user -- create/get/update/delete group -- add/remove/list group-user mappings -- list groups for user, list users in group - -Example: +### User And Group Management ```python from apache_ranger.client.ranger_client import RangerClient @@ -89,534 +82,198 @@ ranger = RangerClient("http://localhost:6080";, ("admin", "rangerR0cks!")) user_mgmt = RangerUserMgmtClient(ranger) users = user_mgmt.find_users() -print(len(users.list)) -``` - -### Ranger KMS (`RangerKMSClient`) - -Base URL: `http(s)://<kms-host>:<port>` - -`RangerKMSClient` wraps KMS APIs such as: - -- create/delete key -- rollover/get metadata/current version -- generate/decrypt/reencrypt encrypted keys -- status and key-name listing - -Authentication options: +groups = user_mgmt.find_groups() -- Hadoop simple auth (`HadoopSimpleAuth("user")`) -- Kerberos/SPNEGO (`requests-kerberos`) -- Basic auth tuple (when enabled) +print(f"{len(users.list)} users found") +print(f"{len(groups.list)} groups found") +``` -Example: +### Ranger KMS ```python -from apache_ranger.client.ranger_kms_client import RangerKMSClient from apache_ranger.client.ranger_client import HadoopSimpleAuth +from apache_ranger.client.ranger_kms_client import RangerKMSClient kms = RangerKMSClient("http://localhost:9292";, HadoopSimpleAuth("keyadmin")) + print(kms.kms_status()) ``` -### Ranger PDP (`RangerPDPClient`) - -`RangerPDPClient` is a thin Python wrapper for the PDP REST APIs exposed at `http(s)://<pdp-host>:<pdp-port>/authz/v1`. - -Endpoints: - -- `POST /authz/v1/authorize` -> single access evaluation -- `POST /authz/v1/authorizeMulti` -> batch access evaluation -- `POST /authz/v1/permissions` -> effective permissions for a resource +### Governed Data Sharing -Request context requirements: - -- `context.serviceType` (for example: `hive`, `hdfs`, `kafka`) -- `context.serviceName` (Ranger service name, for example: `dev_hive`) -- for `authorize` and `authorizeMulti`, `user.name` must be present - -Authentication options: - -- **Kerberos/SPNEGO** - - install dependency: `pip install requests-kerberos` - - use `HTTPKerberosAuth()` as `auth` in `RangerPDPClient` -- **Trusted header** - - pass caller header (must be configured using `ranger.pdp.authn.header.username`) - - only behind a trusted proxy -- **JWT bearer** - - pass `Authorization: Bearer <token>` in request headers +```python +from apache_ranger.client.ranger_client import RangerClient +from apache_ranger.client.ranger_gds_client import RangerGdsClient -Delegation behavior: +ranger = RangerClient("http://localhost:6080";, ("admin", "rangerR0cks!")) +gds = RangerGdsClient(ranger) -- if caller differs from `request.user.name`, delegation permission is required -- delegation users are configured with `ranger.pdp.service.<service-name>.delegation.users` - (or wildcard `ranger.pdp.service.*.delegation.users`) -- without delegation permission, PDP returns `403 FORBIDDEN` +datasets = gds.find_datasets() +print(f"{len(datasets.list)} datasets found") +``` -`RangerPDPClient` example (Kerberos): +### Policy Decision Point ```python -from requests_kerberos import HTTPKerberosAuth from apache_ranger.client.ranger_pdp_client import RangerPDPClient from apache_ranger.model.ranger_authz import ( - RangerAccessContext, RangerAccessInfo, RangerAuthzRequest, - RangerResourceInfo, RangerUserInfo + RangerAccessContext, + RangerAccessInfo, + RangerAuthzRequest, + RangerResourceInfo, + RangerUserInfo, ) -pdp = RangerPDPClient("http://localhost:6500";, HTTPKerberosAuth()) +pdp = RangerPDPClient( + "http://localhost:6500";, + auth=None, + headers={"X-Forwarded-User": "hive"}, +) -req = RangerAuthzRequest({ - 'requestId': 'req-1', - 'user': RangerUserInfo({'name': 'alice'}), - 'access': RangerAccessInfo({ - 'resource': RangerResourceInfo({'name': 'table:default/test_tbl1', 'subResources': ['column:id', 'column:name', 'column:email']}), - 'action': 'QUERY', - 'permissions': ['select'] +request = RangerAuthzRequest({ + "requestId": "req-1", + "user": RangerUserInfo({"name": "alice"}), + "access": RangerAccessInfo({ + "resource": RangerResourceInfo({"name": "table:default/test_tbl1"}), + "permissions": ["select"], + }), + "context": RangerAccessContext({ + "serviceType": "hive", + "serviceName": "dev_hive", }), - 'context': RangerAccessContext({'serviceType': 'hive', 'serviceName': 'dev_hive'}) }) -res = pdp.authorize(req) -print(res.decision) +result = pdp.authorize(request) +print(result.decision) ``` -Raw REST example using `requests` (JWT bearer): - -```python -import requests - -url = "http://localhost:6500/authz/v1/authorize"; -headers = { - "Authorization": "Bearer <jwt-token>", - "Content-Type": "application/json" -} -payload = { - "requestId": "req-1", - "user": {"name": "alice"}, - "access": { - "resource": {"name": "table:default/test_tbl1", "subResources": ["column:id", "column:name", "column:email"]}, - "action": "QUERY", - "permissions": ["select"] - }, - "context": {"serviceType": "hive", "serviceName": "dev_hive"} -} - -resp = requests.post(url, headers=headers, json=payload, timeout=30) -resp.raise_for_status() -print(resp.json()) -``` - -## Usage - -```python test_ranger.py``` -```python -# test_ranger.py - -from apache_ranger.model.ranger_service import * -from apache_ranger.client.ranger_client import * -from apache_ranger.model.ranger_policy import * - - -## Step 1: create a client to connect to Apache Ranger admin -ranger_url = 'http://localhost:6080' -ranger_auth = ('admin', 'rangerR0cks!') - -# For Kerberos authentication -# -# from requests_kerberos import HTTPKerberosAuth -# -# ranger_auth = HTTPKerberosAuth() - -ranger = RangerClient(ranger_url, ranger_auth) - -# to disable SSL certificate validation (not recommended for production use!) -# -# ranger.session.verify = False - +## Authentication -## Step 2: Let's create a service -service = RangerService() -service.name = 'test_hive' -service.type = 'hive' -service.configs = {'username':'hive', 'password':'hive', 'jdbc.driverClassName': 'org.apache.hive.jdbc.HiveDriver', 'jdbc.url': 'jdbc:hive2://ranger-hadoop:10000', 'hadoop.security.authorization': 'true'} +Use the authentication mechanism configured for the Ranger service you are +calling: -print('Creating service: name=' + service.name) +- Basic auth: pass a `(username, password)` tuple to `RangerClient`. +- Kerberos/SPNEGO: pass `requests_kerberos.HTTPKerberosAuth()` after installing + `requests-kerberos`. +- KMS Hadoop simple auth: use `HadoopSimpleAuth("user")` when simple auth is + enabled for Ranger KMS. +- Custom headers or query parameters: pass `headers` or `query_params` to the + client constructor when the deployment requires them. +- PDP trusted-header or JWT auth: pass the configured trusted caller header or + `Authorization: Bearer <token>` header to `RangerPDPClient`. -created_service = ranger.create_service(service) +Example Kerberos setup: -print(' created service: name=' + created_service.name + ', id=' + str(created_service.id)) - - -## Step 3: Let's create a policy -policy = RangerPolicy() -policy.service = service.name -policy.name = 'test policy' -policy.resources = { 'database': RangerPolicyResource({ 'values': ['test_db'] }), - 'table': RangerPolicyResource({ 'values': ['test_tbl'] }), - 'column': RangerPolicyResource({ 'values': ['*'] }) } - -allowItem1 = RangerPolicyItem() -allowItem1.users = [ 'admin' ] -allowItem1.accesses = [ RangerPolicyItemAccess({ 'type': 'create' }), - RangerPolicyItemAccess({ 'type': 'alter' }) ] - -denyItem1 = RangerPolicyItem() -denyItem1.users = [ 'admin' ] -denyItem1.accesses = [ RangerPolicyItemAccess({ 'type': 'drop' }) ] - -policy.policyItems = [ allowItem1 ] -policy.denyPolicyItems = [ denyItem1 ] - -print('Creating policy: name=' + policy.name) - -created_policy = ranger.create_policy(policy) - -print(' created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) - - -## Step 4: Delete policy and service created above -print('Deleting policy: id=' + str(created_policy.id)) - -ranger.delete_policy_by_id(created_policy.id) - -print(' deleted policy: id=' + str(created_policy.id)) - -print('Deleting service: id=' + str(created_service.id)) - -ranger.delete_service_by_id(created_service.id) - -print(' deleted service: id=' + str(created_service.id)) - -``` - -```python test_ranger_user_mgmt.py``` ```python -# test_ranger_user_mgmt.py -from apache_ranger.client.ranger_client import * -from apache_ranger.utils import * -from apache_ranger.model.ranger_user_mgmt import * -from apache_ranger.client.ranger_user_mgmt_client import * -from datetime import datetime - -## -## Step 1: create a client to connect to Apache Ranger -## -ranger_url = 'http://localhost:6080' -ranger_auth = ('admin', 'rangerR0cks!') - -# For Kerberos authentication -# -# from requests_kerberos import HTTPKerberosAuth -# -# ranger_auth = HTTPKerberosAuth() -# -# For HTTP Basic authentication -# -# ranger_auth = ('admin', 'rangerR0cks!') - -ranger = RangerClient(ranger_url, ranger_auth) -user_mgmt = RangerUserMgmtClient(ranger) - - - -## -## Step 2: Let's call User Management APIs -## - -print('\nListing users') - -users = user_mgmt.find_users() - -print(f' {len(users.list)} users found') - -for user in users.list: - print(f' id: {user.id}, name: {user.name}') - - -print('\nListing groups') - -groups = user_mgmt.find_groups() - -print(f' {len(groups.list)} groups found') - -for group in groups.list: - print(f' id: {group.id}, name: {group.name}') - -print('\nListing group-users') - -group_users = user_mgmt.find_group_users() - -print(f' {len(group_users.list)} group-users found') - -for group_user in group_users.list: - print(f' id: {group_user.id}, groupId: {group_user.parentGroupId}, userId: {group_user.userId}') - - -now = datetime.now() - -name_suffix = '-' + now.strftime('%Y%m%d-%H%M%S-%f') -user_name = 'test-user' + name_suffix -group_name = 'test-group' + name_suffix - - -user = RangerUser({ 'name': user_name, 'firstName': user_name, 'lastName': 'user', 'emailAddress': user_name + '@test.org', 'password': 'Welcome1', 'userRoleList': [ 'ROLE_USER' ], 'otherAttributes': '{ "dept": "test" }' }) - -print(f'\nCreating user: name={user.name}') - -created_user = user_mgmt.create_user(user) - -print(f' created user: {created_user}') - - -group = RangerGroup({ 'name': group_name, 'otherAttributes': '{ "dept": "test" }' }) - -print(f'\nCreating group: name={group.name}') - -created_group = user_mgmt.create_group(group) - -print(f' created group: {created_group}') - - -group_user = RangerGroupUser({ 'name': created_group.name, 'parentGroupId': created_group.id, 'userId': created_user.id }) - -print(f'\nAdding user {created_user.name} to group {created_group.name}') - -created_group_user = user_mgmt.create_group_user(group_user) - -print(f' created group-user: {created_group_user}') - - -print('\nListing group-users') - -group_users = user_mgmt.find_group_users() - -print(f' {len(group_users.list)} group-users found') - -for group_user in group_users.list: - print(f' id: {group_user.id}, groupId: {group_user.parentGroupId}, userId: {group_user.userId}') - - -print(f'\nListing users for group {group.name}') - -users = user_mgmt.get_users_in_group(group.name) - -print(f' users: {users}') - - -print(f'\nListing groups for user {user.name}') +from requests_kerberos import HTTPKerberosAuth +from apache_ranger.client.ranger_client import RangerClient -groups = user_mgmt.get_groups_for_user(user.name) +ranger = RangerClient("https://ranger.example.com:6182";, HTTPKerberosAuth()) +``` -print(f' groups: {groups}') +## PDP Request Notes +`RangerPDPClient` sends requests to the PDP REST APIs exposed under +`/authz/v1`. -print(f'\nDeleting group-user {created_group_user.id}') +- `authorize(request)` calls `POST /authz/v1/authorize`. +- `authorize_multi(request)` calls `POST /authz/v1/authorizeMulti`. +- `get_resource_permissions(request)` calls `POST /authz/v1/permissions`. -user_mgmt.delete_group_user_by_id(created_group_user.id) +PDP authorization requests should include: +- `context.serviceType`, for example `hive`, `hdfs`, or `kafka`. +- `context.serviceName`, the Ranger service name. +- `user.name` for `authorize` and `authorize_multi` calls. -print(f'\nDeleting group {group.name}') +If the authenticated caller is different from `request.user.name`, the caller +must be allowed to delegate for that service. Without delegation permission, PDP +returns `403 FORBIDDEN`. -user_mgmt.delete_group_by_id(created_group.id, True) +## Examples and Code References +The quick starts above cover the basics. For day-to-day API usage — creating +services and policies, managing users, calling KMS or PDP, or working with GDS +— use the runnable samples and client sources below. -print(f'\nDeleting user {user.name}') +### Running sample scripts -user_mgmt.delete_user_by_id(created_user.id, True) -``` +After `pip install apache-ranger`, run a sample directly: -```python test_ranger_kms.py``` -```python -# test_ranger_kms.py -from apache_ranger.client.ranger_kms_client import RangerKMSClient -from apache_ranger.client.ranger_client import HadoopSimpleAuth -from apache_ranger.model.ranger_kms import RangerKey -import time - - -## -## Step 1: create a client to connect to Apache Ranger KMS -## -kms_url = 'http://localhost:9292' -kms_auth = HadoopSimpleAuth('keyadmin') - -# For Kerberos authentication -# -# from requests_kerberos import HTTPKerberosAuth -# -# kms_auth = HTTPKerberosAuth() -# -# For HTTP Basic authentication -# -# kms_auth = ('keyadmin', 'rangerR0cks!') - -kms_client = RangerKMSClient(kms_url, kms_auth) - - - -## -## Step 2: Let's call KMS APIs -## - -kms_status = kms_client.kms_status() -print('kms_status():', kms_status) -print() - -key_name = 'test_' + str(int(time.time() * 1000)) - -key = kms_client.create_key(RangerKey({'name':key_name})) -print('create_key(' + key_name + '):', key) -print() - -rollover_key = kms_client.rollover_key(key_name, key.material) -print('rollover_key(' + key_name + '):', rollover_key) -print() - -kms_client.invalidate_cache_for_key(key_name) -print('invalidate_cache_for_key(' + key_name + ')') -print() - -key_metadata = kms_client.get_key_metadata(key_name) -print('get_key_metadata(' + key_name + '):', key_metadata) -print() - -current_key = kms_client.get_current_key(key_name) -print('get_current_key(' + key_name + '):', current_key) -print() - -encrypted_keys = kms_client.generate_encrypted_key(key_name, 6) -print('generate_encrypted_key(' + key_name + ', ' + str(6) + '):') -for i in range(len(encrypted_keys)): - encrypted_key = encrypted_keys[i] - decrypted_key = kms_client.decrypt_encrypted_key(key_name, encrypted_key.versionName, encrypted_key.iv, encrypted_key.encryptedKeyVersion.material) - reencrypted_key = kms_client.reencrypt_encrypted_key(key_name, encrypted_key.versionName, encrypted_key.iv, encrypted_key.encryptedKeyVersion.material) - print(' encrypted_keys[' + str(i) + ']: ', encrypted_key) - print(' decrypted_key[' + str(i) + ']: ', decrypted_key) - print(' reencrypted_key[' + str(i) + ']:', reencrypted_key) -print() - -reencrypted_keys = kms_client.batch_reencrypt_encrypted_keys(key_name, encrypted_keys) -print('batch_reencrypt_encrypted_keys(' + key_name + ', ' + str(len(encrypted_keys)) + '):') -for i in range(len(reencrypted_keys)): - print(' batch_reencrypt_encrypted_key[' + str(i) + ']:', reencrypted_keys[i]) -print() - -key_versions = kms_client.get_key_versions(key_name) -print('get_key_versions(' + key_name + '):', len(key_versions)) -for i in range(len(key_versions)): - print(' key_versions[' + str(i) + ']:', key_versions[i]) -print() - -for i in range(len(key_versions)): - key_version = kms_client.get_key_version(key_versions[i].versionName) - print('get_key_version(' + str(i) + '):', key_version) -print() - -key_names = kms_client.get_key_names() -print('get_key_names():', len(key_names)) -for i in range(len(key_names)): - print(' key_name[' + str(i) + ']:', key_names[i]) -print() - -keys_metadata = kms_client.get_keys_metadata(key_names) -print('get_keys_metadata(' + str(key_names) + '):', len(keys_metadata)) -for i in range(len(keys_metadata)): - print(' key_metadata[' + str(i) + ']:', keys_metadata[i]) -print() - -key = kms_client.get_key(key_name) -print('get_key(' + key_name + '):', key) -print() - -kms_client.delete_key(key_name) -print('delete_key(' + key_name + ')') +```bash +python ranger-examples/sample-client/src/main/python/sample_client.py ``` -```python test_ranger_pdp.py``` -```python -from apache_ranger.client.ranger_pdp_client import RangerPDPClient -from apache_ranger.model.ranger_authz import RangerAccessContext, RangerAccessInfo -from apache_ranger.model.ranger_authz import RangerAuthzRequest, RangerMultiAuthzRequest -from apache_ranger.model.ranger_authz import RangerResourceInfo, RangerResourcePermissionsRequest, RangerUserInfo - -## -## Step 1: create a client to connect to Ranger PDP -## -pdp_url = 'http://localhost:6500' - -# For Kerberos authentication -# -# from requests_kerberos import HTTPKerberosAuth -# -# pdp = RangerPDPClient(pdp_url, HTTPKerberosAuth()) - -# For trusted-header authN with PDP (example only): -# -pdp = RangerPDPClient(pdp_url, auth=None, headers={ 'X-Forwarded-User': 'hive' }) - -## -## Step 2: call PDP authorization APIs -## -req = RangerAuthzRequest({ - 'requestId': 'req-1', - 'user': RangerUserInfo({'name': 'alice'}), - 'access': RangerAccessInfo({'resource': RangerResourceInfo({'name': 'table:default/test_tbl1'}), 'permissions': ['create']}), - 'context': RangerAccessContext({'serviceType': 'hive', 'serviceName': 'dev_hive'}) -}) - -res = pdp.authorize(req) - -print('authorize():') -print(f' {req}') -print(f' {res}') -print('\n') +When working from a Ranger source checkout without installing the package, set +`PYTHONPATH` to the client sources first: +```bash +# from the repository root +PYTHONPATH=intg/src/main/python \ + python ranger-examples/sample-client/src/main/python/sample_client.py +``` -req = RangerAuthzRequest({ - 'requestId': 'req-2', - 'user': RangerUserInfo({'name': 'alice'}), - 'access': RangerAccessInfo({'resource': RangerResourceInfo({'name': 'table:default/test_tbl1', 'subResources': ['column:id', 'column:name', 'column:email']}), 'permissions': ['select']}), - 'context': RangerAccessContext({'serviceType': 'hive', 'serviceName': 'dev_hive'}) -}) +Edit the `ranger_url`, `ranger_auth`, and other connection settings at the top +of each sample before running it against your environment. -res = pdp.authorize(req) +### Runnable examples by task -print('authorize():') -print(f' {req}') -print(f' {res}') -print('\n') +| Task | Sample script | What it demonstrates | +| --- | --- | --- | +| Ranger Admin — services, policies, roles, tags | [`sample_client.py`](https://github.com/apache/ranger/blob/master/ranger-examples/sample-client/src/main/python/sample_client.py) | Service-def and service CRUD, access/data-masking/row-filter policies, roles, service tags | +| User and group management | [`user_mgmt.py`](https://github.com/apache/ranger/blob/master/ranger-examples/sample-client/src/main/python/user_mgmt.py) | List/create/delete users and groups, group-user mappings | +| Ranger KMS | [`sample_kms_client.py`](https://github.com/apache/ranger/blob/master/ranger-examples/sample-client/src/main/python/sample_kms_client.py) | Key lifecycle, encrypt/decrypt/reencrypt, metadata and status | +| PDP authorization | [`sample_pdp_client.py`](https://github.com/apache/ranger/blob/master/ranger-examples/sample-client/src/main/python/sample_pdp_client.py) | `authorize`, `authorize_multi`, `get_resource_permissions` | +| Governed Data Sharing (GDS) | [`sample_gds_client.py`](https://github.com/apache/ranger/blob/master/ranger-examples/sample-client/src/main/python/sample_gds_client.py) | Datasets, projects, datashares, shared resources, GDS policies | +| Security zones (v2) | [`security_zone_v2.py`](https://github.com/apache/ranger/blob/master/ranger-examples/sample-client/src/main/python/security_zone_v2.py) | Create/update security zones and zone resources | +All samples live under +[`ranger-examples/sample-client/src/main/python`](https://github.com/apache/ranger/tree/master/ranger-examples/sample-client/src/main/python). -req = RangerMultiAuthzRequest({ - 'requestId': 'req-3', - 'user': RangerUserInfo({'name': 'alice'}), - 'accesses': [ - RangerAccessInfo({'resource': RangerResourceInfo({'name': 'table:default/test_tbl1', 'subResources': ['column:id', 'column:name', 'column:email'], 'attributes': {'OWNER': 'alice'}}), 'permissions': ['select']}), - RangerAccessInfo({'resource': RangerResourceInfo({'name': 'table:default/test_vw1'}), 'permissions': ['create']}) - ], - 'context': RangerAccessContext({'serviceType': 'hive', 'serviceName': 'dev_hive'}) -}) +### Client API source code -res = pdp.authorize_multi(req) +Each public client is implemented under +[`intg/src/main/python/apache_ranger/client/`](https://github.com/apache/ranger/tree/master/intg/src/main/python/apache_ranger/client): -print('authorize_multi():') -print(f' {req}') -print(f' {res}') -print('\n') +| Client | Source file | REST base path | +| --- | --- | --- | +| `RangerClient` | [`ranger_client.py`](https://github.com/apache/ranger/blob/master/intg/src/main/python/apache_ranger/client/ranger_client.py) | `service/public/v2/api` | +| `RangerUserMgmtClient` | [`ranger_user_mgmt_client.py`](https://github.com/apache/ranger/blob/master/intg/src/main/python/apache_ranger/client/ranger_user_mgmt_client.py) | user/group APIs on Ranger Admin | +| `RangerKMSClient` | [`ranger_kms_client.py`](https://github.com/apache/ranger/blob/master/intg/src/main/python/apache_ranger/client/ranger_kms_client.py) | KMS REST APIs | +| `RangerGdsClient` | [`ranger_gds_client.py`](https://github.com/apache/ranger/blob/master/intg/src/main/python/apache_ranger/client/ranger_gds_client.py) | GDS APIs on Ranger Admin | +| `RangerPDPClient` | [`ranger_pdp_client.py`](https://github.com/apache/ranger/blob/master/intg/src/main/python/apache_ranger/client/ranger_pdp_client.py) | `/authz/v1` | +Request and response models are under +[`intg/src/main/python/apache_ranger/model/`](https://github.com/apache/ranger/tree/master/intg/src/main/python/apache_ranger/model). -req = RangerResourcePermissionsRequest({ - 'requestId': 'req-4', - 'resource': RangerResourceInfo({'name': 'table:default/test_tbl1'}), - 'context': RangerAccessContext({'serviceType': 'hive', 'serviceName': 'dev_hive'}) -}) +### Unit tests -res = pdp.get_resource_permissions(req) +[`intg/src/test/python/test_ranger_client.py`](https://github.com/apache/ranger/blob/master/intg/src/test/python/test_ranger_client.py) +contains mocked examples for Admin, GDS, and PDP client wiring. Run from +`intg/`: -print('get_resource_permissions():') -print(f' {req}') -print(f' {res}') -print('\n') +```bash +PYTHONPATH=src/main/python python -B src/test/python/test_ranger_client.py ``` -For more examples, checkout `sample-client` python project in [ranger-examples](https://github.com/apache/ranger/blob/master/ranger-examples/sample-client/src/main/python) module (including `sample_client.py`, `user_mgmt.py`, `sample_kms_client.py`, and `sample_pdp_client.py`). +## Troubleshooting + +- `ModuleNotFoundError: requests_kerberos`: install `requests-kerberos`. +- `401 Unauthorized`: verify the credentials, Kerberos ticket, or auth headers + used by the target Ranger service. +- `403 Forbidden`: verify Ranger permissions and PDP delegation configuration + when authorizing on behalf of another user. +- SSL certificate errors: configure a valid trust store for production. For + local testing only, `ranger.session.verify = False` can disable certificate + verification. +- Connection timeouts: verify the Ranger Admin, KMS, or PDP URL and confirm the + service is reachable from the client host. + +## Release 0.0.13 Highlights + +- Python 3.13+ support. +- New GDS client coverage for datasets, projects, datashares, shared resources, + and GDS policy APIs. +- New PDP client coverage for Ranger authorization APIs. +- New authorization request and response models. +- User/group management, KMS, Ranger Admin, and model updates. diff --git a/intg/src/main/python/setup.py b/intg/src/main/python/setup.py index 89db40e64..f57f08536 100644 --- a/intg/src/main/python/setup.py +++ b/intg/src/main/python/setup.py @@ -19,25 +19,27 @@ from setuptools import setup, find_packages # External dependencies -requirements = ['requests>=2.24','strenum>=0.4.15'] +requirements = ['requests>=2.34.2', 'strenum>=0.4.15'] long_description = '' -with open("README.md", "r") as fh: +with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() setup( name="apache-ranger", - version="0.0.12", + version="0.0.13", author="Apache Ranger", author_email="[email protected]", description="Apache Ranger Python client", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/apache/ranger/tree/master/intg/src/main/python";, - license='Apache LICENSE 2.0', + license="Apache-2.0", + license_files=("LICENSE",), classifiers=[ - "Programming Language :: Python :: 2.7", - "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", ], packages=find_packages(), @@ -45,5 +47,5 @@ include_package_data=True, zip_safe=False, keywords='ranger client, apache ranger', - python_requires='>=2.7', + python_requires='>=3.13', ) diff --git a/intg/src/test/python/test_ranger_client.py b/intg/src/test/python/test_ranger_client.py index cbedcbf76..ae56dc9e2 100644 --- a/intg/src/test/python/test_ranger_client.py +++ b/intg/src/test/python/test_ranger_client.py @@ -18,14 +18,29 @@ import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch + from apache_ranger.exceptions import RangerServiceException from apache_ranger.model.ranger_service import RangerService try: - from apache_ranger.client.ranger_client import API, HttpMethod, HTTPStatus, RangerClient -except ModuleNotFoundError: # requests not installed - exit() # skipping unit tests + import apache_ranger.client as ranger_client_package + import apache_ranger.model as ranger_model_package + + from apache_ranger.client.ranger_client import API, HttpMethod, HTTPStatus, RangerClient + from apache_ranger.client.ranger_gds_client import RangerGdsClient + from apache_ranger.client.ranger_pdp_client import RangerPDPClient + from apache_ranger.model.ranger_authz import ( + RangerAccessContext, RangerAccessInfo, RangerAuthzRequest, RangerAuthzResult, + RangerPermissionResult, RangerResourceInfo, RangerResultInfo, RangerUserInfo, + ) + from apache_ranger.model.ranger_gds import ( + DatasetSummary, GdsPermission, GdsShareStatus, RangerDataset, + RangerGdsMaskInfo, RangerGdsObjectACL, + ) + from apache_ranger.model.ranger_principal import PrincipalType +except ModuleNotFoundError: # requests not installed + exit() # skipping unit tests class MockResponse: @@ -33,7 +48,6 @@ def __init__(self, status_code, response=None, content=None): self.status_code = status_code self.response = response self.content = content - return def json(self): return self.response @@ -49,67 +63,198 @@ class TestRangerClient(unittest.TestCase): @patch('apache_ranger.client.ranger_client.Session') def test_get_service_unavailable(self, mock_session): mock_session.return_value.get.return_value = MockResponse(HTTPStatus.SERVICE_UNAVAILABLE) - result = RangerClient(TestRangerClient.URL, TestRangerClient.AUTH).find_services() - - self.assertTrue(result is None) + result = RangerClient(self.URL, self.AUTH).find_services() + self.assertIsNone(result) @patch('apache_ranger.client.ranger_client.Session') def test_get_success(self, mock_session): - response = [ RangerService() ] - mock_session.return_value.get.return_value = MockResponse(HTTPStatus.OK, response=response, content='Success') - result = RangerClient(TestRangerClient.URL, TestRangerClient.AUTH).find_services() + response = [RangerService()] + mock_session.return_value.get.return_value = MockResponse( + HTTPStatus.OK, response=response, content='Success') + result = RangerClient(self.URL, self.AUTH).find_services() self.assertEqual(response, result) - @patch('apache_ranger.client.ranger_client.Session') @patch('apache_ranger.client.ranger_client.Response') def test_get_unexpected_status_code(self, mock_response, mock_session): - content = 'Internal Server Error' - mock_response.text = content - mock_response.content = content - mock_response.status_code = HTTPStatus.INTERNAL_SERVER_ERROR + content = 'Internal Server Error' + mock_response.text = content + mock_response.content = content + mock_response.status_code = HTTPStatus.INTERNAL_SERVER_ERROR mock_session.return_value.get.return_value = mock_response - try: - RangerClient(TestRangerClient.URL, TestRangerClient.AUTH).find_services() - except RangerServiceException as e: - self.assertTrue(HTTPStatus.INTERNAL_SERVER_ERROR, e.statusCode) + with self.assertRaises(RangerServiceException) as context: + RangerClient(self.URL, self.AUTH).find_services() + self.assertEqual(HTTPStatus.INTERNAL_SERVER_ERROR, context.exception.statusCode) @patch('apache_ranger.client.ranger_client.RangerClient.FIND_SERVICES') def test_unexpected_http_method(self, mock_api): - mock_api.method.return_value = "PATCH" - mock_api.url = TestRangerClient.URL - mock_api.path = RangerClient.URI_SERVICE + mock_api.method = "PATCH" + mock_api.path = RangerClient.URI_SERVICE - try: - RangerClient(TestRangerClient.URL, TestRangerClient.AUTH).find_services() - except RangerServiceException as e: - self.assertTrue('Unsupported HTTP Method' in repr(e)) + result = RangerClient(self.URL, self.AUTH).find_services() + self.assertIsNone(result) def test_url_missing_format(self): - params = {'arg1': 1, 'arg2': 2} + with self.assertRaises(KeyError): + API("{arg1}test{arg2}path{arg3}", HttpMethod.GET, HTTPStatus.OK).format_path( + {'arg1': 1, 'arg2': 2}) - try: - API("{arg1}test{arg2}path{arg3}", HttpMethod.GET, HTTPStatus.OK).format_path(params) + def test_url_invalid_format(self): + with self.assertRaises(TypeError): + API("{}test{}path{}", HttpMethod.GET, HTTPStatus.OK).format_path({'1', '2'}) - self.fail("Supposed to fail") - except KeyError as e: - self.assertTrue('KeyError' in repr(e)) +class TestGDSClient(unittest.TestCase): + def test_package_export(self): + self.assertIs(ranger_client_package.RangerGdsClient, RangerGdsClient) + self.assertIn("RangerGdsClient", ranger_client_package.__all__) - def test_url_invalid_format(self): - params = {'1', '2'} + def test_dataset_type_coercion(self): + dataset = RangerDataset({ + "name": "sales", + "acl": { + "public": { + "users": { + "alice": "VIEW", + }, + "groups": { + "analytics": "POLICY_ADMIN", + }, + }, + }, + }) + + dataset.type_coerce_attrs() + + acl = dataset.acl["public"] + self.assertIsInstance(acl, RangerGdsObjectACL) + self.assertEqual(GdsPermission.VIEW, acl.users["alice"]) + self.assertEqual(GdsPermission.POLICY_ADMIN, acl.groups["analytics"]) + + def test_dataset_summary_type_coercion(self): + summary = DatasetSummary({ + "name": "sales", + "permissionForCaller": "VIEW", + "principalsCount": { + "USER": 2, + "GROUP": 1, + }, + "dataShares": [{ + "name": "sales-share", + "shareStatus": "ACTIVE", + }], + }) + + summary.type_coerce_attrs() + + self.assertEqual(GdsPermission.VIEW, summary.permissionForCaller) + self.assertEqual(2, summary.principalsCount[PrincipalType.USER]) + self.assertEqual(GdsShareStatus.ACTIVE, summary.dataShares[0].shareStatus) + + def test_mask_info_type_coercion(self): + mask_info = RangerGdsMaskInfo({ + "values": ["email"], + "maskInfo": { + "dataMaskType": "MASK_HASH", + "valueExpr": "hash(email)", + }, + }) + + mask_info.type_coerce_attrs() + + self.assertEqual(["email"], mask_info.values) + self.assertEqual("MASK_HASH", mask_info.maskInfo.dataMaskType) + + def test_find_datasets_calls_expected_api(self): + ranger_client = Mock() + ranger_client.client_http.call_api.return_value = { + "list": [{ + "id": 1, + "name": "sales", + }], + } + + result = RangerGdsClient(ranger_client).find_datasets({"name": "sales"}) + + ranger_client.client_http.call_api.assert_called_once_with( + RangerGdsClient.FIND_DATASETS, + {"name": "sales"}, + ) + self.assertEqual(1, len(result.list)) + self.assertIsInstance(result.list[0], RangerDataset) + + +class TestPDPClient(unittest.TestCase): + PDP_URL = "http://localhost:6500"; + AUTHZ_REQUEST = { + "requestId": "req-1", + "user": {"name": "alice", "groups": ["analytics"]}, + "access": {"resource": {"name": "table:default/sales", "subResources": ["column:id"]}, "permissions": ["select"]}, + "context": {"serviceType": "hive", "serviceName": "dev_hive"}, + } + AUTHZ_RESULT = { + "requestId": "req-1", + "decision": RangerAuthzResult.DECISION_ALLOW, + "permissions": {"select": { + "access": {"decision": RangerAuthzResult.DECISION_ALLOW, "policy": {"id": 7, "version": 3}}, + "subResources": {"column:id": {"access": {"decision": RangerAuthzResult.DECISION_DENY}}}, + }}, + } + + def test_package_export(self): + self.assertIs(ranger_client_package.RangerPDPClient, RangerPDPClient) + self.assertIn("RangerPDPClient", ranger_client_package.__all__) + + def test_authz_model_exports(self): + self.assertIs(ranger_model_package.RangerAuthzRequest, RangerAuthzRequest) + self.assertIs(ranger_model_package.RangerAuthzResult, RangerAuthzResult) + self.assertIn("RangerAuthzRequest", ranger_model_package.__all__) + self.assertIn("RangerResourcePermissions", ranger_model_package.__all__) + + def test_authz_request_type_coercion(self): + request = RangerAuthzRequest(self.AUTHZ_REQUEST) + request.type_coerce_attrs() + + self.assertIsInstance(request.user, RangerUserInfo) + self.assertIsInstance(request.access, RangerAccessInfo) + self.assertIsInstance(request.access.resource, RangerResourceInfo) + self.assertIsInstance(request.context, RangerAccessContext) + self.assertEqual("alice", request.user.name) + self.assertEqual(["select"], request.access.permissions) + + def test_authz_result_type_coercion(self): + result = RangerAuthzResult(self.AUTHZ_RESULT) + result.type_coerce_attrs() + + permission = result.permissions["select"] + self.assertIsInstance(permission, RangerPermissionResult) + self.assertEqual(7, permission.access.policy.id) + self.assertIsInstance(permission.subResources["column:id"], RangerResultInfo) + + def test_authorize_calls_expected_api(self): + client = RangerPDPClient(self.PDP_URL, auth=None) + client.client_http.call_api = Mock(return_value={ + "requestId": "req-1", + "decision": RangerAuthzResult.DECISION_ALLOW, + }) + + result = client.authorize(self.AUTHZ_REQUEST) - try: - API("{}test{}path{}", HttpMethod.GET, HTTPStatus.OK).format_path(params) + client.client_http.call_api.assert_called_once() + api = client.client_http.call_api.call_args.args[0] - self.fail("Supposed to fail") - except TypeError as e: - self.assertTrue('TypeError' in repr(e)) + self.assertEqual(RangerPDPClient.URI_AUTHORIZE, api.path) + self.assertEqual(HttpMethod.POST, api.method) + self.assertIsInstance( + client.client_http.call_api.call_args.kwargs["request_data"], + RangerAuthzRequest, + ) + self.assertIsInstance(result, RangerAuthzResult) if __name__ == '__main__': diff --git a/ranger-examples/sample-client/scripts/run-pyclient.sh b/ranger-examples/sample-client/scripts/run-pyclient.sh deleted file mode 100755 index 8649f0bfb..000000000 --- a/ranger-examples/sample-client/scripts/run-pyclient.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage() { - echo "usage: run-pyclient.sh - -n <arg> url to connect to - -h show help." - exit 1 -} - -cd venv || exit -source bin/activate -INSTALL="python3 ./../setup.py install" -$INSTALL >> build.log -PYTHON_CMD="python3 build/lib/pylib/sample_client.py" -while getopts "n:h" opt; do - case $opt in - n) URL=$OPTARG - PYTHON_CMD="$PYTHON_CMD --url $URL" - ;; - h) usage - ;; - \?) echo -e \\n"Option -$OPTARG not allowed." - usage - ;; - esac -done - -prompt="Sample Authentication User Name:" -read -p "$prompt" userName -prompt="Sample Authentication User Password:" -read -p "$prompt" -s password -printf "\n" -PYTHON_CMD="$PYTHON_CMD --username $userName --password $password" -printf "Python command : %s\n" "$PYTHON_CMD" -$PYTHON_CMD \ No newline at end of file diff --git a/ranger-examples/sample-client/scripts/setup.py b/ranger-examples/sample-client/scripts/setup.py deleted file mode 100644 index c4e740769..000000000 --- a/ranger-examples/sample-client/scripts/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup, find_packages - -# External dependencies -requirements = ['requests>=2.24'] - -setup( - name='sample_pyclient', - packages=['pylib', 'pylib/ranger/client', 'pylib/ranger/model'], - data_files=[('../../lib/resources', ['resources/logging.conf'])], - install_requires=requirements, - zip_safe=False -)
