This is an automated email from the ASF dual-hosted git repository.
pradeepagrawal8184 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 178b76d46 RANGER-5839: Add Docker system tests to verify HDFS
group-policy cases (#1312)
178b76d46 is described below
commit 178b76d46d712676ada4d0f058a480c7c68a82a2
Author: Vishal Bhavsar <[email protected]>
AuthorDate: Wed Oct 14 20:32:05 2026 +0530
RANGER-5839: Add Docker system tests to verify HDFS group-policy cases
(#1312)
---
functional-tests/hdfs/conftest.py | 57 +++-
functional-tests/hdfs/test_hdfs_group_policy_03.py | 271 +++++++++++++++++++
functional-tests/hdfs/utils.py | 10 +-
functional-tests/kms/utils.py | 49 +++-
functional-tests/pytest.ini | 1 +
functional-tests/ranger_test_utils/hdfs_utils.py | 301 +++++++++++++++++++++
functional-tests/ranger_test_utils/utils.py | 21 ++
functional-tests/readme.md | 25 +-
8 files changed, 711 insertions(+), 24 deletions(-)
diff --git a/functional-tests/hdfs/conftest.py
b/functional-tests/hdfs/conftest.py
index deb0ea4cc..f72b448f4 100644
--- a/functional-tests/hdfs/conftest.py
+++ b/functional-tests/hdfs/conftest.py
@@ -16,18 +16,36 @@
import docker
import pytest
import time
-from hdfs.test_config import CORE_SITE_XML_PATH, HADOOP_CONTAINER, HDFS_USER,
KMS_CONTAINER, KMS_PROPERTY, TEST_EZ_PATHS, TEST_KMS_KEYS
-from hdfs.utils import cleanup_test_artifacts, ensure_hadoop_user_keytab,
ensure_kms_hdfs_policy, ensure_kms_kerberos_rules
-from kms.utils import ensure_keyadmin_keytab, ensure_ticket
+from hdfs.test_config import (
+ CORE_SITE_XML_PATH,
+ HADOOP_CONTAINER,
+ HDFS_USER,
+ KMS_CONTAINER,
+ KMS_PROPERTY,
+ TEST_EZ_PATHS,
+ TEST_KMS_KEYS,
+)
# Setup Docker Client
client = docker.from_env()
+
+def _session_needs_encryption_setup(session):
+ """Encryption/KMS session setup — skip for hdfs_policy-only runs."""
+ if not session.items:
+ return False
+ for item in session.items:
+ if item.get_closest_marker("hdfs_policy") is None:
+ return True
+ return False
+
+
@pytest.fixture(scope="session")
def hadoop_container():
- container = client.containers.get(HADOOP_CONTAINER) #to get hadoop
container instance
+ container = client.containers.get(HADOOP_CONTAINER)
return container
+
def ensure_key_provider_and_simple_auth(container) -> bool:
"""
Ensures:
@@ -63,6 +81,7 @@ def ensure_key_provider_and_simple_auth(container) -> bool:
return changed
+
def ensure_user_exists(container, username: str) -> None:
exit_code, _ = container.exec_run(f"id -u {username}", user="root")
if exit_code == 0:
@@ -71,6 +90,7 @@ def ensure_user_exists(container, username: str) -> None:
container.exec_run(f"useradd -m -s /bin/bash {username}", user="root")
container.exec_run(f"usermod -aG hadoop {username}", user="root")
+
def ensure_kms_provider_configured(container) -> bool:
"""
Ensures KMS provider property exists in core-site.xml.
@@ -104,8 +124,23 @@ def ensure_kms_provider_configured(container) -> bool:
return changed
+
@pytest.fixture(scope="session", autouse=True)
-def setup_environment(hadoop_container):
+def setup_encryption_environment(request):
+ if not _session_needs_encryption_setup(request.session):
+ yield
+ return
+
+ hadoop_container = client.containers.get(HADOOP_CONTAINER)
+
+ from hdfs.utils import (
+ cleanup_test_artifacts,
+ ensure_hadoop_user_keytab,
+ ensure_kms_hdfs_policy,
+ ensure_kms_kerberos_rules,
+ )
+ from kms.utils import ensure_keyadmin_keytab, ensure_ticket
+
changed = ensure_kms_provider_configured(hadoop_container)
if changed:
hadoop_container.restart()
@@ -118,14 +153,17 @@ def setup_environment(hadoop_container):
# safemode leave with kerberos (hdfs principal already has keytab in
container)
hadoop_container.exec_run(
- ["bash", "-c",
- "kinit -kt /etc/keytabs/hdfs.keytab
hdfs/[email protected] 2>/dev/null;"
- "hdfs dfsadmin -safemode leave"],
+ [
+ "bash",
+ "-c",
+ "kinit -kt /etc/keytabs/hdfs.keytab
hdfs/[email protected] 2>/dev/null;"
+ "hdfs dfsadmin -safemode leave",
+ ],
user=HDFS_USER,
)
kms_container = client.containers.get(KMS_CONTAINER)
- ensure_kms_kerberos_rules(kms_container) # already sleeps 30s internally
when restart is needed
+ ensure_kms_kerberos_rules(kms_container)
ensure_keyadmin_keytab()
ensure_ticket()
@@ -134,4 +172,3 @@ def setup_environment(hadoop_container):
cleanup_test_artifacts(hadoop_container, TEST_KMS_KEYS, TEST_EZ_PATHS)
yield
cleanup_test_artifacts(hadoop_container, TEST_KMS_KEYS, TEST_EZ_PATHS)
-
diff --git a/functional-tests/hdfs/test_hdfs_group_policy_03.py
b/functional-tests/hdfs/test_hdfs_group_policy_03.py
new file mode 100644
index 000000000..ff518e50b
--- /dev/null
+++ b/functional-tests/hdfs/test_hdfs_group_policy_03.py
@@ -0,0 +1,271 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Group write on /demo_<suffix>/data_<suffix> and read+execute on / and
/demo_<suffix>
+for group finance.
+"""
+
+import pytest
+
+from ranger_test_utils.hdfs_utils import (
+ GROUP_AUDIT,
+ GROUP_FINANCE,
+ HDFS_AUTHZ_DENIED_SUBSTR,
+ HDFS_SERVICE_NAME,
+ HDFS_SERVICE_USER,
+ USER_AUDIT,
+ USER_FINANCE,
+ build_hdfs_policy_payload,
+ cleanup_hdfs_paths,
+ create_hdfs_policy,
+ ensure_kerberos_user_on_hadoop,
+ ensure_unix_user_with_primary_group,
+ HADOOP_CONTAINER,
+ get_docker_client,
+ preflight_hdfs_plugin_stack,
+ run_hdfs_dfs,
+ wait_for_hdfs_policy_propagation,
+)
+from ranger_test_utils.hive_utils import delete_policies_by_id,
set_all_policies_enabled
+from ranger_test_utils.utils import (
+ DOCKER_PLUGIN_TEST_USER_PASSWORD,
+ configure_test_logging,
+ create_ranger_admin_session,
+ ensure_argus_policy_principals,
+ ensure_user_assigned_to_groups,
+ ensure_user_exists,
+ get_test_logger,
+ log_testcase_begin_for_pytest,
+ unique_suffix,
+)
+
+pytestmark = pytest.mark.hdfs_policy
+
+logger = get_test_logger(__name__)
+
+_admin_session = None
+_policy_ids = []
+_hdfs_paths_to_cleanup = []
+_run_suffix = None
+_demo_base = None
+_demo_data = None
+
+# Container-local files for copyFromLocal (case 3B/3D).
+_LOCAL_STAGING_ROOT = "/tmp/ranger-ft/demo/data"
+
+
+def _demo_paths_for_suffix(run_suffix):
+ """QE /demo/data layout with unique paths: /demo_<suffix>/data_<suffix>."""
+ demo_base = "/demo_" + run_suffix
+ return demo_base, demo_base + "/data_" + run_suffix
+
+
+def _prepare_demo_data_hdfs_path(hdfs_data_path):
+ """Recreate path with mode 000 — Ranger grants access, not POSIX."""
+ run_hdfs_dfs(HDFS_SERVICE_USER, "-rm -r -f " + hdfs_data_path,
service_user=True)
+ run_hdfs_dfs(
+ HDFS_SERVICE_USER,
+ "-mkdir -p " + hdfs_data_path,
+ service_user=True,
+ )
+ run_hdfs_dfs(
+ HDFS_SERVICE_USER,
+ "-chmod -R 000 " + hdfs_data_path,
+ service_user=True,
+ )
+
+
+def _prepare_local_copy_sources(container):
+ container.exec_run("rm -rf /tmp/ranger-ft", user="root")
+ container.exec_run("mkdir -p " + _LOCAL_STAGING_ROOT, user="root")
+ for filename in ("filefinance.txt", "fileaudit.txt"):
+ path = _LOCAL_STAGING_ROOT + "/" + filename
+ container.exec_run(
+ 'bash -c "echo test content > ' + path + '"',
+ user="root",
+ )
+ container.exec_run("chmod a+r " + path, user="root")
+
+
+def setup_module():
+ configure_test_logging(module_name=__name__)
+ logger.info("=== setup_module: start (HDFS group policy case 3) ===")
+ preflight_hdfs_plugin_stack()
+
+ global _admin_session, _policy_ids, _run_suffix, _demo_base, _demo_data
+ _run_suffix = unique_suffix(4)
+ _demo_base, _demo_data = _demo_paths_for_suffix(_run_suffix)
+ logger.info("Test HDFS paths: demo_base=%s demo_data=%s", _demo_base,
_demo_data)
+
+ _admin_session = create_ranger_admin_session()
+ hadoop = get_docker_client().containers.get(HADOOP_CONTAINER)
+
+ logger.info("Ensuring Ranger users, groups, and group membership")
+ ensure_argus_policy_principals(_admin_session)
+ ensure_user_exists(
+ _admin_session,
+ USER_FINANCE,
+ password=DOCKER_PLUGIN_TEST_USER_PASSWORD,
+ last_name="hdfs_test",
+ )
+ ensure_user_exists(
+ _admin_session,
+ USER_AUDIT,
+ password=DOCKER_PLUGIN_TEST_USER_PASSWORD,
+ last_name="hdfs_test",
+ )
+ ensure_user_assigned_to_groups(_admin_session, USER_FINANCE,
[GROUP_FINANCE])
+ ensure_user_assigned_to_groups(_admin_session, USER_AUDIT, [GROUP_AUDIT])
+
+ logger.info("Ensuring Unix users and Kerberos principals on ranger-hadoop")
+ ensure_unix_user_with_primary_group(hadoop, USER_FINANCE, GROUP_FINANCE)
+ ensure_unix_user_with_primary_group(hadoop, USER_AUDIT, GROUP_AUDIT)
+ ensure_kerberos_user_on_hadoop(USER_FINANCE)
+ ensure_kerberos_user_on_hadoop(USER_AUDIT)
+
+ _prepare_local_copy_sources(hadoop)
+
+ logger.info("Disabling existing policies on %s", HDFS_SERVICE_NAME)
+ set_all_policies_enabled(_admin_session, False, HDFS_SERVICE_NAME)
+
+ policy_data_write = build_hdfs_policy_payload(
+ policy_name="gp3-data-write-" + _run_suffix,
+ path_values=[_demo_data],
+ perm_map_list=[
+ {
+ "groupList": [GROUP_FINANCE],
+ "userList": [],
+ "permList": ["write"],
+ }
+ ],
+ description=_demo_data + " Write for Group:Finance",
+ is_recursive=False,
+ )
+ policy_root_demo_rx = build_hdfs_policy_payload(
+ policy_name="gp3-root-demo-rx-" + _run_suffix,
+ path_values=["/", _demo_base],
+ perm_map_list=[
+ {
+ "groupList": [GROUP_FINANCE],
+ "userList": [],
+ "permList": ["read", "execute"],
+ }
+ ],
+ description="/, " + _demo_base + " RX for Group:Finance",
+ is_recursive=False,
+ )
+ _policy_ids = [
+ create_hdfs_policy(_admin_session, policy_data_write),
+ create_hdfs_policy(_admin_session, policy_root_demo_rx),
+ ]
+ logger.info("Created HDFS policies: %s", _policy_ids)
+
+ _prepare_demo_data_hdfs_path(_demo_data)
+ wait_for_hdfs_policy_propagation()
+ logger.info("=== setup_module: done ===")
+
+
+def teardown_module():
+ if _admin_session is None:
+ return
+
+ logger.info("=== teardown_module: start ===")
+ if _hdfs_paths_to_cleanup:
+ cleanup_hdfs_paths(_hdfs_paths_to_cleanup)
+ if _demo_base is not None:
+ cleanup_hdfs_paths([_demo_base])
+
+ if _policy_ids:
+ delete_policies_by_id(_admin_session, _policy_ids)
+
+ logger.info("Re-enabling all policies on %s", HDFS_SERVICE_NAME)
+ set_all_policies_enabled(_admin_session, True, HDFS_SERVICE_NAME)
+ wait_for_hdfs_policy_propagation()
+ logger.info("=== teardown_module: done ===")
+
+
+def test_03_01_create_hdfs_directory_case3a(request):
+ log_testcase_begin_for_pytest(logger, request.node.name, suite="HDFS")
+ hdfsdir = _demo_data + "/findir_test_03_01"
+ _hdfs_paths_to_cleanup.append(hdfsdir)
+
+ exit_code, output = run_hdfs_dfs(
+ USER_FINANCE,
+ "-mkdir -p " + hdfsdir,
+ fail_on_error=False,
+ )
+ assert HDFS_AUTHZ_DENIED_SUBSTR not in output, (
+ "Expected mkdir to succeed for finance user; exit_code="
+ + str(exit_code)
+ + " output="
+ + output
+ )
+
+
+def test_03_02_create_hdfs_file_case3b(request):
+ log_testcase_begin_for_pytest(logger, request.node.name, suite="HDFS")
+ localfile = _LOCAL_STAGING_ROOT + "/filefinance.txt"
+ hdfsfile = _demo_data + "/filefinance_test_03_02.txt"
+ _hdfs_paths_to_cleanup.append(hdfsfile)
+
+ exit_code, output = run_hdfs_dfs(
+ USER_FINANCE,
+ "-copyFromLocal " + localfile + " " + hdfsfile,
+ fail_on_error=False,
+ )
+ assert HDFS_AUTHZ_DENIED_SUBSTR not in output, (
+ "Expected copyFromLocal to succeed for finance user; exit_code="
+ + str(exit_code)
+ + " output="
+ + output
+ )
+
+
+def test_03_03_create_hdfs_directory_case3c(request):
+ log_testcase_begin_for_pytest(logger, request.node.name, suite="HDFS")
+ hdfsdir = _demo_data + "/auditdir_test_03_03"
+ _hdfs_paths_to_cleanup.append(hdfsdir)
+
+ exit_code, output = run_hdfs_dfs(
+ USER_AUDIT,
+ "-mkdir -p " + hdfsdir,
+ fail_on_error=False,
+ )
+ assert HDFS_AUTHZ_DENIED_SUBSTR in output, (
+ "Expected mkdir to be denied for audit user; exit_code="
+ + str(exit_code)
+ + " output="
+ + output
+ )
+
+
+def test_03_04_create_hdfs_file_case3d(request):
+ log_testcase_begin_for_pytest(logger, request.node.name, suite="HDFS")
+ localfile = _LOCAL_STAGING_ROOT + "/fileaudit.txt"
+ hdfsfile = _demo_data + "/fileaudit_test_03_04.txt"
+ _hdfs_paths_to_cleanup.append(hdfsfile)
+
+ exit_code, output = run_hdfs_dfs(
+ USER_AUDIT,
+ "-copyFromLocal " + localfile + " " + hdfsfile,
+ fail_on_error=False,
+ )
+ assert HDFS_AUTHZ_DENIED_SUBSTR in output, (
+ "Expected copyFromLocal to be denied for audit user; exit_code="
+ + str(exit_code)
+ + " output="
+ + output
+ )
diff --git a/functional-tests/hdfs/utils.py b/functional-tests/hdfs/utils.py
index 3728fda9e..bffe31dce 100644
--- a/functional-tests/hdfs/utils.py
+++ b/functional-tests/hdfs/utils.py
@@ -18,7 +18,13 @@
import docker
import requests
from hdfs.test_config import (KMS_CONTAINER, HADOOP_NAMENODE_LOG_PATH,
KMS_LOG_PATH)
-from kms.utils import krb_requests, ensure_ticket, BASE_URL, PARAMS, container
as kms_container
+from kms.utils import (
+ BASE_URL,
+ PARAMS,
+ ensure_ticket,
+ get_kms_container,
+ krb_requests,
+)
from hdfs.test_config import HDFS_USER
import subprocess
import tempfile
@@ -318,7 +324,7 @@ def delete_kms_key(key_name: str) -> None:
# Delete KMS key if present. Ignores missing-key responses.
# Only refresh the keyadmin ticket when no valid ticket exists — avoids a
full
# kdestroy+kinit on every key deletion (e.g. 6 calls in
cleanup_test_artifacts).
- ec, _ = kms_container.exec_run("klist -s 2>/dev/null", user="root")
+ ec, _ = get_kms_container().exec_run("klist -s 2>/dev/null", user="root")
if ec != 0:
ensure_ticket()
resp = krb_requests.delete(f"{BASE_URL}/key/{key_name}", params=PARAMS)
diff --git a/functional-tests/kms/utils.py b/functional-tests/kms/utils.py
index 27560e597..9705a50c3 100644
--- a/functional-tests/kms/utils.py
+++ b/functional-tests/kms/utils.py
@@ -37,8 +37,31 @@
TESTUSER_PARAMS = {}
BASE_URL = "http://ranger-kms.rangernw:9292/kms/v1";
PARAMS = {"user.name": "keyadmin"}
-client = docker.from_env()
-container = client.containers.get(KMS_CONTAINER_NAME)
+
+_docker_client = None
+
+
+def get_docker_client():
+ global _docker_client
+ if _docker_client is None:
+ _docker_client = docker.from_env()
+ return _docker_client
+
+
+def get_kms_container():
+ """Return ranger-kms container (lazy — no Docker call at import time)."""
+ return get_docker_client().containers.get(KMS_CONTAINER_NAME)
+
+
+class _LazyKmsContainer:
+ """Backward-compatible alias; resolves ranger-kms on first use."""
+
+ def __getattr__(self, name):
+ return getattr(get_kms_container(), name)
+
+
+container = _LazyKmsContainer()
+
KEYADMIN_PRINCIPAL = "[email protected]"
KEYADMIN_KEYTAB = "/etc/keytabs/keyadmin.keytab"
KDC_CONTAINER = "ranger-kdc"
@@ -111,7 +134,9 @@ def _sh_quote(s):
# Output format: <status_code>\n---BODY---\n<body text>
shell_cmd = f'{curl_cmd}; echo "\\n---BODY---"; cat /tmp/curl_body.txt'
- exit_code, output = container.exec_run(["sh", "-c", shell_cmd],
user="root")
+ exit_code, output = get_kms_container().exec_run(
+ ["sh", "-c", shell_cmd], user="root"
+ )
raw = output.decode() if output else ""
separator = "\n---BODY---\n"
@@ -157,8 +182,8 @@ def __repr__(self):
def _ensure_kdc_principal_keytab(principal: str, dest_container_name: str,
dest_keytab_path: str, kdc_tmp_path: str, ) -> None:
# create principal in KDC, export keytab, copy to target container
- kdc = client.containers.get(KDC_CONTAINER)
- dest = client.containers.get(dest_container_name)
+ kdc = get_docker_client().containers.get(KDC_CONTAINER)
+ dest = get_docker_client().containers.get(dest_container_name)
exit_code, output = kdc.exec_run(f'kadmin.local -q "getprinc
{principal}"', user="root")
if exit_code != 0 or b"does not exist" in output:
@@ -184,13 +209,14 @@ def ensure_keyadmin_keytab():
def _kinit_with_retry(principal: str, keytab: str, ensure_keytab_fn,
max_attempts: int = 5, wait: int = 10) -> None:
"""Ensure the keytab exists then kinit, retrying to absorb KDC warm-up
delays."""
- exit_code, _ = container.exec_run(f"test -f {keytab}", user="root")
+ kms = get_kms_container()
+ exit_code, _ = kms.exec_run(f"test -f {keytab}", user="root")
if exit_code != 0:
ensure_keytab_fn()
- container.exec_run("kdestroy -A 2>/dev/null || true", user="root")
+ kms.exec_run("kdestroy -A 2>/dev/null || true", user="root")
for _ in range(max_attempts):
- exit_code, output = container.exec_run(
+ exit_code, output = kms.exec_run(
f"kinit -kt {keytab} {principal}", user="root"
)
if exit_code == 0:
@@ -226,7 +252,7 @@ def modify_blacklist_property(operation, users,
action="add"):
ensure_ticket()
- result = container.exec_run(f"cat {dbks_site_path}", user="root")
+ result = get_kms_container().exec_run(f"cat {dbks_site_path}", user="root")
if result.exit_code != 0:
raise RuntimeError(f"Cannot read dbks-site.xml:
{result.output.decode()}")
@@ -266,7 +292,10 @@ def modify_blacklist_property(operation, users,
action="add"):
tar.addfile(info, io.BytesIO(data))
tarstream.seek(0)
-
container.put_archive(path="/opt/ranger/ranger-3.0.0-SNAPSHOT-kms/ews/webapp/WEB-INF/classes/conf/",
data=tarstream)
+ get_kms_container().put_archive(
+
path="/opt/ranger/ranger-3.0.0-SNAPSHOT-kms/ews/webapp/WEB-INF/classes/conf/",
+ data=tarstream,
+ )
print(f"Successfully {'added' if action == 'add' else 'removed'} {users}
in {prop_name}")
diff --git a/functional-tests/pytest.ini b/functional-tests/pytest.ini
index 2777e38c3..132c00b2c 100644
--- a/functional-tests/pytest.ini
+++ b/functional-tests/pytest.ini
@@ -29,6 +29,7 @@ markers =
api: public API v2 test cases
hive: Hive plugin enforcement (Docker HS2)
hbase: HBase plugin enforcement (Docker HBase shell)
+ hdfs_policy: HDFS Ranger plugin path policy enforcement (Docker dfs)
pythonpath = .
# No console logging during pytest; logger.info -> logs/pytest.log (tail -f
logs/pytest.log).
log_cli = false
diff --git a/functional-tests/ranger_test_utils/hdfs_utils.py
b/functional-tests/ranger_test_utils/hdfs_utils.py
new file mode 100644
index 000000000..b9bef7e5e
--- /dev/null
+++ b/functional-tests/ranger_test_utils/hdfs_utils.py
@@ -0,0 +1,301 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""HDFS Docker helpers for Ranger functional-tests (dfs + plugin
enforcement)."""
+
+import socket
+import subprocess
+import tempfile
+import time
+
+import pytest
+
+from ranger_test_utils.utils import (
+ POLICY_URL,
+ assert_http_ok,
+ get_test_logger,
+ perm_map_list_to_policy_items,
+)
+
+logger = get_test_logger(__name__)
+
+HDFS_SERVICE_NAME = "dev_hdfs"
+HADOOP_CONTAINER = "ranger-hadoop"
+KDC_CONTAINER = "ranger-kdc"
+ADMIN_CONTAINER = "ranger"
+
+KERBEROS_REALM = "EXAMPLE.COM"
+HDFS_SERVICE_USER = "hdfs"
+HDFS_SERVICE_PRINCIPAL = "hdfs/ranger-hadoop.rangernw@" + KERBEROS_REALM
+HDFS_SERVICE_KEYTAB = "/etc/keytabs/hdfs.keytab"
+
+USER_FINANCE = "hrt_21"
+USER_AUDIT = "hrt_22"
+GROUP_FINANCE = "finance"
+GROUP_AUDIT = "audit"
+
+HDFS_AUTHZ_DENIED_SUBSTR = "Permission denied"
+
+POLICY_PROPAGATION_SLEEP_SEC = 15
+
+_docker_client = None
+
+
+def get_docker_client():
+ global _docker_client
+ if _docker_client is None:
+ import docker
+
+ _docker_client = docker.from_env()
+ return _docker_client
+
+
+def _container_running(name):
+ client = get_docker_client()
+ try:
+ container = client.containers.get(name)
+ except Exception:
+ return False
+ container.reload()
+ return container.status == "running"
+
+
+def _namenode_rpc_port_open(host="127.0.0.1", port=9000, timeout=2.0):
+ """NameNode RPC — published by docker-compose.ranger-hadoop.yml
(9000:9000)."""
+ try:
+ with socket.create_connection((host, port), timeout=timeout):
+ return True
+ except OSError:
+ return False
+
+
+def preflight_hdfs_plugin_stack():
+ """Skip tests when Ranger Admin, KDC, or ranger-hadoop are unavailable."""
+ required = [ADMIN_CONTAINER, KDC_CONTAINER, HADOOP_CONTAINER]
+ missing = [name for name in required if not _container_running(name)]
+ if missing:
+ pytest.skip(
+ "HDFS plugin functional-tests need running containers: "
+ + ", ".join(missing)
+ + ". Run ./run-tests.sh postgres hdfs with
SELECTED_CONTAINERS=hadoop."
+ )
+
+ if not _namenode_rpc_port_open():
+ pytest.skip(
+ "HDFS NameNode RPC is not accepting connections on localhost:9000 "
+ "(is ranger-hadoop up and healthy? export
SELECTED_CONTAINERS=hadoop)."
+ )
+
+
+def _user_principal(username):
+ return username + "@" + KERBEROS_REALM
+
+
+def _user_keytab_in_hadoop_container(username):
+ return "/etc/keytabs/" + username + ".keytab"
+
+
+def ensure_unix_user_with_primary_group(container, username, primary_group):
+ """Create Linux user/group on ranger-hadoop for UGI group resolution."""
+ container.exec_run("groupadd -f " + primary_group, user="root")
+ exit_code, _ = container.exec_run("id -u " + username, user="root")
+ if exit_code == 0:
+ container.exec_run(
+ "usermod -g " + primary_group + " " + username,
+ user="root",
+ )
+ container.exec_run("usermod -aG hadoop " + username, user="root")
+ return
+
+ container.exec_run(
+ "useradd -m -s /bin/bash -g "
+ + primary_group
+ + " -G hadoop "
+ + username,
+ user="root",
+ )
+
+
+def ensure_kerberos_user_on_hadoop(username):
+ """Create user@REALM principal and install keytab on ranger-hadoop."""
+ kdc = get_docker_client().containers.get(KDC_CONTAINER)
+ hadoop = get_docker_client().containers.get(HADOOP_CONTAINER)
+ principal = _user_principal(username)
+ keytab_name = username + ".keytab"
+ kdc_tmp = "/tmp/" + keytab_name
+ dest_path = _user_keytab_in_hadoop_container(username)
+
+ exit_code, output = kdc.exec_run(
+ 'kadmin.local -q "getprinc ' + principal + '"',
+ user="root",
+ )
+ combined = (output or b"").decode()
+ if exit_code != 0 or "does not exist" in combined:
+ ec, out = kdc.exec_run(
+ 'kadmin.local -q "addprinc -randkey ' + principal + '"',
+ user="root",
+ )
+ if ec != 0:
+ raise RuntimeError(
+ "addprinc failed for " + principal + ": " + (out or
b"").decode()
+ )
+
+ kdc.exec_run("rm -f " + kdc_tmp, user="root")
+ ec, out = kdc.exec_run(
+ 'kadmin.local -q "xst -k ' + kdc_tmp + " " + principal + '"',
+ user="root",
+ )
+ if ec != 0:
+ raise RuntimeError(
+ "keytab export failed for " + principal + ": " + (out or
b"").decode()
+ )
+
+ local = tempfile.NamedTemporaryFile(delete=False, suffix=".keytab")
+ local.close()
+ try:
+ subprocess.check_call(
+ ["docker", "cp", KDC_CONTAINER + ":" + kdc_tmp, local.name]
+ )
+ subprocess.check_call(
+ ["docker", "cp", local.name, HADOOP_CONTAINER + ":" + dest_path]
+ )
+ finally:
+ import os
+
+ os.remove(local.name)
+
+ hadoop.exec_run("chmod 444 " + dest_path, user="root")
+ hadoop.exec_run("chown " + username + ":" + username + " " + dest_path,
user="root")
+ logger.info("Kerberos keytab installed for %s at %s", principal, dest_path)
+
+
+def _log_hdfs_output(exit_code, stdout, stderr, cmd):
+ logger.info("HDFS command exit_code=%s cmd=%s", exit_code, cmd)
+ for label, text in (("stdout", stdout), ("stderr", stderr)):
+ body = text if text else "(empty)"
+ logger.info("HDFS %s:", label)
+ for line in body.splitlines():
+ logger.info(" %s", line)
+
+
+def run_hdfs_dfs(
+ username,
+ dfs_subcommand,
+ service_user=False,
+ fail_on_error=True,
+):
+ """
+ Run `hdfs dfs <dfs_subcommand>` inside ranger-hadoop.
+
+ Returns (exit_code, combined_stdout_stderr).
+ """
+ hadoop = get_docker_client().containers.get(HADOOP_CONTAINER)
+ if service_user:
+ principal = HDFS_SERVICE_PRINCIPAL
+ keytab = HDFS_SERVICE_KEYTAB
+ exec_user = HDFS_SERVICE_USER
+ ranger_user_label = HDFS_SERVICE_USER
+ else:
+ principal = _user_principal(username)
+ keytab = _user_keytab_in_hadoop_container(username)
+ exec_user = username
+ ranger_user_label = username
+
+ cmd = "hdfs dfs " + dfs_subcommand
+ shell = (
+ "kdestroy -A 2>/dev/null || true; "
+ "kinit -kt "
+ + keytab
+ + " "
+ + principal
+ + "; "
+ + 'kinit_rc=$?; if [ "$kinit_rc" -ne 0 ]; then exit "$kinit_rc"; fi; '
+ + 'HADOOP_OPTS="-Dhadoop.security.authentication=kerberos" '
+ + cmd
+ )
+ logger.info(
+ "HDFS context: container=%s exec_user=%s ranger_user=%s",
+ HADOOP_CONTAINER,
+ exec_user,
+ ranger_user_label,
+ )
+ exit_code, output = hadoop.exec_run(["bash", "-c", shell], user=exec_user,
demux=True)
+ stdout = (output[0] or b"").decode()
+ stderr = (output[1] or b"").decode()
+ combined = stdout + stderr
+ _log_hdfs_output(exit_code, stdout, stderr, cmd)
+
+ if fail_on_error and exit_code != 0:
+ raise AssertionError(
+ "HDFS command failed (exit "
+ + str(exit_code)
+ + "): "
+ + cmd
+ + "\n"
+ + combined
+ )
+ return exit_code, combined
+
+
+def cleanup_hdfs_paths(paths):
+ """Remove HDFS paths if present (as hdfs service user)."""
+ for path in paths:
+ run_hdfs_dfs(
+ HDFS_SERVICE_USER,
+ "-rm -r -f " + path,
+ service_user=True,
+ fail_on_error=False,
+ )
+
+
+def build_hdfs_policy_payload(
+ policy_name,
+ path_values,
+ perm_map_list,
+ description="",
+ is_recursive=False,
+ is_audit_enabled=True,
+ service_name=HDFS_SERVICE_NAME,
+):
+ """JSON body for POST/PUT on /public/v2/api/policy/ (HDFS path
resource)."""
+ return {
+ "service": service_name,
+ "name": policy_name,
+ "description": description,
+ "isEnabled": True,
+ "isAuditEnabled": is_audit_enabled,
+ "resources": {
+ "path": {
+ "values": path_values,
+ "isExcludes": False,
+ "isRecursive": is_recursive,
+ }
+ },
+ "policyItems": perm_map_list_to_policy_items(perm_map_list),
+ }
+
+
+def create_hdfs_policy(session, payload):
+ response = session.post(POLICY_URL, json=payload)
+ assert_http_ok(response, (200, 201), "Create HDFS policy", POLICY_URL)
+ body = response.json()
+ policy_id = body.get("id")
+ assert policy_id is not None, "Create policy response missing id: " +
response.text
+ return policy_id
+
+
+def wait_for_hdfs_policy_propagation(seconds=POLICY_PROPAGATION_SLEEP_SEC):
+ logger.info("Waiting %ss for HDFS plugin policy propagation", seconds)
+ time.sleep(seconds)
diff --git a/functional-tests/ranger_test_utils/utils.py
b/functional-tests/ranger_test_utils/utils.py
index eb0931dbb..534c8bfac 100644
--- a/functional-tests/ranger_test_utils/utils.py
+++ b/functional-tests/ranger_test_utils/utils.py
@@ -246,6 +246,27 @@ def ensure_argus_policy_principals(session):
ensure_group_exists(session, group_name)
+def ensure_user_assigned_to_groups(session, username, group_names):
+ """Assign Ranger xuser group membership (idempotent for
functional-tests)."""
+ if not group_names:
+ return
+ url = XUSERS_BASE + "/users/userinfo"
+ payload = {
+ "xuserInfo": {"name": username},
+ "xgroupInfo": [{"name": group_name} for group_name in group_names],
+ }
+ response = session.post(url, json=payload)
+ assert_http_ok(
+ response,
+ 200,
+ "Assign groups "
+ + ",".join(group_names)
+ + " to user "
+ + username,
+ url,
+ )
+
+
# --- Hive public API v2 JSON builders ---
diff --git a/functional-tests/readme.md b/functional-tests/readme.md
index f9f1e3d22..c46bee8ec 100644
--- a/functional-tests/readme.md
+++ b/functional-tests/readme.md
@@ -24,7 +24,7 @@ A Docker-based test execution framework that validates Apache
Ranger service int
| Suite | Description |
|---|---|
-| **hdfs** | Test cases for HDFS encryption lifecycle using KMS |
+| **hdfs** | HDFS encryption lifecycle (KMS) and Ranger HDFS plugin path
policies (`hdfs_policy` marker) |
| **kms** | Test cases for KMS REST API functionality |
| **xuserrest** | Test cases for Ranger Admin User/Group REST APIs |
| **rolerest** | Test cases for Ranger Admin Role REST APIs |
@@ -52,6 +52,7 @@ functional-tests/
├── hbase/ # HBase plugin enforcement tests
├── ranger_test_utils/hive_utils.py
├── ranger_test_utils/hbase_utils.py
+├── ranger_test_utils/hdfs_utils.py
├── ranger_test_utils/access_audit_utils.py # For future access-audit asserts
│
├── ranger_test_utils/utils.py # Shared logging, HTTP asserts, Public API v2
helpers
@@ -127,6 +128,11 @@ The containers prompt always appears unless
`SELECTED_CONTAINERS` is exported.
**Examples:**
```text
+# Run hdfs suite (encryption + hdfs_policy tests) — requires ranger-hadoop
+export AUDIT_INDEX_STORE=none
+export SELECTED_CONTAINERS=hadoop
+./run-tests.sh postgres hdfs
+
# Run hdfs and kms tests — postgres db, no audit pipeline, hadoop container
only
export AUDIT_INDEX_STORE=none
export SELECTED_CONTAINERS=hadoop
@@ -153,7 +159,7 @@ All variables have sensible defaults and can be exported
before running or passe
| `CLEAN_CONTAINERS` | `0` | Set to `1` to wipe all containers and force a
full rebuild from scratch |
| `RUN_TESTS` | `1` | Set to `0` to bring up infrastructure only without
running any tests |
| `AUDIT_INDEX_STORE` | `opensearch` | Audit backend: `opensearch`, `solr`, or
`none` to skip the audit pipeline entirely |
-| `SELECTED_CONTAINERS` | _(prompted)_ | Space-separated optional containers
to bring up, e.g. `"hadoop hive"`. Set to `"all"` to bring up every optional
container. |
+| `SELECTED_CONTAINERS` | _(prompted)_ | Space-separated optional containers
to bring up, e.g. `"hadoop hive"`. Set to `"all"` to bring up every optional
container. **Required for the hdfs suite:** export `hadoop` or `all` (starts
`ranger-hadoop`). |
### `CLEAN_CONTAINERS`
@@ -228,6 +234,21 @@ Brought up based on what you enter at the containers
prompt (or `SELECTED_CONTAI
- Tests currently assert **HBase shell authorization output only**.
Access-audit helpers live in `ranger_test_utils/access_audit_utils.py` for a
follow-up once auditing works.
- Example: `./run-tests.sh postgres hbase`
+### HDFS suite notes (encryption + `hdfs_policy`)
+
+- Service repo for plugin tests is `dev_hdfs` (seeded by ranger-docker).
+- **`./run-tests.sh postgres hdfs` does not start `ranger-hadoop` by itself.**
Export `SELECTED_CONTAINERS=hadoop` or `SELECTED_CONTAINERS=all` before running
(or choose `hadoop` / `all` at the containers prompt).
+- `./run-tests.sh` runs the whole `hdfs/` folder (no marker filter). Use `-m
hdfs_policy` only for selective local runs.
+- Example:
+ ```text
+ export AUDIT_INDEX_STORE=none
+ export SELECTED_CONTAINERS=hadoop
+ ./run-tests.sh postgres hdfs
+ ```
+- Plugin tests only (skips encryption/KMS session setup in `hdfs/conftest.py`):
+ `pytest hdfs/test_hdfs_group_policy_03.py -m hdfs_policy`
+ Still requires `ranger-hadoop` (and Ranger Admin) running.
+
## Test Reports
After execution, an HTML report is automatically generated for each suite in
the `functional-tests/` directory:
