This is an automated email from the ASF dual-hosted git repository. ramackri pushed a commit to branch RANGER-5712 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit e27068de3a6446dd8d5db5872a20a0e12b4d1243 Author: ramk <[email protected]> AuthorDate: Thu Jul 30 15:19:45 2026 +0530 RANGER-5712: Ozone 2.1.1 docker pin and plugin Jersey classpath fix Pin OZONE_VERSION=2.1.1 in docker .env to match pom.xml. Exclude bundled Jersey/HK2 from the ozone plugin tarball to avoid PolicyRefresher ClassCastException on Ozone 2.1.x. Extend verify-ozone-pr1116-e2e.sh to recreate Solr and Solr dispatcher in Kerberos mode for audit pipeline tests. --- dev-support/ranger-docker/.env | 4 +- .../scripts/ozone/verify-ozone-pr1116-e2e.sh | 56 +++++++++++++++++----- distro/src/main/assembly/plugin-ozone.xml | 25 +++++----- 3 files changed, 59 insertions(+), 26 deletions(-) diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index c6439db0e..9ec9bdf7c 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -75,8 +75,8 @@ ORACLE_VERSION=23.6 SQLSERVER_VERSION=2022-latest # Ozone Configuration -# Align runtime with Ranger plugin compile target (pom.xml ozone.version=2.1.0) -OZONE_VERSION=2.1.0 +# Align runtime with Ranger plugin compile target (pom.xml ozone.version=2.1.1) +OZONE_VERSION=2.1.1 OZONE_PLUGIN_VERSION=3.0.0-SNAPSHOT OZONE_RUNNER_IMAGE=apache/ozone-runner # Ozone 2.0+ requires jdk21 runner (jdk11/jdk17 tags are for Ozone 1.x). Multi-arch arm64+amd64. diff --git a/dev-support/ranger-docker/scripts/ozone/verify-ozone-pr1116-e2e.sh b/dev-support/ranger-docker/scripts/ozone/verify-ozone-pr1116-e2e.sh index 177387313..fab827283 100755 --- a/dev-support/ranger-docker/scripts/ozone/verify-ozone-pr1116-e2e.sh +++ b/dev-support/ranger-docker/scripts/ozone/verify-ozone-pr1116-e2e.sh @@ -57,8 +57,9 @@ AUDIT_COMPOSE=( TEST_POLICY_NAME="${TEST_POLICY_NAME:-pr1116-test-policy}" TEST_ROLE_NAME="${TEST_ROLE_NAME:-pr1116_test_role}" -TEST_VOLUME="${TEST_VOLUME:-testvol-pr1116}" -AUDIT_BUCKET="${AUDIT_BUCKET:-audit-bucket}" +TEST_VOLUME="${TEST_VOLUME:-s3Vol}" +OZONE_CLI_VOLUME="${OZONE_CLI_VOLUME:-s3v}" +AUDIT_BUCKET="${AUDIT_BUCKET:-audit-bucket-pr1116}" AUDIT_MARKER="${AUDIT_MARKER:-pr1116-audit-marker}" PR_GIT_REF="${PR_GIT_REF:-RANGER-5712}" @@ -93,7 +94,7 @@ Usage: $(basename "$0") <apply|verify|audit-e2e|revert-ingestor|revert|run> apply Apply PR #1116 ozone docker files, recreate om/scm/datanode verify Kerberos + PolicyRefresher policy/role download - audit-e2e Ingestor static+Kerberos, ozone audit -> kafka -> solr + audit-e2e Static ingestor + Kerberos Solr/dispatcher, ozone audit pipeline revert-ingestor Restore ingestor dynamic+SPIFFE (if audit phase interrupted) revert Restore backed-up ozone docker files run apply + verify + audit-e2e @@ -232,18 +233,26 @@ ingestor_env_snapshot() { > "${INGESTOR_ENV_BACKUP}" 2>/dev/null || true } -restart_ingestor_static_kerberos() { +restart_audit_pipeline_kerberos_mode() { local stack="${RANGER_DOCKER_STACK}" cd "${stack}" # shellcheck source=/dev/null source .env ingestor_env_snapshot - echo "== Switching audit-ingestor to static partition + Kerberos inbound ==" + echo "== Audit pipeline Kerberos mode (static ingestor + Kerberos Solr/dispatcher) ==" + echo " ingestor: partition.plan.admin.enabled=false, inbound Kerberos (not SPIFFE)" + echo " solr + solr-dispatcher: KERBEROS_ENABLED=true" RANGER_AUDIT_PARTITION_PLAN_ADMIN_ENABLED=false \ RANGER_AUDIT_INGESTOR_SPIFFE_ENABLED=false \ KERBEROS_ENABLED=true \ - docker compose "${AUDIT_COMPOSE[@]}" up -d --no-deps --force-recreate ranger-audit-ingestor - sleep 25 + docker compose "${AUDIT_COMPOSE[@]}" up -d --no-deps --force-recreate \ + ranger-solr ranger-audit-ingestor ranger-audit-dispatcher-solr + sleep 45 + verify_audit_kerberos_stack +} + +verify_audit_kerberos_stack() { + echo "== Verify audit pipeline Kerberos/static config ==" docker exec ranger-audit-ingestor python3 -c " import xml.etree.ElementTree as ET r=ET.parse('/opt/ranger/audit-ingestor/conf/ranger-audit-ingestor-site.xml').getroot() @@ -252,6 +261,27 @@ for n in ['ranger.audit.ingestor.partition.plan.admin.enabled','ranger.audit.ing if p.find('name') is not None and p.find('name').text==n: print(n,'=',p.find('value').text) " + local solr_auth + if docker exec ranger-solr ps aux 2>/dev/null | grep -q 'solr.kerberos.principal'; then + echo "PASS: ranger-solr running with Kerberos (solr.kerberos.* JVM opts)" + elif docker exec ranger-solr grep -q KerberosPlugin /var/solr/data/security.json 2>/dev/null; then + echo "PASS: ranger-solr security.json uses KerberosPlugin" + else + echo "FAIL: ranger-solr Kerberos not detected (check KERBEROS_ENABLED=true)" + return 1 + fi + docker exec ranger-audit-dispatcher-solr python3 -c " +import xml.etree.ElementTree as ET +r=ET.parse('/opt/ranger/audit-dispatcher/conf/ranger-audit-dispatcher-solr-site.xml').getroot() +for n in ['ranger.audit.dispatcher.hadoop.security.authentication','xasecure.audit.jaas.Client.option.keyTab','xasecure.audit.jaas.Client.option.principal']: + for p in r.findall('property'): + if p.find('name') is not None and p.find('name').text==n: + print(n,'=',p.find('value').text) +" || echo "WARN: could not read solr dispatcher site XML" +} + +restart_ingestor_static_kerberos() { + restart_audit_pipeline_kerberos_mode } restart_ingestor_dynamic_spiffe() { @@ -306,7 +336,12 @@ EOF policy_id=$(docker exec ranger curl -s -u admin:rangerR0cks! \ -H 'Content-Type: application/json' -X POST \ 'http://localhost:6080/service/public/v2/api/policy' \ - -d "${policy_json}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))") + -d "${policy_json}" | python3 -c "import sys,json; r=json.load(sys.stdin); print(r.get('id') or r.get('msg',''))") + if [ -z "${policy_id}" ] || ! [[ "${policy_id}" =~ ^[0-9]+$ ]]; then + policy_id=$(docker exec ranger curl -s -u admin:rangerR0cks! \ + "http://localhost:6080/service/public/v2/api/policy?serviceName=dev_ozone&policyName=${TEST_POLICY_NAME}"; \ + | python3 -c "import sys,json; ps=json.load(sys.stdin); print(ps[0]['id'] if ps else '')") + fi role_json="{\"name\":\"${TEST_ROLE_NAME}\",\"isEnabled\":true,\"users\":[],\"groups\":[],\"roles\":[]}" role_id=$(docker exec ranger curl -s -u admin:rangerR0cks! \ -H 'Content-Type: application/json' -X POST \ @@ -381,14 +416,13 @@ wait_for_om_kerberos() { trigger_ozone_audit() { echo "== Trigger policy-enforced Ozone audit (hdfs -> OM -> ingestor) ==" - local audit_vol="${TEST_VOLUME}" audit_bucket="${AUDIT_BUCKET:-audit-bucket}" + local audit_vol="${OZONE_CLI_VOLUME}" audit_bucket="${AUDIT_BUCKET}" docker exec ozone-om bash -c " set -e export KRB5_CONFIG=/etc/krb5.conf kinit -kt /etc/keytabs/om.keytab om/[email protected] - /opt/hadoop/bin/ozone sh volume create ${audit_vol} 2>/dev/null || true /opt/hadoop/bin/ozone sh bucket create ${audit_vol}/${audit_bucket} 2>/dev/null || true - " || echo "WARN: om setup for audit volume failed" + " || echo "WARN: om setup for audit bucket failed" docker cp ranger-hadoop:/etc/keytabs/hdfs.keytab /tmp/ozone-audit-hdfs.keytab docker cp /tmp/ozone-audit-hdfs.keytab ozone-om:/tmp/hdfs.keytab diff --git a/distro/src/main/assembly/plugin-ozone.xml b/distro/src/main/assembly/plugin-ozone.xml index 7d14d2ab5..c8fb54a84 100644 --- a/distro/src/main/assembly/plugin-ozone.xml +++ b/distro/src/main/assembly/plugin-ozone.xml @@ -110,20 +110,7 @@ <include>io.airlift:aircompressor:jar:${aircompressor.version}</include> <include>javax.annotation:javax.annotation-api:jar:${javax.annotation-api.version}</include> <include>javax.inject:javax.inject</include> - <include>javax.ws.rs:javax.ws.rs-api</include> <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include> - <include>org.glassfish.hk2:hk2-api</include> - <include>org.glassfish.hk2:hk2-locator</include> - <include>org.glassfish.hk2:hk2-utils</include> - <include>org.glassfish.hk2.external:aopalliance-repackaged</include> - <include>org.glassfish.hk2.external:jakarta.inject</include> - <include>org.glassfish.hk2.external:javax.inject</include> - <include>org.glassfish:osgi-resource-locator</include> - <include>org.glassfish.jersey.core:jersey-client</include> - <include>org.glassfish.jersey.core:jersey-common</include> - <include>org.glassfish.jersey.ext:jersey-entity-filtering:jar:${jersey-client.version}</include> - <include>org.glassfish.jersey.inject:jersey-hk2</include> - <include>org.glassfish.jersey.media:jersey-media-json-jackson:jar:${jersey-client.version}</include> <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${fasterxml.jackson.version}</include> <!-- GraalJS 25.x runtime (js is now a POM, not a JAR) --> <include>org.graalvm.js:js-language:jar:${graalvm.version}</include> @@ -141,6 +128,18 @@ <include>org.graalvm.shadowed:xz:jar:${graalvm.version}</include> </includes> + <!-- Use Ozone 2.1.x host Jersey/HK2 (share/ozone/lib) via RangerPluginClassLoader fallback; + bundling our own copies causes WadlAutoDiscoverable ClassCastException on OM. --> + <excludes> + <exclude>org.glassfish.jersey.core:*</exclude> + <exclude>org.glassfish.jersey.ext:*</exclude> + <exclude>org.glassfish.jersey.inject:*</exclude> + <exclude>org.glassfish.jersey.media:*</exclude> + <exclude>org.glassfish.hk2:*</exclude> + <exclude>org.glassfish:osgi-resource-locator</exclude> + <exclude>javax.ws.rs:javax.ws.rs-api</exclude> + <exclude>jakarta.ws.rs:jakarta.ws.rs-api</exclude> + </excludes> </binaries> </moduleSet> </moduleSets>
