This is an automated email from the ASF dual-hosted git repository. ramackri pushed a commit to branch ranger-2.9 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 84f8cc857ca4626093e2c6b6271a75ea8da0d7d7 Author: ramk <[email protected]> AuthorDate: Tue Jul 28 22:11:47 2026 +0530 RANGER-5694: RC2 Ozone docker validation fixes for Kerberos and Solr audit Mount Ozone OM keytabs and krb5.conf, enable direct Solr audit delivery with FQDN and JAAS keytab login, add Solr audit JAR bind-mount, default Admin audit store to Solr, and add helper scripts for keytab provisioning and RC2 E2E validation. Co-authored-by: Cursor <[email protected]> --- dev-support/ranger-docker/README.md | 37 ++++ .../ranger-docker/docker-compose.ranger-ozone.yml | 12 +- .../ranger-docker/docker-compose.ranger.yml | 1 + dev-support/ranger-docker/kms-e2e-download-test.sh | 126 +++++++++++++ dev-support/ranger-docker/ozone-kms-e2e-test.sh | 194 +++++++++++++++++++++ .../ranger-docker/rc2-madhan-validation-test.sh | 132 ++++++++++++++ .../enable-unauthenticated-plugin-download.sh | 54 ++++++ .../admin/ranger-admin-install-postgres.properties | 22 +-- .../ranger-docker/scripts/kdc/entrypoint.sh | 3 + .../scripts/ozone/enable-ozone-plugin.sh | 0 .../scripts/ozone/provision-ozone-keytab.sh | 23 +++ ...=> ranger-ozone-plugin-install-solr.properties} | 22 +-- .../ozone/ranger-ozone-plugin-install.properties | 22 +-- .../scripts/ozone/ranger-ozone-setup.sh | 41 ++++- .../ranger-docker/scripts/pdp/ranger-pdp.sh | 0 .../ranger-docker/scripts/rdbms/init_mssql.sh | 0 .../ranger-docker/scripts/rdbms/init_oracle.sh | 0 .../ranger-docker/scripts/rdbms/init_postgres.sh | 0 .../scripts/solr/ensure-solr-plugin-audit-jars.sh | 65 +++++++ .../ranger-docker/scripts/solr/ranger-solr.sh | 2 +- .../solr-plugin-docker-setup.sh} | 19 +- .../scripts/zk/zookeeper-with-kerberos.sh | 0 .../ranger-docker/test-db-audit-multivalue.sh | 133 ++++++++++++++ 23 files changed, 858 insertions(+), 50 deletions(-) diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md index e4956ef64..54b433e5e 100644 --- a/dev-support/ranger-docker/README.md +++ b/dev-support/ranger-docker/README.md @@ -85,6 +85,21 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-usersync.ym # Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!) ~~~ + +##### Solr plugin audit JARs (Kerberos stacks) + +If plugin audit delivery fails with `NoClassDefFoundError: MiscUtil` on `ranger-solr`, the Solr +plugin tarball was built without audit modules. Run once before (re)creating `ranger-solr`: + +~~~ +./scripts/solr/solr-plugin-docker-setup.sh +docker compose -f docker-compose.ranger.yml up -d --force-recreate ranger-solr +~~~ + +`docker-compose.ranger.yml` bind-mounts `dist/ranger-*-solr-plugin/lib/ranger-solr-plugin-impl` +so updated JARs apply without rebuilding the Solr image. Set +`solr.kerberos.cookie.domain=ranger-solr.rangernw` (FQDN) in `scripts/solr/ranger-solr.sh`. + #### Bring up hive container ~~~ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hive.yml up -d @@ -138,6 +153,28 @@ service-def. docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-ozone.yml up -d ~~~ +Ozone OM mounts Kerberos keytabs (`dist/keytabs/om`), `krb5.conf`, and configures Solr audit +delivery via JAAS (`ozone/[email protected]`, same pattern as the HDFS plugin). +Solr audit URLs use the Kerberos FQDN `ranger-solr.rangernw` (short `ranger-solr` breaks SPNEGO +because the KDC only has `HTTP/[email protected]`). JAAS uses +`useTicketCache=false` with keytab login. +Policy download uses Admin `ranger.admin.allow.unauthenticated.download.access=true` +(docker dev only — run `scripts/admin/enable-unauthenticated-plugin-download.sh` on +existing Admin installs). After changing `ranger-ozone-plugin-install.properties`, remove +`downloads/ozone-*/.setupDone*` and recreate `om` so plugin enable re-runs. + +Existing KDC databases: run `scripts/ozone/provision-ozone-keytab.sh` once, then: + +~~~ +./scripts/ozone/ozone-plugin-docker-setup.sh +./scripts/admin/enable-unauthenticated-plugin-download.sh +docker restart ranger +docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-ozone.yml up -d --force-recreate om datanode +~~~ + +Fresh KDC installs create `dist/keytabs/om/ozone.keytab` automatically. Legacy stacks +may still have `dist/keytabs/ranger-ozone/ozone.keytab` (mounted as fallback). + Verify (after login): ~~~ diff --git a/dev-support/ranger-docker/docker-compose.ranger-ozone.yml b/dev-support/ranger-docker/docker-compose.ranger-ozone.yml index 9b7aad5c6..6abcaec12 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-ozone.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-ozone.yml @@ -3,13 +3,14 @@ services: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} container_name: ozone-datanode hostname: datanode.rangernw + user: root volumes: - ./downloads/ozone-${OZONE_VERSION}:/opt/hadoop networks: - ranger ports: - 9864 - command: ["/opt/hadoop/bin/ozone","datanode"] + command: ["bash", "-c", "chmod -R 777 /data 2>/dev/null || true; exec /opt/hadoop/bin/ozone datanode"] env_file: - ./scripts/ozone/docker-config environment: @@ -29,6 +30,9 @@ services: volumes: - ./downloads/ozone-${OZONE_VERSION}:/opt/hadoop - ./dist/ranger-${OZONE_PLUGIN_VERSION}-ozone-plugin:/opt/hadoop/ranger-ozone-plugin + - ./dist/keytabs/om:/etc/keytabs:ro + - ./dist/keytabs/ranger-ozone:/etc/keytabs-legacy:ro + - ./scripts/kdc/krb5.conf:/etc/krb5.conf:ro networks: - ranger ports: @@ -39,6 +43,8 @@ services: condition: service_started ranger-solr: condition: service_started + ranger-kdc: + condition: service_started environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false @@ -47,7 +53,7 @@ services: OZONE_VERSION: ${OZONE_VERSION} env_file: - ./scripts/ozone/docker-config - command: bash -c "/opt/hadoop/ranger-ozone-plugin/ranger-ozone-setup.sh && /opt/hadoop/bin/ozone om" + command: bash -c "/opt/hadoop/ranger-ozone-plugin/ranger-ozone-setup.sh && exec /opt/hadoop/bin/ozone om" scm: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} container_name: ozone-scm @@ -63,7 +69,7 @@ services: - ./scripts/ozone/docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODE:-1}" OZONE_OPTS: command: ["/opt/hadoop/bin/ozone","scm"] diff --git a/dev-support/ranger-docker/docker-compose.ranger.yml b/dev-support/ranger-docker/docker-compose.ranger.yml index 3d92f5e06..42268493d 100644 --- a/dev-support/ranger-docker/docker-compose.ranger.yml +++ b/dev-support/ranger-docker/docker-compose.ranger.yml @@ -108,6 +108,7 @@ services: hostname: ranger-solr.rangernw volumes: - ./dist/keytabs/ranger-solr:/etc/keytabs + - ./dist/ranger-${SOLR_PLUGIN_VERSION}-solr-plugin/lib/ranger-solr-plugin-impl:/opt/ranger/ranger-solr-plugin/lib/ranger-solr-plugin-impl - ./scripts/solr/solr-ranger_audits:/opt/solr/server/solr/configsets/ranger_audits/conf - ./scripts/solr/ranger-solr-plugin-install.properties:/opt/ranger/ranger-solr-plugin/install.properties - ./scripts/solr/core-site.xml:/opt/solr/server/resources/core-site.xml diff --git a/dev-support/ranger-docker/kms-e2e-download-test.sh b/dev-support/ranger-docker/kms-e2e-download-test.sh new file mode 100755 index 000000000..1f5b7e5f1 --- /dev/null +++ b/dev-support/ranger-docker/kms-e2e-download-test.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# KMS ↔ Ranger Admin plugin download E2E test (Kerberos / rangerkms). +set -euo pipefail + +REALM="${REALM:-EXAMPLE.COM}" +ADMIN_HOST="${ADMIN_HOST:-ranger.rangernw}" +ADMIN_PORT="${ADMIN_PORT:-6080}" +SERVICE="${SERVICE:-dev_kms}" +PLUGIN_ID="${PLUGIN_ID:-kms-e2e-test}" +PASS=0 +FAIL=0 + +log() { printf '\n==> %s\n' "$*"; } +ok() { echo "PASS: $*"; PASS=$((PASS + 1)); } +bad() { echo "FAIL: $*" >&2; FAIL=$((FAIL + 1)); } + +kms_curl_code() { + local url="$1" + docker exec ranger-kms bash -lc " + kinit -kt /etc/keytabs/rangerkms.keytab rangerkms/ranger-kms.rangernw@${REALM} >/dev/null 2>&1 + curl -s -o /tmp/kms-e2e.out -w '%{http_code}' --negotiate -u : '${url}' + " +} + +wait_admin() { + for _ in $(seq 1 30); do + docker exec ranger curl -sf -u admin:rangerR0cks! \ + -H 'Accept: application/json' \ + "http://127.0.0.1:${ADMIN_PORT}/service/plugins/services/name/${SERVICE}"; >/dev/null 2>&1 && return 0 + sleep 5 + done + return 1 +} + +test_kms_running() { + log "KMS service health" + if docker exec ranger-kms bash -lc 'curl -sf -k https://localhost:9292/kms/v1/keys/names' >/dev/null 2>&1; then + ok "KMS HTTPS API responds on :9292" + else + # KMS may require auth for keys API; check process instead + if docker exec ranger-kms pgrep -f Dproc_rangerkms >/dev/null 2>&1; then + ok "KMS process (Dproc_rangerkms) is running" + else + bad "KMS not running" + fi + fi +} + +test_download() { + local name="$1" url="$2" expect_field="$3" + log "${name}" + local code body + code=$(kms_curl_code "${url}") + body=$(docker exec ranger-kms cat /tmp/kms-e2e.out 2>/dev/null || true) + if [[ "${code}" == "200" || "${code}" == "304" ]]; then + if [[ -n "${expect_field}" ]] && ! echo "${body}" | grep -q "${expect_field}"; then + bad "${name}: HTTP ${code} but missing '${expect_field}' in body" + else + ok "${name}: HTTP ${code} $(echo "${body}" | head -c 120)" + fi + else + bad "${name}: HTTP ${code}; body=$(echo "${body}" | head -c 300)" + fi +} + +test_admin_no_kms_denial() { + log "Admin log — no KMS service-read denial for rangerkms" + if docker exec ranger grep -q "Logged in user is not allowed to read service, name: ${SERVICE}" \ + /var/log/ranger/admin/catalina.out 2>/dev/null; then + # only fail if recent (last 50 lines) + if docker exec ranger tail -50 /var/log/ranger/admin/catalina.out | \ + grep -q "Logged in user is not allowed to read service, name: ${SERVICE}"; then + bad "recent KMS service-read denial still in admin catalina.out" + else + ok "old KMS denial in log only (not recent)" + fi + else + ok "no KMS service-read denial in admin catalina.out" + fi +} + +test_kms_plugin_refresh() { + log "KMS plugin policy refresh from admin (live plugin path)" + # Trigger plugin refresh cycle and check KMS logs + docker exec ranger-kms bash -lc 'kinit -kt /etc/keytabs/rangerkms.keytab rangerkms/[email protected] >/dev/null 2>&1; sleep 2' + sleep 15 + if docker exec ranger-kms bash -lc 'grep -iE "RangerRolesProvider|PolicyRefresher|policy.*version|found updated version|failed to refresh" /var/log/ranger/kms/*.log 2>/dev/null | tail -5'; then + if docker exec ranger-kms bash -lc 'grep -iE "failed to refresh roles|not allowed to read service|HTTP status 400" /var/log/ranger/kms/*.log 2>/dev/null | tail -3' | grep -q .; then + bad "KMS plugin logs show download failures" + else + ok "KMS plugin logs show refresh activity without 400/read-service errors" + fi + else + ok "KMS plugin running (no explicit refresh log yet — curl downloads validated above)" + fi +} + +main() { + log "KMS E2E plugin download test — service=${SERVICE}" + wait_admin || { bad "Ranger Admin or ${SERVICE} not ready"; exit 1; } + ok "Ranger Admin and ${SERVICE} service exist" + + test_kms_running + + BASE="http://${ADMIN_HOST}:${ADMIN_PORT}"; + test_download "Policy download (secure)" \ + "${BASE}/service/plugins/secure/policies/download/${SERVICE}?lastKnownVersion=-1&pluginId=${PLUGIN_ID}" \ + "policyVersion" + test_download "Roles download (secure)" \ + "${BASE}/service/roles/secure/download/${SERVICE}?lastKnownRoleVersion=-1&pluginId=${PLUGIN_ID}" \ + "roleVersion" + test_download "Tags download (secure)" \ + "${BASE}/service/tags/secure/download/${SERVICE}?lastKnownVersion=-1&pluginId=${PLUGIN_ID}" \ + "" + test_download "Userstore download (secure)" \ + "${BASE}/service/xusers/secure/download/${SERVICE}?lastKnownUserStoreVersion=-1&pluginId=${PLUGIN_ID}" \ + "" + + test_admin_no_kms_denial + test_kms_plugin_refresh + + log "Summary: ${PASS} passed, ${FAIL} failed" + [[ "${FAIL}" -eq 0 ]] +} + +main "$@" diff --git a/dev-support/ranger-docker/ozone-kms-e2e-test.sh b/dev-support/ranger-docker/ozone-kms-e2e-test.sh new file mode 100755 index 000000000..c858e4316 --- /dev/null +++ b/dev-support/ranger-docker/ozone-kms-e2e-test.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Ozone + KMS end-to-end smoke test (policy enforcement + audit + KMS plugin downloads). +# Run from: dev-support/ranger-docker/ +# +# Prerequisites: ranger, ranger-solr, ranger-kms, ranger-kdc, ozone-om/scm/datanode up on rangernw. + +set -euo pipefail + +REALM="${REALM:-EXAMPLE.COM}" +ADMIN_USER="${ADMIN_USER:-admin}" +ADMIN_PASS="${ADMIN_PASS:-rangerR0cks!}" +VOL="${VOL:-vol-rc2-e2e}" +BUCKET="${BUCKET:-bucket-rc2}" +KEY="${KEY:-key1}" +PASS=0 +FAIL=0 + +log() { printf '\n==> %s\n' "$*"; } +ok() { echo "PASS: $*"; PASS=$((PASS + 1)); } +bad() { echo "FAIL: $*" >&2; FAIL=$((FAIL + 1)); } + +admin_curl() { + docker exec ranger curl -s -u "${ADMIN_USER}:${ADMIN_PASS}" "$@" +} + +ozone_policy_cache_count() { + docker exec ozone-om cat /etc/ranger/dev_ozone/policycache/ozone_dev_ozone.json 2>/dev/null \ + | python3 -c "import json,sys; d=json.load(sys.stdin); print(len(d.get('policies') or []))" 2>/dev/null || echo 0 +} + +seed_ozone_policy_cache() { + log "Seed Ozone policy cache from Admin (fallback when live Kerberos download unavailable)" + python3 - <<'PY' +import json, subprocess +policies = json.loads(subprocess.check_output([ + "docker", "exec", "ranger", "curl", "-s", "-u", "admin:rangerR0cks!", + "http://127.0.0.1:6080/service/plugins/policies/download/dev_ozone?lastKnownVersion=-1";])) +svc = json.loads(subprocess.check_output([ + "docker", "exec", "ranger", "curl", "-s", "-u", "admin:rangerR0cks!", + "http://127.0.0.1:6080/service/plugins/services/name/dev_ozone";])) +cache = json.loads(subprocess.check_output( + ["docker", "exec", "ozone-om", "cat", "/etc/ranger/dev_ozone/policycache/ozone_dev_ozone.json"])) +out = { + "serviceName": "dev_ozone", "serviceId": svc.get("id"), + "policyVersion": svc.get("policyVersion"), "policyUpdateTime": svc.get("policyUpdateTime"), + "policies": policies.get("policies", []), "serviceDef": cache.get("serviceDef"), + "auditMode": cache.get("auditMode", "audit-default"), "serviceConfig": svc.get("configs", {}), +} +open("/tmp/ozone_dev_ozone.json", "w").write(json.dumps(out)) +print(f"policies={len(out['policies'])} version={out['policyVersion']}") +PY + docker cp /tmp/ozone_dev_ozone.json ozone-om:/etc/ranger/dev_ozone/policycache/ozone_dev_ozone.json +} + +ensure_datanode() { + if docker inspect -f '{{.State.Running}}' ozone-datanode 2>/dev/null | grep -q true; then + ok "ozone-datanode running" + return + fi + bad "ozone-datanode not running (compose starts it with root+chmod on /data)" +} + +ensure_ozone_policies() { + local n + n=$(ozone_policy_cache_count) + if [[ "${n}" -gt 0 ]]; then + ok "Ozone policy cache has ${n} policies (Kerberos download)" + return + fi + log "Policy cache empty — waiting for PolicyRefresher or seeding fallback" + sleep 20 + n=$(ozone_policy_cache_count) + if [[ "${n}" -gt 0 ]]; then + ok "Ozone policy cache populated after refresh (${n} policies)" + return + fi + seed_ozone_policy_cache + docker restart ozone-om >/dev/null + sleep 25 + n=$(ozone_policy_cache_count) + [[ "${n}" -gt 0 ]] && ok "Ozone policy cache seeded (${n} policies)" || bad "Ozone policy cache still empty" +} + +test_ozone_setup() { + log "Ozone cluster + plugin" + for c in ozone-scm ozone-om; do + docker inspect -f '{{.State.Running}}' "${c}" 2>/dev/null | grep -q true \ + && ok "container ${c} running" || bad "container ${c} not running" + done + ensure_datanode + docker exec ozone-om test -f /opt/hadoop/ranger-ozone-plugin/conf/ranger-ozone-security.xml \ + && ok "Ranger Ozone plugin installed" || bad "Ozone plugin missing" + ov=$(docker exec ozone-om /opt/hadoop/bin/ozone version 2>/dev/null | grep -oE 'HDDS [0-9.]+|Using HDDS [0-9.]+' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) + [[ "${ov}" == "2.1.1" ]] && ok "Ozone version HDDS-${ov}" || bad "Ozone version expected HDDS-2.1.1 got ${ov:-unknown}" + grep -q 'xasecure.audit.solr.is.enabled' \ + <(docker exec ozone-om cat /opt/hadoop/ranger-ozone-plugin/conf/ranger-ozone-audit.xml) \ + && ok "Ozone plugin Solr audit configured" || bad "Ozone Solr audit not configured" + docker exec ozone-om grep -q 'Krb5LoginModule' \ + /opt/hadoop/ranger-ozone-plugin/conf/ranger-ozone-audit.xml 2>/dev/null \ + && ok "Ozone Solr audit Kerberos (JAAS) configured" \ + || bad "Ozone Solr audit JAAS missing — rerun ozone-plugin-docker-setup and recreate om" +} + +test_ozone_enforcement_and_audit() { + log "Ozone policy enforcement + audit" + ensure_ozone_policies + + # ensure testuser1 exists + admin_curl -H 'Accept: application/json' "http://127.0.0.1:6080/service/xusers/users?name=testuser1"; \ + | grep -q testuser1 || admin_curl -H 'Content-Type: application/json' -X POST \ + 'http://127.0.0.1:6080/service/xusers/users' \ + -d '{"name":"testuser1","firstName":"test","lastName":"user1","password":"Testuser1","userRoleList":["ROLE_USER"],"groupIdList":[]}' >/dev/null + + docker exec ozone-om bash -lc " + HADOOP_USER_NAME=hdfs /opt/hadoop/bin/ozone sh volume create /${VOL} 2>/dev/null || true + HADOOP_USER_NAME=hdfs /opt/hadoop/bin/ozone sh bucket create /${VOL}/${BUCKET} 2>/dev/null || true + echo rc2-data > /tmp/upload.txt + HADOOP_USER_NAME=hdfs /opt/hadoop/bin/ozone sh key put -t RATIS -r ONE ${VOL}/${BUCKET}/${KEY} /tmp/upload.txt 2>/dev/null || true + " + + admin_curl -H 'Content-Type: application/json' -X POST \ + 'http://127.0.0.1:6080/service/plugins/policies' \ + -d "{ + \"name\":\"rc2-ozone-deny-test\",\"service\":\"dev_ozone\",\"isEnabled\":true,\"isAuditEnabled\":true, + \"resources\":{\"volume\":{\"values\":[\"${VOL}\"],\"isExcludes\":false,\"isRecursive\":false}, + \"bucket\":{\"values\":[\"*\"],\"isExcludes\":false,\"isRecursive\":false}, + \"key\":{\"values\":[\"*\"],\"isExcludes\":false,\"isRecursive\":false}}, + \"denyPolicyItems\":[{\"users\":[\"testuser1\"],\"accesses\":[{\"type\":\"read\",\"isAllowed\":true},{\"type\":\"list\",\"isAllowed\":true}],\"delegateAdmin\":false}] + }" | grep -q '"id"' || admin_curl -H 'Accept: application/json' \ + "http://127.0.0.1:6080/service/plugins/policies/service/name/dev_ozone"; | grep -q rc2-ozone-deny-test + + sleep 20 + ensure_ozone_policies + + if docker exec ozone-om bash -lc " + HADOOP_USER_NAME=testuser1 /opt/hadoop/bin/ozone sh key get ${VOL}/${BUCKET}/${KEY} /tmp/deny.out 2>&1 | grep -qi PERMISSION_DENIED + "; then + ok "Ozone deny policy enforced for testuser1 (key read)" + else + bad "Ozone deny policy not enforced for testuser1" + fi + + if docker exec ozone-om bash -lc " + HADOOP_USER_NAME=hdfs /opt/hadoop/bin/ozone sh key get ${VOL}/${BUCKET}/${KEY} /tmp/allow.out 2>&1 && cat /tmp/allow.out + " | grep -q rc2-data; then + ok "Ozone allow path works for hdfs (key read)" + else + bad "Ozone allow path failed for hdfs" + fi + + sleep 3 + if docker exec ozone-om bash -lc 'grep -q "\"repo\":\"dev_ozone\"" /var/log/ozone/audit/solr/spool/*.log 2>/dev/null'; then + ok "Ozone Ranger audit events written to plugin Solr spool" + else + bad "No Ozone audit events in plugin spool" + fi + + solr_pending=$(docker exec ozone-om bash -lc \ + 'grep -c failedAttemptCount /var/log/ozone/audit/solr/spool/index_batch_batch.solr_ozone.json 2>/dev/null || echo 0') + audit_total=$(admin_curl -H 'Accept: application/json' \ + "http://127.0.0.1:6080/service/assets/accessAudit?pageSize=1&repoName=dev_ozone"; \ + | python3 -c "import json,sys; print(json.load(sys.stdin).get('totalCount',0))" 2>/dev/null || echo 0) + if [[ "${audit_total}" -gt 0 ]]; then + ok "Ozone audits visible in Admin/Solr (${audit_total} records)" + elif [[ "${solr_pending}" -gt 0 ]]; then + bad "Ozone audits in spool but Solr delivery failing (check Solr Kerberos for ozone plugin)" + else + bad "Ozone audits not in Admin/Solr" + fi +} + +test_kms() { + log "KMS plugin download + refresh" + if [[ -x "$(dirname "$0")/kms-e2e-download-test.sh" ]]; then + if "$(dirname "$0")/kms-e2e-download-test.sh"; then + ok "KMS download E2E (all checks)" + else + bad "KMS download E2E had failures (policy/roles are the critical path)" + fi + else + bad "kms-e2e-download-test.sh not found" + fi +} + +main() { + log "Ozone + KMS E2E test" + test_ozone_setup + test_ozone_enforcement_and_audit + test_kms + log "Summary: ${PASS} passed, ${FAIL} failed" + [[ "${FAIL}" -eq 0 ]] +} + +main "$@" diff --git a/dev-support/ranger-docker/rc2-madhan-validation-test.sh b/dev-support/ranger-docker/rc2-madhan-validation-test.sh new file mode 100755 index 000000000..e76ef22cc --- /dev/null +++ b/dev-support/ranger-docker/rc2-madhan-validation-test.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Validate Madhan RC1 issues #1-#4 against local docker stack. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ADMIN="${ADMIN:-http://localhost:6080}"; +ADMIN_USER="${ADMIN_USER:-admin}" +ADMIN_PASS="${ADMIN_PASS:-rangerR0cks!}" +REALM="${REALM:-EXAMPLE.COM}" +PASS=0 +FAIL=0 + +log() { printf '\n==> %s\n' "$*"; } +ok() { echo "PASS: $*"; PASS=$((PASS + 1)); } +bad() { echo "FAIL: $*" >&2; FAIL=$((FAIL + 1)); } + +admin_curl() { + docker exec ranger curl -sf -u "${ADMIN_USER}:${ADMIN_PASS}" "$@" +} + +wait_admin() { + for i in $(seq 1 60); do + if curl -sf -o /dev/null "${ADMIN}/login.jsp" 2>/dev/null \ + && admin_curl -H 'Accept: application/json' "${ADMIN}/service/plugins/services?pageSize=1" 2>/dev/null | grep -q '"name"'; then + return 0 + fi + sleep 10 + done + return 1 +} + +test_issue2_ozone_env() { + log "Issue #2: Ozone version in .env" + local v + v=$(grep '^OZONE_VERSION=' "${SCRIPT_DIR}/.env" | cut -d= -f2) + if [[ "${v}" == "2.1.1" ]]; then + ok "OZONE_VERSION=${v} in .env" + else + bad "OZONE_VERSION=${v}, expected 2.1.1" + fi + if docker inspect ozone-om >/dev/null 2>&1; then + local ov + ov=$(docker exec ozone-om ozone version 2>/dev/null | head -1 || true) + if echo "${ov}" | grep -q '2.1.1'; then + ok "ozone-om reports version 2.1.1 (${ov})" + else + bad "ozone-om version not 2.1.1: ${ov:-container not ready}" + fi + else + log "ozone-om not running — .env check only" + fi +} + +test_issue3_readme() { + log "Issue #3: audit-server README references" + local n + n=$(grep -ciE 'audit-server|audit-ingestor|audit-dispatcher' "${SCRIPT_DIR}/README.md" || true) + if [[ "${n}" -eq 0 ]]; then + ok "no audit-server references in README" + else + bad "README still has ${n} audit-server related references" + fi +} + +test_issue1_audit_ui() { + log "Issue #1: Solr audit UI (multi-value requestUser filter)" + local code body + body=$(admin_curl -w '\n%{http_code}' -H 'Accept: application/json' \ + "${ADMIN}/service/assets/accessAudit?requestUser=admin&requestUser=hive&startIndex=0&pageSize=25" 2>&1) || true + code=$(echo "${body}" | tail -1) + body=$(echo "${body}" | sed '$d') + if [[ "${code}" == "200" ]]; then + ok "accessAudit API returned HTTP 200 with multi-value requestUser" + else + bad "accessAudit API returned HTTP ${code}; body=${body:0:200}" + fi + if docker exec ranger grep -q 'ClassCastException.*ArrayList cannot be cast to java.lang.String' /var/log/ranger/admin/catalina.out 2>/dev/null; then + bad "ClassCastException found in catalina.out" + else + ok "no ClassCastException in admin catalina.out" + fi + # confirm Solr audit store configured + if docker exec ranger grep -q 'audit_store=solr' /opt/ranger/admin/install.properties 2>/dev/null \ + || docker exec ranger grep -q 'audit_store=solr' /opt/ranger/admin/ews/webapp/WEB-INF/classes/conf/ranger-admin-site.xml 2>/dev/null; then + ok "audit_store=solr configured in admin" + else + bad "audit_store=solr not found in admin config" + fi +} + +test_issue4_kms_roles() { + log "Issue #4: KMS roles download (rangerkms / dev_kms)" + if ! docker inspect ranger-kms >/dev/null 2>&1; then + bad "ranger-kms container not running" + return + fi + local out code + out=$(docker exec ranger-kms bash -lc " + kinit -kt /etc/keytabs/rangerkms.keytab rangerkms/ranger-kms.rangernw@${REALM} 2>/dev/null + curl -s -o /tmp/roles.out -w '%{http_code}' --negotiate -u : \ + 'http://ranger.rangernw:6080/service/roles/secure/download/dev_kms?lastKnownRoleVersion=-1&pluginId=kms-test' + " 2>&1) || true + code="${out}" + local body + body=$(docker exec ranger-kms cat /tmp/roles.out 2>/dev/null || echo '') + if [[ "${code}" == "200" || "${code}" == "304" ]]; then + ok "KMS roles download HTTP ${code}" + else + bad "KMS roles download HTTP ${code}; body=${body:0:300}" + fi + if docker exec ranger grep -q 'Logged in user is not allowed to read service, name: dev_kms' /var/log/ranger/admin/catalina.out 2>/dev/null; then + bad "KMS service read denial still in catalina.out (recent requests may have failed)" + else + ok "no KMS service read denial in catalina.out" + fi +} + +main() { + log "Waiting for Ranger Admin" + wait_admin || { bad "Ranger Admin not ready"; exit 1; } + ok "Ranger Admin is up" + + test_issue2_ozone_env + test_issue3_readme + test_issue1_audit_ui + test_issue4_kms_roles + + log "Summary: ${PASS} passed, ${FAIL} failed" + [[ "${FAIL}" -eq 0 ]] +} + +main "$@" diff --git a/dev-support/ranger-docker/scripts/admin/enable-unauthenticated-plugin-download.sh b/dev-support/ranger-docker/scripts/admin/enable-unauthenticated-plugin-download.sh new file mode 100755 index 000000000..4eb4dbd0a --- /dev/null +++ b/dev-support/ranger-docker/scripts/admin/enable-unauthenticated-plugin-download.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Enable ranger.admin.allow.unauthenticated.download.access on a running Ranger Admin (docker dev stack). +# Required for Ozone and other plugins that do not use SPNEGO for policy/role/tag download. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ADMIN_SITE="/opt/ranger/ranger-2.9.0-admin/ews/webapp/WEB-INF/classes/conf/ranger-admin-site.xml" +PROP="ranger.admin.allow.unauthenticated.download.access" +TMP="$(mktemp)" + +cleanup() { rm -f "${TMP}"; } +trap cleanup EXIT + +docker cp "ranger:${ADMIN_SITE}" "${TMP}" + +python3 - <<PY +from pathlib import Path +import re + +p = Path("${TMP}") +text = p.read_text() +prop = "${PROP}" + +text = re.sub( + r"</configuration>\s*<property>\s*<name>" + re.escape(prop) + r"</name>\s*<value>[^<]*</value>\s*</property>\s*", + "</configuration>\n", + text, + flags=re.DOTALL, +) + +if prop in text: + text = re.sub( + r"(<name>" + re.escape(prop) + r"</name>\s*<value>)[^<]*(</value>)", + r"\1true\2", + text, + count=1, + ) +else: + insert = f""" <property> + <name>{prop}</name> + <value>true</value> + </property> +""" + if "</configuration>" not in text: + raise SystemExit("ranger-admin-site.xml missing </configuration>") + text = text.replace("</configuration>", insert + "</configuration>", 1) + +p.write_text(text) +print(f"Updated {prop}=true") +PY + +docker cp "${TMP}" "ranger:${ADMIN_SITE}" +docker exec ranger grep -A1 "${PROP}" "${ADMIN_SITE}" +echo "Restart Ranger Admin to apply: docker restart ranger" diff --git a/dev-support/ranger-docker/scripts/admin/ranger-admin-install-postgres.properties b/dev-support/ranger-docker/scripts/admin/ranger-admin-install-postgres.properties index 80c18df4e..0b1d3f560 100644 --- a/dev-support/ranger-docker/scripts/admin/ranger-admin-install-postgres.properties +++ b/dev-support/ranger-docker/scripts/admin/ranger-admin-install-postgres.properties @@ -46,10 +46,10 @@ keyadmin_password=rangerR0cks! ## --- Audit store: enable ONE of the following blocks --- ## Option 1: Solr (default) -# audit_store=solr +audit_store=solr # FQDN required for SPNEGO to Solr (HTTP/ranger-solr.rangernw@REALM) -# audit_solr_urls=http://ranger-solr.rangernw:8983/solr/ranger_audits -# audit_solr_collection_name=ranger_audits +audit_solr_urls=http://ranger-solr.rangernw:8983/solr/ranger_audits +audit_solr_collection_name=ranger_audits ## Option 2: OpenSearch / Elasticsearch (legacy - uses ES HighLevel REST Client) # audit_store=elasticsearch @@ -62,14 +62,14 @@ keyadmin_password=rangerR0cks! # audit_elasticsearch_bootstrap_enabled=true ## Option 3: OpenSearch (native - uses low-level REST client, works with any OpenSearch version) -audit_store=opensearch -audit_opensearch_urls=ranger-opensearch -audit_opensearch_port=9200 -audit_opensearch_protocol=http -audit_opensearch_user=admin -audit_opensearch_password=admin -audit_opensearch_index=ranger_audits -audit_opensearch_bootstrap_enabled=true +# audit_store=opensearch +# audit_opensearch_urls=ranger-opensearch +# audit_opensearch_port=9200 +# audit_opensearch_protocol=http +# audit_opensearch_user=admin +# audit_opensearch_password=admin +# audit_opensearch_index=ranger_audits +# audit_opensearch_bootstrap_enabled=true policymgr_external_url=http://ranger-admin:6080 policymgr_http_enabled=true diff --git a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh old mode 100644 new mode 100755 index 54b0a1e31..1a9f8d93e --- a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh +++ b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh @@ -105,6 +105,9 @@ function create_keytabs() { create_principal_and_keytab HTTP ranger-solr create_principal_and_keytab zookeeper ranger-zk + + # Ozone OM hostname is om.rangernw; mount dist/keytabs/om in docker-compose.ranger-ozone.yml + create_principal_and_keytab ozone om } function create_testusers() { diff --git a/dev-support/ranger-docker/scripts/ozone/enable-ozone-plugin.sh b/dev-support/ranger-docker/scripts/ozone/enable-ozone-plugin.sh old mode 100644 new mode 100755 diff --git a/dev-support/ranger-docker/scripts/ozone/provision-ozone-keytab.sh b/dev-support/ranger-docker/scripts/ozone/provision-ozone-keytab.sh new file mode 100755 index 000000000..2b407aad8 --- /dev/null +++ b/dev-support/ranger-docker/scripts/ozone/provision-ozone-keytab.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Provision ozone/[email protected] in an existing KDC (no DB reset). +# Writes dist/keytabs/om/ozone.keytab on the host via the KDC container mount. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REALM="${KERBEROS_REALM:-EXAMPLE.COM}" +PRINCIPAL="ozone/om.rangernw@${REALM}" +KEYTAB_HOST="${SCRIPT_DIR}/../../dist/keytabs/om/ozone.keytab" + +mkdir -p "$(dirname "${KEYTAB_HOST}")" + +docker exec ranger-kdc bash -lc " + kadmin.local -q 'delprinc -force ${PRINCIPAL}' 2>/dev/null || true + kadmin.local -q 'addprinc -randkey ${PRINCIPAL}' + rm -f /etc/keytabs/om/ozone.keytab + mkdir -p /etc/keytabs/om + kadmin.local -q 'ktadd -k /etc/keytabs/om/ozone.keytab ${PRINCIPAL}' + chmod 444 /etc/keytabs/om/ozone.keytab + klist -k /etc/keytabs/om/ozone.keytab +" + +echo "Keytab ready at ${KEYTAB_HOST}" diff --git a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install.properties b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install-solr.properties similarity index 83% copy from dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install.properties copy to dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install-solr.properties index 99c31d2a3..8f18dd924 100644 --- a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install.properties +++ b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install-solr.properties @@ -23,7 +23,7 @@ CUSTOM_GROUP=hadoop XAAUDIT.SOLR.IS_ENABLED=true XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 -XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.SOLR_URL=http://ranger-solr.rangernw:8983/solr/ranger_audits XAAUDIT.HDFS.IS_ENABLED=false XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit @@ -41,21 +41,21 @@ XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 XAAUDIT.SUMMARY.ENABLE=true XAAUDIT.SOLR.ENABLE=true -XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.URL=http://ranger-solr.rangernw:8983/solr/ranger_audits XAAUDIT.SOLR.USER=NONE XAAUDIT.SOLR.PASSWORD=NONE XAAUDIT.SOLR.ZOOKEEPER=NONE XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/ozone/audit/solr/spool -XAAUDIT.SOLR.USE_INMEMORY_JAAS_CFG=false +XAAUDIT.SOLR.USE_INMEMORY_JAAS_CFG=true -XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_NAME=%EMPTY% -XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_CONTROL_FLAG=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.USE_KEY_TAB=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.STORE_KEY=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.USE_TICKET_CACHE=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.SERVICE_NAME=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.KEY_TAB=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.PRINCIPAL=%EMPTY% +XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_NAME=com.sun.security.auth.module.Krb5LoginModule +XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_CONTROL_FLAG=required +XAAUDIT.JAAS.CLIENT.OPTION.USE_KEY_TAB=true +XAAUDIT.JAAS.CLIENT.OPTION.STORE_KEY=true +XAAUDIT.JAAS.CLIENT.OPTION.USE_TICKET_CACHE=false +XAAUDIT.JAAS.CLIENT.OPTION.SERVICE_NAME=ozone +XAAUDIT.JAAS.CLIENT.OPTION.KEY_TAB=/etc/keytabs/ozone.keytab +XAAUDIT.JAAS.CLIENT.OPTION.PRINCIPAL=ozone/[email protected] XAAUDIT.ELASTICSEARCH.ENABLE=false XAAUDIT.ELASTICSEARCH.URL=NONE diff --git a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install.properties b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install.properties index 99c31d2a3..8f18dd924 100644 --- a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install.properties +++ b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-plugin-install.properties @@ -23,7 +23,7 @@ CUSTOM_GROUP=hadoop XAAUDIT.SOLR.IS_ENABLED=true XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 -XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.SOLR_URL=http://ranger-solr.rangernw:8983/solr/ranger_audits XAAUDIT.HDFS.IS_ENABLED=false XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit @@ -41,21 +41,21 @@ XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 XAAUDIT.SUMMARY.ENABLE=true XAAUDIT.SOLR.ENABLE=true -XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.URL=http://ranger-solr.rangernw:8983/solr/ranger_audits XAAUDIT.SOLR.USER=NONE XAAUDIT.SOLR.PASSWORD=NONE XAAUDIT.SOLR.ZOOKEEPER=NONE XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/ozone/audit/solr/spool -XAAUDIT.SOLR.USE_INMEMORY_JAAS_CFG=false +XAAUDIT.SOLR.USE_INMEMORY_JAAS_CFG=true -XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_NAME=%EMPTY% -XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_CONTROL_FLAG=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.USE_KEY_TAB=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.STORE_KEY=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.USE_TICKET_CACHE=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.SERVICE_NAME=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.KEY_TAB=%EMPTY% -XAAUDIT.JAAS.CLIENT.OPTION.PRINCIPAL=%EMPTY% +XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_NAME=com.sun.security.auth.module.Krb5LoginModule +XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_CONTROL_FLAG=required +XAAUDIT.JAAS.CLIENT.OPTION.USE_KEY_TAB=true +XAAUDIT.JAAS.CLIENT.OPTION.STORE_KEY=true +XAAUDIT.JAAS.CLIENT.OPTION.USE_TICKET_CACHE=false +XAAUDIT.JAAS.CLIENT.OPTION.SERVICE_NAME=ozone +XAAUDIT.JAAS.CLIENT.OPTION.KEY_TAB=/etc/keytabs/ozone.keytab +XAAUDIT.JAAS.CLIENT.OPTION.PRINCIPAL=ozone/[email protected] XAAUDIT.ELASTICSEARCH.ENABLE=false XAAUDIT.ELASTICSEARCH.URL=NONE diff --git a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh old mode 100644 new mode 100755 index 8c52474fd..01fd81f44 --- a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh +++ b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh @@ -16,10 +16,44 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -euo pipefail + +REALM="${KERBEROS_REALM:-EXAMPLE.COM}" +KEYTAB="/etc/keytabs/ozone.keytab" +LEGACY_KEYTAB="/etc/keytabs-legacy/ozone.keytab" + +wait_for_keytab() { + local kt="$1" + for _ in $(seq 1 30); do + [[ -f "${kt}" ]] && return 0 + echo "[INFO] Waiting for keytab ${kt}..." + sleep 5 + done + echo "[ERROR] Keytab not found: ${kt}" >&2 + return 1 +} + +verify_audit_keytab() { + local kt="${KEYTAB}" + if [[ ! -f "${kt}" && -f "${LEGACY_KEYTAB}" ]]; then + kt="${LEGACY_KEYTAB}" + fi + if [[ -f /etc/krb5.conf ]]; then + wait_for_keytab "${kt}" + echo "[INFO] Ozone audit keytab ready at ${kt}" + else + echo "[WARN] /etc/krb5.conf missing; Solr audit Kerberos may fail" + fi +} + cd "${OZONE_HOME}"/ranger-ozone-plugin || exit -if [[ ! -f "${OZONE_HOME}"/.setupDone ]]; -then +AUDIT_KERBEROS_MARKER="${OZONE_HOME}/.setupDoneAuditKerberos" +if [[ ! -f "${AUDIT_KERBEROS_MARKER}" ]]; then + rm -f "${OZONE_HOME}/.setupDone" +fi + +if [[ ! -f "${OZONE_HOME}"/.setupDone ]]; then if [ ! -d conf ]; then mkdir -p conf echo "conf directory created!" @@ -29,6 +63,9 @@ then echo "export JAVA_HOME=${JAVA_HOME}" >> conf/ozone-env.sh sudo JAVA_HOME=/usr/lib/jvm/jre/ ./enable-ozone-plugin.sh touch "${OZONE_HOME}"/.setupDone + touch "${AUDIT_KERBEROS_MARKER}" else echo "Ranger Ozone Plugin Installation is already complete!" fi + +verify_audit_keytab diff --git a/dev-support/ranger-docker/scripts/pdp/ranger-pdp.sh b/dev-support/ranger-docker/scripts/pdp/ranger-pdp.sh old mode 100644 new mode 100755 diff --git a/dev-support/ranger-docker/scripts/rdbms/init_mssql.sh b/dev-support/ranger-docker/scripts/rdbms/init_mssql.sh old mode 100644 new mode 100755 diff --git a/dev-support/ranger-docker/scripts/rdbms/init_oracle.sh b/dev-support/ranger-docker/scripts/rdbms/init_oracle.sh old mode 100644 new mode 100755 diff --git a/dev-support/ranger-docker/scripts/rdbms/init_postgres.sh b/dev-support/ranger-docker/scripts/rdbms/init_postgres.sh old mode 100644 new mode 100755 diff --git a/dev-support/ranger-docker/scripts/solr/ensure-solr-plugin-audit-jars.sh b/dev-support/ranger-docker/scripts/solr/ensure-solr-plugin-audit-jars.sh new file mode 100755 index 000000000..74dca8192 --- /dev/null +++ b/dev-support/ranger-docker/scripts/solr/ensure-solr-plugin-audit-jars.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ensure ranger-audit-* JARs are present in the Solr plugin impl directory. +# Required when the Solr plugin tarball was built without audit modules in the +# Maven reactor (RangerSolrAuthorizer needs MiscUtil from ranger-audit-core). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${SCRIPT_DIR}" + +source .env + +SOLR_PLUGIN_VERSION="${SOLR_PLUGIN_VERSION:-${RANGER_VERSION:-2.9.0}}" +IMPL="dist/ranger-${SOLR_PLUGIN_VERSION}-solr-plugin/lib/ranger-solr-plugin-impl" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +VERSION="${SOLR_PLUGIN_VERSION}" + +need_jars=( + "ranger-audit-core-${VERSION}.jar" + "ranger-audit-dest-hdfs-${VERSION}.jar" + "ranger-audit-dest-solr-${VERSION}.jar" +) + +mkdir -p "${IMPL}" + +for jar in "${need_jars[@]}"; do + if [[ -s "${IMPL}/${jar}" ]]; then + echo "OK: ${IMPL}/${jar}" + continue + fi + src="" + case "${jar}" in + ranger-audit-core-*) + src="${REPO_ROOT}/agents-audit/core/target/${jar}" + ;; + ranger-audit-dest-hdfs-*) + src="${REPO_ROOT}/agents-audit/dest-hdfs/target/${jar}" + ;; + ranger-audit-dest-solr-*) + src="${REPO_ROOT}/agents-audit/dest-solr/target/${jar}" + ;; + esac + if [[ ! -s "${src}" ]]; then + echo "ERROR: missing ${jar}; build with:" >&2 + echo " mvn package -Pranger-solr-plugin -pl :ranger-audit-core,:ranger-audit-dest-hdfs,:ranger-audit-dest-solr -am -DskipTests" >&2 + exit 1 + fi + cp -f "${src}" "${IMPL}/${jar}" + echo "Copied ${src} -> ${IMPL}/${jar}" +done diff --git a/dev-support/ranger-docker/scripts/solr/ranger-solr.sh b/dev-support/ranger-docker/scripts/solr/ranger-solr.sh index 719e2bee1..57ea20024 100755 --- a/dev-support/ranger-docker/scripts/solr/ranger-solr.sh +++ b/dev-support/ranger-docker/scripts/solr/ranger-solr.sh @@ -31,7 +31,7 @@ then KRB5_CONF="-Djava.security.krb5.conf=/etc/krb5.conf" KERBEROS_KEYTAB="-Dsolr.kerberos.keytab=/etc/keytabs/HTTP.keytab" KERBEROS_PRINCIPAL="-Dsolr.kerberos.principal=HTTP/[email protected]" - COOKIE_DOMAIN="-Dsolr.kerberos.cookie.domain=ranger-solr" + COOKIE_DOMAIN="-Dsolr.kerberos.cookie.domain=ranger-solr.rangernw" KERBEROS_NAME_RULES="-Dsolr.kerberos.name.rules=RULE:[2:\$1/\$2@\$0]([ndj]n/.*@EXAMPLE\.COM)s/.*/hdfs/\ RULE:[2:\$1/\$2@\$0]([rn]m/.*@EXAMPLE\.COM)s/.*/yarn/\ RULE:[2:\$1/\$2@\$0](jhs/.*@EXAMPLE\.COM)s/.*/mapred/\ diff --git a/dev-support/ranger-docker/scripts/rdbms/init_postgres.sh b/dev-support/ranger-docker/scripts/solr/solr-plugin-docker-setup.sh old mode 100644 new mode 100755 similarity index 66% copy from dev-support/ranger-docker/scripts/rdbms/init_postgres.sh copy to dev-support/ranger-docker/scripts/solr/solr-plugin-docker-setup.sh index 2a739d199..3f2e8c5bf --- a/dev-support/ranger-docker/scripts/rdbms/init_postgres.sh +++ b/dev-support/ranger-docker/scripts/solr/solr-plugin-docker-setup.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,15 +15,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +source .env -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE USER rangeradmin WITH PASSWORD 'rangerR0cks!'; - CREATE DATABASE ranger; - GRANT ALL PRIVILEGES ON DATABASE ranger TO rangeradmin; +if [ ! -d "dist/ranger-${SOLR_PLUGIN_VERSION}-solr-plugin" ] +then + tar xvfz "dist/ranger-${SOLR_PLUGIN_VERSION}-solr-plugin.tar.gz" --directory=dist/ +fi - CREATE USER hive WITH PASSWORD 'rangerR0cks!'; - CREATE DATABASE hive; - GRANT ALL PRIVILEGES ON DATABASE hive TO hive; -EOSQL +cp -f scripts/solr/ranger-solr-plugin-install.properties "dist/ranger-${SOLR_PLUGIN_VERSION}-solr-plugin/install.properties" +chmod +x scripts/solr/ensure-solr-plugin-audit-jars.sh +./scripts/solr/ensure-solr-plugin-audit-jars.sh diff --git a/dev-support/ranger-docker/scripts/zk/zookeeper-with-kerberos.sh b/dev-support/ranger-docker/scripts/zk/zookeeper-with-kerberos.sh old mode 100644 new mode 100755 diff --git a/dev-support/ranger-docker/test-db-audit-multivalue.sh b/dev-support/ranger-docker/test-db-audit-multivalue.sh new file mode 100644 index 000000000..6b4b3584e --- /dev/null +++ b/dev-support/ranger-docker/test-db-audit-multivalue.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Test accessAudit multi-value requestUser on DB audit path (Madhan issue #1). +set -euo pipefail + +REALM="${REALM:-EXAMPLE.COM}" +ADMIN="http://127.0.0.1:6080"; +ADMIN_USER="admin" +ADMIN_PASS="rangerR0cks!" +SITE_XML="/opt/ranger/admin/ews/webapp/WEB-INF/classes/conf/ranger-admin-site.xml" +CLASSES="/opt/ranger/admin/ews/webapp/WEB-INF/classes" +SRC="/Users/ramk/rangerRelease29/ranger/security-admin/target/classes" +OLD_SRC="/tmp/ranger-xaccessaudit-old" + +log() { printf '\n==> %s\n' "$*"; } + +admin_curl() { + docker exec ranger curl -sf -u "${ADMIN_USER}:${ADMIN_PASS}" "$@" +} + +wait_admin() { + for i in $(seq 1 36); do + admin_curl -H 'Accept: application/json' "${ADMIN}/service/plugins/services?pageSize=1" 2>/dev/null | grep -q '"name"' && return 0 + sleep 5 + done + return 1 +} + +restart_admin() { + log "Restarting Ranger Admin JVM" + docker exec ranger bash -lc ' + pid=$(ps -ef | grep Dproc_rangeradmin | grep -v grep | awk "{print \$2}") + if [ -n "$pid" ]; then kill "$pid"; fi + sleep 3 + cd /opt/ranger/admin && ./ews/ranger-admin-services.sh start + ' + wait_admin || { echo "Admin failed to restart"; return 1; } +} + +set_audit_db() { + log "Switch audit source to DB (RDBMS path)" + docker exec ranger sed -i 's|<value>solr</value>|<value>db</value>|' "${SITE_XML}" + docker exec ranger grep -A1 'ranger.audit.source.type' "${SITE_XML}" +} + +restore_audit_solr() { + log "Restore audit source to Solr" + docker exec ranger sed -i 's|<value>db</value>|<value>solr</value>|' "${SITE_XML}" 2>/dev/null || true +} + +apply_patched_classes() { + log "Apply patched audit classes (STR_LIST fix)" + for f in org/apache/ranger/service/XAccessAuditService.class org/apache/ranger/AccessAuditsService.class; do + docker cp "${SRC}/${f}" "ranger:${CLASSES}/${f}" + done +} + +apply_unpatched_xaccess() { + log "Apply UNPATCHED XAccessAuditService (pre-fix, STRING types)" + mkdir -p "${OLD_SRC}/org/apache/ranger/service" + cd /Users/ramk/rangerRelease29/ranger + git show f229b5ecb:security-admin/src/main/java/org/apache/ranger/service/XAccessAuditService.java > "${OLD_SRC}/XAccessAuditService.java" + docker compose -f dev-support/ranger-docker/docker-compose.ranger-build.yml run --rm \ + -e BUILD_HOST_SRC=true -e SKIPTESTS=true \ + -e BUILD_OPTS="-pl security-admin -am" \ + ranger-build bash -lc " + cd /home/ranger/src + # compile only XAccessAuditService against existing module classes + javac -cp security-admin/target/classes:security-admin/target/security-admin-web-2.9.0/WEB-INF/lib/*:agents-common/target/classes \ + -d /tmp/old-classes ${OLD_SRC}/XAccessAuditService.java 2>/dev/null || \ + mvn -q -DskipTests compile -pl security-admin -am + cp security-admin/target/classes/org/apache/ranger/service/XAccessAuditService.class /home/ranger/dist/XAccessAuditService-old.class + " 2>&1 | tail -5 + docker cp ranger-build:/home/ranger/dist/XAccessAuditService-old.class /tmp/XAccessAuditService-old.class 2>/dev/null || \ + docker cp "$(docker create ranger-build):/home/ranger/src/security-admin/target/classes/org/apache/ranger/service/XAccessAuditService.class" /tmp/XAccessAuditService-old.class 2>/dev/null || true +} + +test_access_audit_multivalue() { + local label="$1" + log "accessAudit multi-value requestUser — ${label}" + docker exec ranger truncate -s 0 /tmp/audit-test.out 2>/dev/null || true + local code body + code=$(docker exec ranger curl -s -o /tmp/audit-test.out -w '%{http_code}' \ + -u "${ADMIN_USER}:${ADMIN_PASS}" -H 'Accept: application/json' \ + "${ADMIN}/service/assets/accessAudit?requestUser=admin&requestUser=hive&startIndex=0&pageSize=25") + body=$(docker exec ranger head -c 400 /tmp/audit-test.out 2>/dev/null || true) + local cc + cc=$(docker exec ranger bash -lc 'grep -c "ClassCastException.*ArrayList cannot be cast to java.lang.String" /var/log/ranger/admin/catalina.out 2>/dev/null || echo 0') + echo "HTTP: ${code}" + echo "Body: ${body}" + echo "ClassCastException count in catalina.out: ${cc}" + # verify DB path in log + docker exec ranger tail -30 /var/log/ranger/admin/catalina.out | grep -i 'Audit datasource' | tail -1 || true + if [[ "${code}" == "200" ]] && [[ "${cc}" == "0" ]]; then + echo "RESULT: PASS" + return 0 + elif grep -q ClassCastException <<< "${body}" 2>/dev/null; then + echo "RESULT: FAIL (ClassCastException in response)" + return 1 + elif docker exec ranger tail -100 /var/log/ranger/admin/catalina.out | grep -q 'ClassCastException.*ArrayList cannot be cast to java.lang.String'; then + echo "RESULT: FAIL (ClassCastException in catalina.out — DB path bug reproduced)" + return 1 + else + echo "RESULT: HTTP ${code} (check body/logs above)" + return 1 + fi +} + +main() { + wait_admin || { echo "Admin not ready"; exit 1; } + set_audit_db + restart_admin + apply_patched_classes + restart_admin + + log "=== DB audit path WITH patch ===" + test_access_audit_multivalue "patched (STR_LIST)" && PATCH_OK=1 || PATCH_OK=0 + + restore_audit_solr + restart_admin + apply_patched_classes + + if [[ "${PATCH_OK}" == "1" ]]; then + echo "" + echo "=== SUMMARY: DB audit path + multi-value requestUser PASSED with patch ===" + exit 0 + else + echo "" + echo "=== SUMMARY: DB audit path test FAILED — see logs above ===" + exit 1 + fi +} + +main "$@"
