This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7af8c44009 HDDS-11454. Ranger integration for Docker Compose
environment (#8575)
7af8c44009 is described below
commit 7af8c44009a9359ec5605813f0aa3fea991b04e0
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Jun 16 05:11:23 2025 +0200
HDDS-11454. Ranger integration for Docker Compose environment (#8575)
---
.../dist/src/main/compose/common/ranger.yaml | 47 +++++++++++++++++
.../dist/src/main/compose/ozonesecure-ha/.env | 5 ++
.../src/main/compose/ozonesecure-ha/ranger.yaml | 54 ++++++++++++++++++++
.../src/main/compose/ozonesecure-ha/test-ranger.sh | 59 ++++++++++++++++++++++
hadoop-ozone/dist/src/main/compose/testlib.sh | 29 ++++++++++-
5 files changed, 192 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/common/ranger.yaml
b/hadoop-ozone/dist/src/main/compose/common/ranger.yaml
new file mode 100644
index 0000000000..53d3c9363f
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/common/ranger.yaml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache Ranger
+#
+# This requires Apache Ranger source to be available in $RANGER_SOURCE_DIR.
+
+services:
+ ranger-db:
+ image: ${RANGER_DB_IMAGE}:${RANGER_DB_IMAGE_VERSION}
+ hostname: ranger-db
+ dns_search: .
+ environment:
+ POSTGRES_PASSWORD: "rangerR0cks!"
+ volumes:
+ -
${RANGER_SOURCE_DIR}/dev-support/ranger-docker/config/init_postgres.sh:/docker-entrypoint-initdb.d/init_postgres.sh
+ healthcheck:
+ test: 'su -c "pg_isready -q" postgres'
+ interval: 10s
+ timeout: 2s
+ retries: 30
+
+ ranger:
+ image: ${RANGER_IMAGE}:${RANGER_IMAGE_VERSION}
+ hostname: ranger
+ dns_search: .
+ ports:
+ - 6080:6080
+ depends_on:
+ ranger-db:
+ condition: service_healthy
+ environment:
+ RANGER_DB_TYPE: postgres
+ RANGER_VERSION:
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
index df30e9989a..75619126ca 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
@@ -22,3 +22,8 @@ OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
OZONE_VOLUME=./data
OZONE_OPTS=
+RANGER_DB_IMAGE=postgres
+RANGER_DB_IMAGE_VERSION=12
+RANGER_IMAGE=ghcr.io/adoroszlai/ranger-admin
+RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.version}-8
+RANGER_VERSION=${ranger.version}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/ranger.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/ranger.yaml
new file mode 100644
index 0000000000..1d3c225531
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/ranger.yaml
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+x-om-ranger-config:
+ &om-ranger-config
+ environment:
+ OZONE_MANAGER_CLASSPATH:
"/opt/ranger/ozone-plugin/lib/libext/*:/opt/ozone/conf"
+ OZONE-SITE.XML_ozone.acl.authorizer.class:
"org.apache.ranger.authorization.ozone.authorizer.RangerOzoneAuthorizer"
+ OZONE-SITE.XML_ozone.om.multitenancy.enabled: "true"
+ OZONE-SITE.XML_ozone.om.ranger.https-address: "http://ranger:6080";
+ OZONE-SITE.XML_ozone.om.ranger.https.admin.api.user: "admin"
+ OZONE-SITE.XML_ozone.om.ranger.https.admin.api.passwd: "rangerR0cks!"
+ OZONE-SITE.XML_ozone.om.ranger.service: "dev_ozone"
+ OZONE-SITE.XML_ozone.om.tenant.dev.skip.ranger: "false"
+ RANGER-OZONE-SECURITY.XML_ranger.plugin.ozone.policy.rest.url:
"http://ranger:6080";
+
RANGER-OZONE-SECURITY.XML_ranger.plugin.ozone.policyengine.option.disable.policy.refresher:
"true"
+ RANGER-OZONE-SECURITY.XML_ranger.plugin.ozone.service.name: "dev_ozone"
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${RANGER_OZONE_PLUGIN_DIR}:/opt/ranger/ozone-plugin
+ tmpfs:
+ - /opt/ozone/conf
+ command: bash -c "sudo --preserve-env
/opt/ranger/ozone-plugin/enable-ozone-plugin.sh && /opt/hadoop/bin/ozone om"
+
+services:
+ om1:
+ <<: *om-ranger-config
+ om2:
+ <<: *om-ranger-config
+ om3:
+ <<: *om-ranger-config
+ ranger:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.200
+ ranger-db:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.201
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh
new file mode 100755
index 0000000000..8fbce08fb5
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:misc
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+: "${RANGER_VERSION:=2.6.0}"
+: "${DOWNLOAD_DIR:=${TEMP_DIR:-/tmp}}"
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+export COMPOSE_FILE=docker-compose.yaml:ranger.yaml:../common/ranger.yaml
+export OM_SERVICE_ID="omservice"
+export SCM=scm1.org
+export SECURITY_ENABLED=true
+
+curl -LO https://downloads.apache.org/ranger/KEYS
+gpg --import KEYS
+
+download_and_verify_apache_release
"ranger/${RANGER_VERSION}/apache-ranger-${RANGER_VERSION}.tar.gz"
+tar -C "${DOWNLOAD_DIR}" -x -z -f
"${DOWNLOAD_DIR}/apache-ranger-${RANGER_VERSION}.tar.gz"
+export RANGER_SOURCE_DIR="${DOWNLOAD_DIR}/apache-ranger-${RANGER_VERSION}"
+chmod -R a+rX "${RANGER_SOURCE_DIR}"
+chmod a+x "${RANGER_SOURCE_DIR}"/dev-support/ranger-docker/config/*.sh
+
+download_and_verify_apache_release
"ranger/${RANGER_VERSION}/plugins/ozone/ranger-${RANGER_VERSION}-ozone-plugin.tar.gz"
+tar -C "${DOWNLOAD_DIR}" -x -z -f
"${DOWNLOAD_DIR}/ranger-${RANGER_VERSION}-ozone-plugin.tar.gz"
+export
RANGER_OZONE_PLUGIN_DIR="${DOWNLOAD_DIR}/ranger-${RANGER_VERSION}-ozone-plugin"
+chmod -R a+rX "${RANGER_OZONE_PLUGIN_DIR}"
+chmod a+x "${RANGER_OZONE_PLUGIN_DIR}"/*.sh
+
+# customizations before install
+perl -wpl -i \
+ -e 's@^POLICY_MGR_URL=.*@POLICY_MGR_URL=http://ranger:6080@;' \
+ -e 's@^REPOSITORY_NAME=.*@REPOSITORY_NAME=dev_ozone@;' \
+ -e 's@^CUSTOM_USER=ozone@CUSTOM_USER=hadoop@;' \
+ "${RANGER_OZONE_PLUGIN_DIR}/install.properties"
+
+start_docker_env
+wait_for_port ranger 6080 120
+
+# execute_robot_test scm security/ozone-secure-tenant.robot
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 8888127ee9..d7a4a29e84 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -273,7 +273,7 @@ reorder_om_nodes() {
if [[ -n "${new_order}" ]] && [[ "${new_order}" != "om1,om2,om3" ]]; then
for c in $(docker-compose ps | cut -f1 -d' ' | grep -v -e '^NAME$' -e
'^om'); do
- docker exec "${c}" sh -c \
+ docker exec "${c}" bash -c \
"if [[ -f /etc/hadoop/ozone-site.xml ]]; then \
sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \
echo 'Replaced OM order with ${new_order} in ${c}'; \
@@ -289,7 +289,7 @@ create_stack_dumps() {
while read -r pid procname; do
echo "jstack $pid > ${RESULT_DIR}/${c}_${procname}.stack"
docker exec "${c}" bash -c "jstack $pid" >
"${RESULT_DIR}/${c}_${procname}.stack"
- done < <(docker exec "${c}" sh -c "jps | grep -v Jps" || true)
+ done < <(docker exec "${c}" bash -c "jps | grep -v Jps" || true)
done
}
@@ -609,3 +609,28 @@ wait_for_root_certificate(){
echo "Timed out waiting on $count root certificates. Current timestamp "
$(date +"%T")
return 1
}
+
+download_if_not_exists() {
+ local url="$1"
+ local f="$2"
+
+ if [[ -e "${f}" ]]; then
+ echo "${f} already downloaded"
+ else
+ echo "Downloading ${f} from ${url}"
+ curl --fail --location --output "${f}" --show-error --silent "${url}" ||
rm -fv "${f}"
+ fi
+}
+
+download_and_verify_apache_release() {
+ local remote_path="$1"
+
+ local f="$(basename "${remote_path}")"
+ local
base_url="${APACHE_MIRROR_URL:-https://www.apache.org/dyn/closer.lua?action=download&filename=}";
+ local
checksum_base_url="${APACHE_OFFICIAL_URL:-https://downloads.apache.org/}";
+ local download_dir="${DOWNLOAD_DIR:-/tmp}"
+
+ download_if_not_exists "${base_url}${remote_path}" "${download_dir}/${f}"
+ download_if_not_exists "${checksum_base_url}${remote_path}.asc"
"${download_dir}/${f}.asc"
+ gpg --verify "${download_dir}/${f}.asc" "${download_dir}/${f}" || exit 1
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
