This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new eadbf11270 [#6618] feat(docker): Build ranger packages with
ranger-dependency.sh (#6848)
eadbf11270 is described below
commit eadbf11270f9c3a6607642a66ca0bc2b57468847
Author: Eric Chang <[email protected]>
AuthorDate: Wed Apr 9 08:19:46 2025 +0800
[#6618] feat(docker): Build ranger packages with ranger-dependency.sh
(#6848)
### What changes were proposed in this pull request?
Currently, to build `ranger` and `hive` Docker image, we fetch ranger
tar files from datastrato's repository, but this is not allowed for an
Apache project. We need to build ranger packages from source.
Fix: #6618
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
First, execute these steps to build image:
```
# First, build hive image, since ranger packages not exists yet,
`ranger-dependency.sh` will be executed,
# and jars will be copied from `${ranger_dir}/packages` to
`${hive_dir}/packages`
./dev/docker/build-docker.sh --platform all --type hive --image <image name
and tag> --tag hive-test --latest
# Then, since ranger packages, has been built, we can execute this command
to build ranger image.
./dev/docker/build-docker.sh --platform all --type ranger --image <image
name and tag> --tag ranger-test --latest
```
And then modify `ContainerSuite.startHiveRangerContainer`,
`ContainerSuite.startRangerContainer`, add `.withImage("<image name>")`
to use newly built image.
<img width="649" alt="image"
src="https://github.com/user-attachments/assets/f1e54dc1-8c20-4702-b38f-02dbee7761e1";
/>
<img width="629" alt="image"
src="https://github.com/user-attachments/assets/52703d69-3ff5-4431-afbc-8abd5e20d31c";
/>
Finally, run tests in `RangerHiveHdfsE2EIT` to verify that image is
built successfully.
---
dev/docker/hive/download-release.sh | 59 -------------------------
dev/docker/hive/hive-dependency.sh | 11 +++--
dev/docker/ranger/.env | 79 ++++++++++++++++++++++++++++++++++
dev/docker/ranger/ranger-dependency.sh | 30 ++++++++++++-
4 files changed, 116 insertions(+), 63 deletions(-)
diff --git a/dev/docker/hive/download-release.sh
b/dev/docker/hive/download-release.sh
deleted file mode 100755
index bf03d3c35b..0000000000
--- a/dev/docker/hive/download-release.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The first parameter is the version of the release, e.g. 2.4.0
-# The second parameter is the name of the asset file, e.g.
ranger-2.4.0-admin.tar.gz
-# The third parameter is the location and name of the output file, e.g.
/tmp/ranger-2.4.0-admin.tar.gz
-
-set -ex
-
-if [ $# -ne 3 ]; then
- echo "Usage: $0 <version> <asset-file> <output-file>"
- exit 1
-fi
-
-
-TOKEN=${PRIVATE_ACCESS_TOKEN}
-REPO="datastrato/ranger"
-FILE=$2 # the name of your release asset file, e.g.
build.tar.gz
-VERSION=$1 # tag name or the word "latest"
-GITHUB="https://api.github.com";
-
-alias err_echo='>&2 echo'
-
-function gh_curl() {
- curl -H "Authorization: token $TOKEN" \
- -H "Accept: application/vnd.github.v3.raw" \
- $@
-}
-
-if [ "$VERSION" = "latest" ]; then
- # Github should return the latest release first.
- parser=".[0].assets | map(select(.name == \"$FILE\"))[0].id"
-else
- parser=". | map(select(.tag_name == \"$VERSION\"))[0].assets |
map(select(.name == \"$FILE\"))[0].id"
-fi;
-
-asset_id=`gh_curl -s $GITHUB/repos/$REPO/releases | jq "$parser"`
-if [ "$asset_id" = "null" ]; then
- err_echo "ERROR: version not found $VERSION"
- exit 1
-fi;
-
-echo $2
-
-curl -sL --header "Authorization: token $TOKEN" --header 'Accept:
application/octet-stream'
https://$TOKEN:@api.github.com/repos/$REPO/releases/assets/$asset_id -o $3
diff --git a/dev/docker/hive/hive-dependency.sh
b/dev/docker/hive/hive-dependency.sh
index ff138c5171..52967c6118 100755
--- a/dev/docker/hive/hive-dependency.sh
+++ b/dev/docker/hive/hive-dependency.sh
@@ -20,6 +20,7 @@
set -ex
hive_dir="$(dirname "${BASH_SOURCE-$0}")"
hive_dir="$(cd "${hive_dir}">/dev/null; pwd)"
+ranger_dir="${hive_dir}/../ranger"
# Environment variables definition
HADOOP2_VERSION="2.7.3"
@@ -89,7 +90,9 @@ if [ ! -f "${hive_dir}/packages/${ZOOKEEPER_PACKAGE_NAME}" ];
then
fi
if [ ! -f "${hive_dir}/packages/${RANGER_HDFS_PACKAGE_NAME}" ]; then
- bash ${hive_dir}/download-release.sh "v${RANGER_VERSION}"
${RANGER_HDFS_PACKAGE_NAME} "${hive_dir}/packages/${RANGER_HDFS_PACKAGE_NAME}"
+ # ranger-hdfs plugin not exist, run ranger-dependency.sh to build from source
+ . ${ranger_dir}/ranger-dependency.sh
+ cp "${ranger_dir}/packages/${RANGER_HDFS_PACKAGE_NAME}"
"${hive_dir}/packages"
fi
if [[ $? -ne 0 ]]; then
@@ -98,7 +101,9 @@ if [[ $? -ne 0 ]]; then
fi
if [ ! -f "${hive_dir}/packages/${RANGER_HIVE_PACKAGE_NAME}" ]; then
- bash ${hive_dir}/download-release.sh "v${RANGER_VERSION}"
${RANGER_HIVE_PACKAGE_NAME} "${hive_dir}/packages/${RANGER_HIVE_PACKAGE_NAME}"
+ # ranger-hive plugin not exist, run ranger-dependency.sh to build from source
+ . ${ranger_dir}/ranger-dependency.sh
+ cp "${ranger_dir}/packages/${RANGER_HIVE_PACKAGE_NAME}"
"${hive_dir}/packages"
fi
if [[ $? -ne 0 ]]; then
@@ -112,4 +117,4 @@ fi
if [ ! -f "${hive_dir}/packages/${HADOOP3_GCS_PACKAGE_NAME}" ]; then
curl -L -s -o "${hive_dir}/packages/${HADOOP3_GCS_PACKAGE_NAME}"
${HADOOP3_GCS_DOWNLOAD_URL}
-fi
\ No newline at end of file
+fi
diff --git a/dev/docker/ranger/.env b/dev/docker/ranger/.env
new file mode 100644
index 0000000000..35a6e9f36e
--- /dev/null
+++ b/dev/docker/ranger/.env
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BUILD_HOST_SRC=false
+SKIPTESTS=true
+GIT_URL=https://github.com/apache/ranger.git
+BRANCH=release-ranger-2.4.0
+BUILD_OPTS=-q
+
+# To build Trino plugins package, use following PROFILE
+PROFILE=ranger-jdk11,!all,!linux
+
+# Java version for RangerBase image.
+# This image gets used as base docker image for all images.
+# Valid values: 8, 11, 17
+RANGER_BASE_JAVA_VERSION=11
+
+# Java version to use to build Apache Ranger
+# Valid values: 8, 11, 17
+# Trino builds on jdk 11 and above
+RANGER_BUILD_JAVA_VERSION=11
+
+# Java version to use to run Ranger Admin server
+# Valid values: 8, 11, 17
+RANGER_ADMIN_JAVA_VERSION=11
+
+# base & third party images
+UBUNTU_VERSION=20.04
+MARIADB_VERSION=10.7.3
+POSTGRES_VERSION=12
+ENABLE_DB_MOUNT=true
+ZK_VERSION=3.5.9
+SOLR_VERSION=8
+
+# service versions
+HADOOP_VERSION=3.3.0
+HBASE_VERSION=2.4.6
+HIVE_VERSION=3.1.2
+HIVE_HADOOP_VERSION=3.1.1
+KAFKA_VERSION=2.8.1
+KNOX_VERSION=1.4.0
+TRINO_VERSION=377
+
+# versions of ranger services
+RANGER_VERSION=2.4.0
+KMS_VERSION=2.4.0
+USERSYNC_VERSION=2.4.0
+TAGSYNC_VERSION=2.4.0
+
+RANGER_VERSION=2.4.0
+KMS_VERSION=2.4.0
+USERSYNC_VERSION=2.4.0
+TAGSYNC_VERSION=2.4.0
+
+# plugin versions
+HDFS_PLUGIN_VERSION=2.4.0
+YARN_PLUGIN_VERSION=2.4.0
+HIVE_PLUGIN_VERSION=2.4.0
+HBASE_PLUGIN_VERSION=2.4.0
+KAFKA_PLUGIN_VERSION=2.4.0
+KNOX_PLUGIN_VERSION=2.4.0
+TRINO_PLUGIN_VERSION=2.4.0
+
+# To enable debug logs
+DEBUG_ADMIN=true
+DEBUG_USERSYNC=true
+DEBUG_TAGSYNC=true
diff --git a/dev/docker/ranger/ranger-dependency.sh
b/dev/docker/ranger/ranger-dependency.sh
index 7582c160f5..6a77dbc4e5 100755
--- a/dev/docker/ranger/ranger-dependency.sh
+++ b/dev/docker/ranger/ranger-dependency.sh
@@ -36,7 +36,35 @@ if [[ ! -d "${ranger_dir}/packages" ]]; then
fi
if [ ! -f "${ranger_dir}/packages/${RANGER_PACKAGE_NAME}" ]; then
- curl -L -s -o "${ranger_dir}/packages/${RANGER_PACKAGE_NAME}"
${RANGER_DOWNLOAD_URL}
+ # Package not exist, we need to build them from source
+ if [ ! -d "${ranger_dir}/packages/apache-ranger" ]; then
+ git clone https://github.com/apache/ranger --branch master --single-branch
${ranger_dir}/packages/apache-ranger
+ # set the commit to RANGER-5146: 500 API Error When Deleting TagDef with a
Linked Tag
+ #
https://github.com/apache/ranger/commit/ff36aabe36169b94862c51a5b403f59c9d728b94
+ git reset --hard ff36aabe36169b94862c51a5b403f59c9d728b94
+ fi
+
+ cp ${ranger_dir}/.env
${ranger_dir}/packages/apache-ranger/dev-support/ranger-docker
+ cd ${ranger_dir}/packages/apache-ranger/dev-support/ranger-docker
+
+ # Prevent builder to pull remote image
+ # https://github.com/moby/buildkit/issues/2343#issuecomment-1311890308
+ docker builder prune -f
+ docker builder use default
+
+ export DOCKER_BUILDKIT=1
+ export COMPOSE_DOCKER_CLI_BUILD=1
+ export RANGER_DB_TYPE=mysql
+
+ # run docker compose command to build packages
+ docker compose -f docker-compose.ranger-base.yml -f
docker-compose.ranger-build.yml up --pull=never
+ # copy packages from volume to host
+ docker compose -f docker-compose.ranger-base.yml -f
docker-compose.ranger-build.yml cp ranger-build:/home/ranger/dist .
+
+ cp ./dist/* ${ranger_dir}/packages
+
+ # change back to gravitino-builder
+ docker builder use gravitino-builder
fi
if [ ! -f "${ranger_dir}/packages/${MYSQL_CONNECTOR_PACKAGE_NAME}" ]; then
