This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch ranger-2.1 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 5a16f9afffc0ba5cc3f9e16e23af8e67090112d4 Author: Madhan Neethiraj <[email protected]> AuthorDate: Wed Aug 26 19:24:05 2020 -0700 RANGER-2971: Docker setup to run Ranger enabled HBase (standalone) (cherry picked from commit f5bcb8232ef27cd5b3dfb3fc5a55a89787537347) --- dev-support/ranger-docker/.dockerignore | 1 + dev-support/ranger-docker/Dockerfile.ranger | 52 +++++--------- .../{Dockerfile.ranger => Dockerfile.ranger-base} | 32 +++++---- dev-support/ranger-docker/Dockerfile.ranger-build | 26 ++----- dev-support/ranger-docker/Dockerfile.ranger-hadoop | 41 +---------- dev-support/ranger-docker/Dockerfile.ranger-hbase | 38 +++++++++++ dev-support/ranger-docker/README.md | 45 +++++++----- .../ranger-docker/docker-compose.ranger-base.yml | 12 ++++ .../ranger-docker/docker-compose.ranger-build.yml | 2 + .../ranger-docker/docker-compose.ranger-hbase.yml | 22 ++++++ .../ranger-docker/docker-compose.ranger.yml | 1 + dev-support/ranger-docker/scripts/ranger-hadoop.sh | 14 ---- .../scripts/ranger-hbase-plugin-install.properties | 79 ++++++++++++++++++++++ .../scripts/ranger-hbase-service-dev_hbase.py | 8 +++ .../ranger-docker/scripts/ranger-hbase-setup.sh | 30 ++++++++ .../scripts/{ranger-hadoop.sh => ranger-hbase.sh} | 31 ++------- dev-support/ranger-docker/scripts/ranger.sh | 23 ++----- 17 files changed, 278 insertions(+), 179 deletions(-) diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore index 5a236e9..3ffb780 100644 --- a/dev-support/ranger-docker/.dockerignore +++ b/dev-support/ranger-docker/.dockerignore @@ -4,4 +4,5 @@ !dist/ranger-*-admin.tar.gz !dist/ranger-*-hdfs-plugin.tar.gz !dist/ranger-*-hive-plugin.tar.gz +!dist/ranger-*-hbase-plugin.tar.gz !scripts/* diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index d57e384..fca32ae 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -14,43 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 +FROM ranger-base:latest -ENV RANGER_VERSION 2.1.0 -# Install curl, wget, tzdata, Python, Java, python-requests -RUN apt-get update && \ - DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping && \ - curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \ - python2 /tmp/get-pip.py && \ - pip3 install requests && \ - pip3 install apache-ranger && \ - pip install requests +COPY ./dist/version ${RANGER_DIST}/ +COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/ +COPY ./scripts/ranger-admin-install.properties ${RANGER_SCRIPTS}/ +COPY ./scripts/ranger-hdfs-service-dev_hdfs.py ${RANGER_SCRIPTS}/ +COPY ./scripts/ranger-hive-service-dev_hive.py ${RANGER_SCRIPTS}/ +COPY ./scripts/ranger-hbase-service-dev_hbase.py ${RANGER_SCRIPTS}/ -# Set environment variables -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 -ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ENV RANGER_DIST /home/ranger/dist -ENV RANGER_SCRIPTS /home/ranger/scripts -ENV RANGER_HOME /opt/ranger - -# setup ranger group, and users -RUN groupadd ranger && \ - useradd -g ranger -ms /bin/bash ranger && \ - useradd -g ranger -ms /bin/bash rangeradmin && \ - useradd -g ranger -ms /bin/bash rangerusersync && \ - useradd -g ranger -ms /bin/bash rangertagsync && \ - useradd -g ranger -ms /bin/bash rangerkms && \ - mkdir -p /home/ranger/dist && \ - mkdir -p /home/ranger/scripts && \ - mkdir -p /opt/ranger && \ - chown -R ranger:ranger /opt/ranger - -COPY ./dist/version /home/ranger/dist/ -COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/ -COPY ./scripts/ranger.sh /home/ranger/scripts/ -COPY ./scripts/ranger-admin-install.properties /home/ranger/scripts/ -COPY ./scripts/ranger-hdfs-service-dev_hdfs.py /home/ranger/scripts/ -COPY ./scripts/ranger-hive-service-dev_hive.py /home/ranger/scripts/ +COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /tmp/ +RUN tar xvfz /tmp/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} && \ + ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin && \ + rm -f /tmp/ranger-${RANGER_VERSION}-admin.tar.gz && \ + cp -f ${RANGER_SCRIPTS}/ranger-admin-install.properties ${RANGER_HOME}/admin/install.properties && \ + mkdir -p /var/run/ranger && \ + mkdir -p /var/log/ranger && \ + chown -R ranger:ranger ${RANGER_HOME}/admin/ /var/run/ranger/ /var/log/ranger/ && \ + mkdir -p /usr/share/java/ && \ + wget "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7/postgresql-42.2.16.jre7.jar"; -O /usr/share/java/postgresql.jar ENTRYPOINT [ "/home/ranger/scripts/ranger.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger-base similarity index 76% copy from dev-support/ranger-docker/Dockerfile.ranger copy to dev-support/ranger-docker/Dockerfile.ranger-base index d57e384..f461f74 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger-base @@ -16,41 +16,45 @@ FROM ubuntu:20.04 + ENV RANGER_VERSION 2.1.0 +ENV HADOOP_VERSION 3.1.1 +ENV HIVE_VERSION 3.1.2 +ENV HBASE_VERSION 2.0.3 # Install curl, wget, tzdata, Python, Java, python-requests RUN apt-get update && \ - DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping && \ + DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \ + python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \ curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \ python2 /tmp/get-pip.py && \ - pip3 install requests && \ pip3 install apache-ranger && \ + pip3 install requests && \ pip install requests # Set environment variables ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 -ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV RANGER_DIST /home/ranger/dist ENV RANGER_SCRIPTS /home/ranger/scripts ENV RANGER_HOME /opt/ranger +ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + -# setup ranger group, and users +# setup groups, users, directories RUN groupadd ranger && \ useradd -g ranger -ms /bin/bash ranger && \ useradd -g ranger -ms /bin/bash rangeradmin && \ useradd -g ranger -ms /bin/bash rangerusersync && \ useradd -g ranger -ms /bin/bash rangertagsync && \ useradd -g ranger -ms /bin/bash rangerkms && \ + groupadd hadoop && \ + useradd -g hadoop -ms /bin/bash hdfs && \ + useradd -g hadoop -ms /bin/bash hive && \ + useradd -g hadoop -ms /bin/bash hbase && \ mkdir -p /home/ranger/dist && \ mkdir -p /home/ranger/scripts && \ - mkdir -p /opt/ranger && \ - chown -R ranger:ranger /opt/ranger - -COPY ./dist/version /home/ranger/dist/ -COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/ -COPY ./scripts/ranger.sh /home/ranger/scripts/ -COPY ./scripts/ranger-admin-install.properties /home/ranger/scripts/ -COPY ./scripts/ranger-hdfs-service-dev_hdfs.py /home/ranger/scripts/ -COPY ./scripts/ranger-hive-service-dev_hive.py /home/ranger/scripts/ + chown -R ranger:ranger /home/ranger && \ + mkdir -p /opt/ranger && \ + chown -R ranger:ranger /opt/ranger -ENTRYPOINT [ "/home/ranger/scripts/ranger.sh" ] +ENTRYPOINT [ "/bin/bash" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-build b/dev-support/ranger-docker/Dockerfile.ranger-build index c5a11a0..56c6d5d 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-build +++ b/dev-support/ranger-docker/Dockerfile.ranger-build @@ -14,30 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 +FROM ranger-base:latest -# Install curl, wget, tzdata, Python, Java, python-requests -RUN apt-get update && \ - DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \ - python python3 python3-pip openjdk-8-jdk bc iputils-ping git maven build-essential && \ - curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \ - python2 /tmp/get-pip.py && \ - pip3 install requests && \ - pip install requests + +# Install necessary packages to build Ranger +RUN apt-get update && apt-get -y install git maven build-essential # Set environment variables -ENV MAVEN_HOME /usr/share/maven -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 -ENV PATH /usr/java/bin:/usr/local/apache-maven/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ENV RANGER_DIST /home/ranger/dist -ENV RANGER_SCRIPTS /home/ranger/scripts +ENV MAVEN_HOME /usr/share/maven +ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/apache-maven/bin # setup ranger group, and users -RUN groupadd ranger && \ - useradd -g ranger -ms /bin/bash ranger && \ - mkdir -p /home/ranger/dist && \ - mkdir -p /home/ranger/scripts && \ - mkdir -p /home/ranger/git && \ +RUN mkdir -p /home/ranger/git && \ mkdir -p /home/ranger/.m2 && \ chown -R ranger:ranger /home/ranger diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop index e866ba7..d6046af 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop +++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop @@ -14,26 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 - -ENV RANGER_VERSION 2.1.0 -ENV HADOOP_VERSION 3.1.1 -ENV HIVE_VERSION 3.1.2 - -# Install curl, wget, tzdata, Python, Java, python-requests -RUN apt-get update && \ - DEBIAN_FRONTEND="noninteractive" apt-get -y install vim sudo curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \ - curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \ - python2 /tmp/get-pip.py && \ - pip3 install requests && \ - pip install requests - -RUN groupadd hadoop && \ - useradd -g hadoop -ms /bin/bash hdfs && \ - useradd -g hadoop -ms /bin/bash hive && \ - mkdir -p /opt/ranger && \ - mkdir -p /home/ranger/dist && \ - mkdir -p /home/ranger/scripts +FROM ranger-base:latest COPY ./dist/version /home/ranger/dist/ @@ -51,30 +32,12 @@ RUN curl https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/ rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-hdfs-plugin-install.properties /opt/ranger/ranger-hdfs-plugin/install.properties -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 -ENV RANGER_DIST /home/ranger/dist -ENV RANGER_SCRIPTS /home/ranger/scripts -ENV RANGER_HOME /opt/ranger - ENV HADOOP_HOME /opt/hadoop ENV HADOOP_CONF_DIR /opt/hadoop/etc/hadoop ENV HADOOP_HDFS_HOME /opt/hadoop ENV HADOOP_MAPRED_HOME /opt/hadoop ENV HADOOP_COMMON_HOME /opt/hadoop ENV YARN_HOME /opt/hadoop -ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin - -# COPY ./dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz /home/ranger/dist/ -# -# RUN curl https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz --output /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && -# tar xvfz /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \ -# ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \ -# rm -f /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ -# tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \ -# ln -s /opt/ranger/ranger-${RANGER_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \ -# rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz -# ENV HIVE_HOME /opt/hive -# ENV HIVE_CONF_DIR /opt/hive/conf -# ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin:/opt/hive/bin +ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin ENTRYPOINT [ "/home/ranger/scripts/ranger-hadoop.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase new file mode 100644 index 0000000..a995250 --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ranger-base:latest + + +COPY ./dist/version /home/ranger/dist/ +COPY ./dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz /home/ranger/dist/ +COPY ./scripts/ranger-hbase-setup.sh /home/ranger/scripts/ +COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/ +COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/ + +RUN curl https://archive.apache.org/dist/hbase/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz --output /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \ + tar xvfz /tmp/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \ + ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \ + rm -f /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \ + tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz --directory=/opt/ranger && \ + ln -s /opt/ranger/ranger-${RANGER_VERSION}-hbase-plugin /opt/ranger/ranger-hbase-plugin && \ + rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz && \ + cp -f /home/ranger/scripts/ranger-hbase-plugin-install.properties /opt/ranger/ranger-hbase-plugin/install.properties + +ENV HBASE_HOME /opt/hbase +ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hbase/bin + +ENTRYPOINT [ "/home/ranger/scripts/ranger-hbase.sh" ] diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md index 938d8ac..0fad420 100644 --- a/dev-support/ranger-docker/README.md +++ b/dev-support/ranger-docker/README.md @@ -34,55 +34,64 @@ deploy Apache Ranger and its dependent services in containers. 3. Using docker-compose is the simpler way to build and deploy Apache Ranger in containers. 3.1. Execute following command to build Apache Ranger: - docker-compose -f docker-compose.ranger-build.yml up + docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up --remove-orphans Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache. - 3.2. Execute following command to start Ranger and dependent services in containers: - docker-compose -f docker-compose.ranger.yml up -d - - 3.2. Execute following command to start Ranger enabled Hadoop services (only HDFS for now) in a continer: - docker-compose -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml up -d - + 3.2. Execute following command to start Ranger, Ranger enabled HDFS, Ranger enabled HBase, and dependeny services (Solr, DB) in continers: + docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml up -f docker-compose.ranger-hbase.yml -d 4. Alternatively docker command can be used to build and deploy Apache Ranger. - 4.1. Execute following command to build Docker image **ranger-build**: - docker build -f Dockerfile.ranger-build -t ranger-build . + 4.1. Execute following command to build Docker image **ranger-base**: + docker build -f Dockerfile.ranger-base -t ranger-base . This might take about 10 minutes to complete. - 4.2. Build Apache Ranger in a container with the following command: + 4.2. Execute following command to build Docker image **ranger-build**: + docker build -f Dockerfile.ranger-build -t ranger-build . + + 4.3. Build Apache Ranger in a container with the following command: docker run -it --rm -v ${HOME}/.m2:/home/ranger/.m2 -v $(pwd)/dist:/home/ranger/dist -e BRANCH=ranger-2.1 -e PROFILE=all -e SKIPTESTS=true ranger-build Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache. - 4.3. Execute following command to build Docker image **ranger**: + 4.4. Execute following command to build Docker image **ranger**: docker build -f Dockerfile.ranger -t ranger . This might take about 10 minutes to complete. - 4.4. Execute following command to build a Docker image **ranger-solr**: + 4.5. Execute following command to build a Docker image **ranger-solr**: docker build -f Dockerfile.ranger-solr -t ranger-solr . - 4.5. Execute following command to start a container that runs database for use by Ranger Admin: + 4.6. Execute following command to start a container that runs database for use by Ranger Admin: docker run --name ranger-db --hostname ranger-db.example.com -e POSTGRES_PASSWORD='rangerR0cks!' -d postgres:12 - 4.6. Execute following command to start a container that runs Solr for use by Ranger Admin: + 4.7. Execute following command to start a container that runs Solr for use by Ranger Admin: docker run --name ranger-solr --hostname ranger-solr.example.com -p 8983:8983 -d ranger-solr solr-precreate ranger_audits /opt/solr/server/solr/configsets/ranger_audits/ - 4.7. Execute following command to install and run Ranger services in a container: + 4.8. Execute following command to install and run Ranger services in a container: docker run -it -d --name ranger --hostname ranger.example.com -p 6080:6080 --link ranger-db:ranger-db --link ranger-solr:ranger-solr ranger This might take few minutes to complete. - 4.8. Execute following command to build Docker image **ranger-hadoop**: + 4.9. Execute following command to build Docker image **ranger-hadoop**: docker build -f Dockerfile.ranger-hadoop -t ranger-hadoop . This steps includes downloading of Hadoop tar balls, and can take a while to complete. - 4.9. Execute following command to install and run Ranger enabled Hadoop services (only HDFS for now) in a container: - docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop + 4.10. Execute following command to install and run Ranger enabled HDFS in a container: + docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop + + This might take few minutes to complete. + + 4.11. Execute following command to build Docker image **ranger-hbase**: + docker build -f Dockerfile.ranger-hbase -t ranger-hbase . + + This steps includes downloading of HBase tar ball, and can take a while to complete. + + 4.12. Execute following command to install and run Ranger enabled HBase in a container: + docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr ranger-hbase This might take few minutes to complete. diff --git a/dev-support/ranger-docker/docker-compose.ranger-base.yml b/dev-support/ranger-docker/docker-compose.ranger-base.yml new file mode 100644 index 0000000..18e78db --- /dev/null +++ b/dev-support/ranger-docker/docker-compose.ranger-base.yml @@ -0,0 +1,12 @@ +version: '3' +services: + ranger-base: + build: + context: . + dockerfile: Dockerfile.ranger-base + image: ranger-base + networks: + - ranger + +networks: + ranger: diff --git a/dev-support/ranger-docker/docker-compose.ranger-build.yml b/dev-support/ranger-docker/docker-compose.ranger-build.yml index a63b3dd..48decd5 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-build.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-build.yml @@ -12,6 +12,8 @@ services: volumes: - ~/.m2:/home/ranger/.m2 - ./dist:/home/ranger/dist + depends_on: + - ranger-base environment: BRANCH: 'ranger-2.1' SKIPTESTS: 'true' diff --git a/dev-support/ranger-docker/docker-compose.ranger-hbase.yml b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml new file mode 100644 index 0000000..81804aa --- /dev/null +++ b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml @@ -0,0 +1,22 @@ +version: '3' +services: + ranger-hbase: + build: + context: . + dockerfile: Dockerfile.ranger-hbase + image: ranger-hbase + container_name: ranger-hbase + hostname: ranger-hbase.example.com + stdin_open: true + tty: true + networks: + - ranger + ports: + - "2181:16181" + - "16010:16010" + - "16020:16020" + depends_on: + - ranger-hadoop + +networks: + ranger: diff --git a/dev-support/ranger-docker/docker-compose.ranger.yml b/dev-support/ranger-docker/docker-compose.ranger.yml index 2c6a3c9..787c2ef 100644 --- a/dev-support/ranger-docker/docker-compose.ranger.yml +++ b/dev-support/ranger-docker/docker-compose.ranger.yml @@ -14,6 +14,7 @@ services: ports: - "6080:6080" depends_on: + - ranger-base - ranger-db - ranger-solr command: diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh index 9d7ebf0..8dc5cd4 100755 --- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh +++ b/dev-support/ranger-docker/scripts/ranger-hadoop.sh @@ -36,19 +36,5 @@ fi su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs su -c "${HADOOP_HOME}/sbin/start-yarn.sh" hdfs -# if [ ! -e ${HIVE_HOME}/.setupDone ] -# then -# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp" hdfs -# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /user/hive/warehouse" hdfs -# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /tmp" hdfs -# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /user/hive/warehouse" hdfs -# -# su -c "${HIVE_HOME}/bin/schematool -dbType postgres -initSchema" hive -# -# touch ${HIVE_HOME}/.setupDone -# fi -# -# su -c "${HIVE_HOME}/bin/hiveserver2" hive - # prevent the container from exiting /bin/bash diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties new file mode 100644 index 0000000..bc80a6d --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +POLICY_MGR_URL=http://ranger:6080 +REPOSITORY_NAME=dev_hbase +COMPONENT_INSTALL_DIR_NAME=/opt/hbase + +CUSTOM_USER=hbase +CUSTOM_GROUP=hadoop + +XAAUDIT.SUMMARY.ENABLE=true +UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true + +XAAUDIT.SOLR.IS_ENABLED=true +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits + +# Following properties are needed to get past installation script! Please don't remove +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit +XAAUDIT.HDFS.DESTINTATION_FILE=hadoop +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hadoop/hbase/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hadoop/hbase/audit/archive +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +XAAUDIT.SOLR.ENABLE=true +XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.USER=NONE +XAAUDIT.SOLR.PASSWORD=NONE +XAAUDIT.SOLR.ZOOKEEPER=NONE +XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/hbase/audit/solr/spool + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.HDFS.ENABLE=false +XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit +XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hbase/audit/hdfs/spool + +XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME +XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY +XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER +XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER + +XAAUDIT.LOG4J.ENABLE=false +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit + +SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py new file mode 100644 index 0000000..9294bf5 --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py @@ -0,0 +1,8 @@ +from apache_ranger.model.ranger_service import RangerService +from apache_ranger.client.ranger_client import RangerClient + +ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!') + +service = RangerService(name='dev_hbase', type='hbase', configs={'username':'hbase', 'password':'hbase', 'hadoop.security.authentication': 'simple', 'hbase.security.authentication': 'simple', 'hadoop.security.authorization': 'true', 'hbase.zookeeper.property.clientPort': '16181', 'hbase.zookeeper.quorum': 'ranger-hbase', 'zookeeper.znode.parent': '/hbase'}) + +ranger_client.create_service(service) diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh new file mode 100755 index 0000000..bc6d226 --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "export JAVA_HOME=${JAVA_HOME}" >> ${HBASE_HOME}/conf/hbase-env.sh + +cat <<EOF > /etc/ssh/ssh_config +Host * + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null +EOF + +chown -R hbase:hadoop /opt/hbase/ + +cd ${RANGER_HOME}/ranger-hbase-plugin +./enable-hbase-plugin.sh diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hbase.sh similarity index 50% copy from dev-support/ranger-docker/scripts/ranger-hadoop.sh copy to dev-support/ranger-docker/scripts/ranger-hbase.sh index 9d7ebf0..7bca8f7 100755 --- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh +++ b/dev-support/ranger-docker/scripts/ranger-hbase.sh @@ -18,37 +18,20 @@ service ssh start -if [ ! -e ${HADOOP_HOME}/.setupDone ] +if [ ! -e ${HBASE_HOME}/.setupDone ] then - su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hdfs - su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hdfs - su -c "chmod 0600 ~/.ssh/authorized_keys" hdfs + su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hbase + su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hbase + su -c "chmod 0600 ~/.ssh/authorized_keys" hbase echo "ssh" > /etc/pdsh/rcmd_default - ${RANGER_SCRIPTS}/ranger-hadoop-setup.sh + ${RANGER_SCRIPTS}/ranger-hbase-setup.sh - su -c "${HADOOP_HOME}/bin/hdfs namenode -format" hdfs - - touch ${HADOOP_HOME}/.setupDone + touch ${HBASE_HOME}/.setupDone fi -su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs -su -c "${HADOOP_HOME}/sbin/start-yarn.sh" hdfs - -# if [ ! -e ${HIVE_HOME}/.setupDone ] -# then -# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp" hdfs -# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /user/hive/warehouse" hdfs -# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /tmp" hdfs -# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /user/hive/warehouse" hdfs -# -# su -c "${HIVE_HOME}/bin/schematool -dbType postgres -initSchema" hive -# -# touch ${HIVE_HOME}/.setupDone -# fi -# -# su -c "${HIVE_HOME}/bin/hiveserver2" hive +su -c "${HBASE_HOME}/bin/start-hbase.sh" hbase # prevent the container from exiting /bin/bash diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger.sh index bf61968..3076556 100755 --- a/dev-support/ranger-docker/scripts/ranger.sh +++ b/dev-support/ranger-docker/scripts/ranger.sh @@ -16,32 +16,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -export RANGER_VERSION=`cat ${RANGER_DIST}/version` - -if [ -e ${RANGER_HOME}/admin ] +if [ ! -e ${RANGER_HOME}/.setupDone ] then - SETUP_RANGER=false -else SETUP_RANGER=true +else + SETUP_RANGER=false fi if [ "${SETUP_RANGER}" == "true" ] then - # Download PostgreSQL JDBC library - wget "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7/postgresql-42.2.16.jre7.jar"; -O /usr/share/java/postgresql.jar - - cd ${RANGER_HOME} - tar xvfz ${RANGER_DIST}/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} - ln -s ranger-${RANGER_VERSION}-admin admin - cp -f ${RANGER_SCRIPTS}/ranger-admin-install.properties admin/install.properties + su -c "cd ${RANGER_HOME}/admin && ./setup.sh" ranger - cd ${RANGER_HOME}/admin - ./setup.sh + touch ${RANGER_HOME}/.setupDone fi -cd ${RANGER_HOME}/admin -./ews/ranger-admin-services.sh start +su -c "cd ${RANGER_HOME}/admin && ./ews/ranger-admin-services.sh start" ranger if [ "${SETUP_RANGER}" == "true" ] then @@ -50,6 +40,7 @@ then python3 ${RANGER_SCRIPTS}/ranger-hdfs-service-dev_hdfs.py python3 ${RANGER_SCRIPTS}/ranger-hive-service-dev_hive.py + python3 ${RANGER_SCRIPTS}/ranger-hbase-service-dev_hbase.py fi # prevent the container from exiting
