This is an automated email from the ASF dual-hosted git repository.
abhi pushed a commit to branch ranger-2.6
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.6 by this push:
new 0b5602c0d5 RANGER-5080: Add docker support for MS SQL Server database
(#483)
0b5602c0d5 is described below
commit 0b5602c0d578781d5b864cc409e5efd31592c7ab
Author: Abhishek Kumar <[email protected]>
AuthorDate: Fri Jan 24 16:40:53 2025 -0800
RANGER-5080: Add docker support for MS SQL Server database (#483)
(cherry picked from commit bc4c95cddcccfe60f9d8c91b5917856406cc2c73)
---
.github/workflows/maven.yml | 5 +-
dev-support/ranger-docker/.env | 1 +
dev-support/ranger-docker/Dockerfile.ranger | 4 +
dev-support/ranger-docker/Dockerfile.ranger-kms | 4 +
.../ranger-docker/Dockerfile.ranger-sqlserver | 35 ++++
dev-support/ranger-docker/config/init_mssql.sh | 75 +++++++
.../docker-compose.ranger-sqlserver.yml | 25 +++
dev-support/ranger-docker/download-archives.sh | 1 +
.../ranger-docker/scripts/hive-site-sqlserver.xml | 50 +++++
.../ranger-admin-install-sqlserver.properties | 99 +++++++++
.../ranger-kms-install-sqlserver.properties | 223 +++++++++++++++++++++
kms/scripts/dba_script.py | 24 ++-
kms/scripts/setup.sh | 9 +-
security-admin/scripts/dba_script.py | 25 ++-
security-admin/scripts/setup.sh | 9 +-
15 files changed, 576 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index f24342e993..b556905e58 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -105,7 +105,7 @@ jobs:
run: |
cp ranger-*.tar.gz dev-support/ranger-docker/dist
cp version dev-support/ranger-docker/dist
-
+
- name: Cache downloaded archives
uses: actions/cache@v4
with:
@@ -159,8 +159,9 @@ jobs:
-f docker-compose.ranger-hive.yml \
-f docker-compose.ranger-knox.yml \
-f docker-compose.ranger-ozone.yml up -d
+
- name: Check status of containers and remove them
- run: |
+ run: |
sleep 60
containers=(ranger ranger-zk ranger-solr ranger-postgres
ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase
ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
flag=true;
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
index b88e29c43d..4d998bf8ef 100644
--- a/dev-support/ranger-docker/.env
+++ b/dev-support/ranger-docker/.env
@@ -36,6 +36,7 @@ UBI_VERSION=latest
MARIADB_VERSION=10.7.3
POSTGRES_VERSION=12
ORACLE_VERSION=23.6
+SQLSERVER_VERSION=2019-latest
ENABLE_DB_MOUNT=true
ZK_VERSION=3.9.2
SOLR_VERSION=8.11.3
diff --git a/dev-support/ranger-docker/Dockerfile.ranger
b/dev-support/ranger-docker/Dockerfile.ranger
index f5081ff57d..6603b731db 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger
@@ -59,6 +59,10 @@ FROM ranger AS ranger_oracle
COPY ./downloads/ojdbc8.jar /home/ranger/dist/
RUN mv /home/ranger/dist/ojdbc8.jar /usr/share/java/oracle.jar
+FROM ranger AS ranger_sqlserver
+COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/
+RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar
/usr/share/java/mssql.jar
+
FROM ranger_${RANGER_DB_TYPE}
USER ranger
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms
b/dev-support/ranger-docker/Dockerfile.ranger-kms
index 5e70d0da53..be85bbcff8 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-kms
+++ b/dev-support/ranger-docker/Dockerfile.ranger-kms
@@ -58,6 +58,10 @@ FROM ranger-kms AS ranger_oracle
COPY ./downloads/ojdbc8.jar /home/ranger/dist/
RUN mv /home/ranger/dist/ojdbc8.jar /usr/share/java/oracle.jar
+FROM ranger-kms AS ranger_sqlserver
+COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/
+RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar
/usr/share/java/mssql.jar
+
FROM ranger_${RANGER_DB_TYPE}
ENTRYPOINT [ "/home/ranger/scripts/ranger-kms.sh" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-sqlserver
b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver
new file mode 100644
index 0000000000..3559fa49dd
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG SQLSERVER_VERSION
+
+FROM mcr.microsoft.com/mssql/server:${SQLSERVER_VERSION}
+
+ENV ACCEPT_EULA=Y
+ENV MSSQL_SA_PASSWORD=rangerR0cks!
+
+EXPOSE 1433
+
+USER root
+
+RUN mkdir -p /docker-entrypoint-initdb.d
+COPY config/init_mssql.sh /docker-entrypoint-initdb.d/
+RUN chown -R mssql /docker-entrypoint-initdb.d/
+RUN chmod +x /docker-entrypoint-initdb.d/init_mssql.sh
+
+USER mssql
+
+ENTRYPOINT ["/docker-entrypoint-initdb.d/init_mssql.sh"]
diff --git a/dev-support/ranger-docker/config/init_mssql.sh
b/dev-support/ranger-docker/config/init_mssql.sh
new file mode 100644
index 0000000000..32036c9eae
--- /dev/null
+++ b/dev-support/ranger-docker/config/init_mssql.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+/opt/mssql/bin/sqlservr &
+
+# Wait for SQL Server to be ready
+echo "Waiting for SQL Server to start..."
+RETRIES=30 # Number of retries
+SLEEP_INTERVAL=5 # Seconds to wait between retries
+for i in $(seq 1 $RETRIES); do
+ # Try to connect to SQL Server
+ /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P "rangerR0cks!" -Q
"SELECT 1" -C > /dev/null 2>&1
+ if [ $? -eq 0 ]; then
+ echo "SQL Server is ready!"
+ break
+ else
+ echo "SQL Server is not ready yet. Waiting..."
+ sleep $SLEEP_INTERVAL
+ fi
+done
+
+if [ $i -eq $RETRIES ]; then
+ echo "SQL Server did not become ready in time. Exiting."
+ exit 1
+fi
+
+/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q "
+
+-- Set the database context
+USE master;
+
+-- Create databases
+CREATE DATABASE ranger;
+CREATE DATABASE rangerkms;
+CREATE DATABASE hive;
+GO
+
+-- Create users and assign permissions
+USE ranger;
+CREATE LOGIN rangeradmin WITH PASSWORD = 'rangerR0cks!';
+CREATE USER rangeradmin FOR LOGIN rangeradmin;
+ALTER ROLE db_owner ADD MEMBER rangeradmin; -- Grant equivalent high-level
permissions
+GO
+
+USE rangerkms;
+CREATE LOGIN rangerkms WITH PASSWORD = 'rangerR0cks!';
+CREATE USER rangerkms FOR LOGIN rangerkms;
+ALTER ROLE db_owner ADD MEMBER rangerkms; -- Grant equivalent high-level
permissions
+GO
+
+USE hive;
+CREATE LOGIN hive WITH PASSWORD = 'rangerR0cks!';
+CREATE USER hive FOR LOGIN hive;
+ALTER ROLE db_owner ADD MEMBER hive; -- Grant equivalent high-level permissions
+GO
+" -C
+
+# Bring SQL Server to the foreground
+wait -n
+exec /opt/mssql/bin/sqlservr
diff --git a/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml
b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml
new file mode 100644
index 0000000000..f1e790019f
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml
@@ -0,0 +1,25 @@
+services:
+ ranger-db:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-sqlserver
+ args:
+ - SQLSERVER_VERSION=${SQLSERVER_VERSION}
+ image: ranger-sqlserver
+ container_name: ranger-sqlserver
+ hostname: ranger-db.example.com
+ networks:
+ - ranger
+ healthcheck:
+ test: [
+ "CMD-SHELL",
+ "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P rangerR0cks! -Q
\"SELECT 1\" -C" # -C bypasses SSL validation
+ ]
+ interval: 15s
+ timeout: 10s
+ retries: 3
+ start_period: 10s
+
+networks:
+ ranger:
+ name: rangernw
diff --git a/dev-support/ranger-docker/download-archives.sh
b/dev-support/ranger-docker/download-archives.sh
index e7a4bf217b..0bfca41ade 100755
--- a/dev-support/ranger-docker/download-archives.sh
+++ b/dev-support/ranger-docker/download-archives.sh
@@ -44,6 +44,7 @@ downloadIfNotPresent() {
downloadIfNotPresent postgresql-42.2.16.jre7.jar
"https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7";
downloadIfNotPresent mysql-connector-java-8.0.28.jar
"https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28";
downloadIfNotPresent ojdbc8.jar
https://download.oracle.com/otn-pub/otn_software/jdbc/236
+downloadIfNotPresent mssql-jdbc-12.8.1.jre8.jar
https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/12.8.1.jre8
downloadIfNotPresent log4jdbc-1.2.jar
https://repo1.maven.org/maven2/com/googlecode/log4jdbc/log4jdbc/1.2
if [[ $# -eq 0 ]]
diff --git a/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml
b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml
new file mode 100644
index 0000000000..5bed21c2d8
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:sqlserver://ranger-db/hive</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.microsoft.sqlserver.jdbc.SQLServerDriver</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>hive</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>rangerR0cks!</value>
+ </property>
+ <property>
+ <name>hive.server2.enable.doAs</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hive.zookeeper.quorum</name>
+ <value>ranger-zk.example.com</value>
+ </property>
+ <property>
+ <name>hive.zookeeper.client.port</name>
+ <value>2181</value>
+ </property>
+</configuration>
diff --git
a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties
b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties
new file mode 100644
index 0000000000..b69e22d1e4
--- /dev/null
+++
b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This file provides a list of the deployment variables for the Policy Manager
Web Application
+#
+
+PYTHON_COMMAND_INVOKER=python3
+RANGER_ADMIN_LOG_DIR=/var/log/ranger
+RANGER_PID_DIR_PATH=/var/run/ranger
+DB_FLAVOR=MSSQL
+SQL_CONNECTOR_JAR=/usr/share/java/mssql.jar
+CONNECTION_STRING_ADDITIONAL_PARAMS="trustServerCertificate=true;"
+RANGER_ADMIN_LOGBACK_CONF_FILE=/opt/ranger/admin/ews/webapp/WEB-INF/classes/conf/logback.xml
+
+db_root_user=sa
+db_root_password=rangerR0cks!
+db_host=ranger-db
+
+db_name=ranger
+db_user=rangeradmin
+db_password=rangerR0cks!
+
+postgres_core_file=db/postgres/optimized/current/ranger_core_db_postgres.sql
+postgres_audit_file=db/postgres/xa_audit_db_postgres.sql
+mysql_core_file=db/mysql/optimized/current/ranger_core_db_mysql.sql
+mysql_audit_file=db/mysql/xa_audit_db.sql
+oracle_core_file=db/oracle/optimized/current/ranger_core_db_oracle.sql
+oracle_audit_file=db/oracle/xa_audit_db_oracle.sql
+sqlserver_core_file=db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
+sqlserver_audit_file=db/sqlserver/xa_audit_db_sqlserver.sql
+
+# For over-riding the jdbc url
+is_override_db_connection_string=true
+db_override_jdbc_connection_string="jdbc:sqlserver://ranger-db;databaseName=ranger;trustServerCertificate=true;"
+
+rangerAdmin_password=rangerR0cks!
+rangerTagsync_password=rangerR0cks!
+rangerUsersync_password=rangerR0cks!
+keyadmin_password=rangerR0cks!
+
+
+audit_store=solr
+audit_solr_urls=http://ranger-solr:8983/solr/ranger_audits
+audit_solr_collection_name=ranger_audits
+
+# audit_store=elasticsearch
+audit_elasticsearch_urls=
+audit_elasticsearch_port=9200
+audit_elasticsearch_protocol=http
+audit_elasticsearch_user=elastic
+audit_elasticsearch_password=elasticsearch
+audit_elasticsearch_index=ranger_audits
+audit_elasticsearch_bootstrap_enabled=true
+
+policymgr_external_url=http://ranger-admin:6080
+policymgr_http_enabled=true
+
+unix_user=ranger
+unix_user_pwd=ranger
+unix_group=ranger
+
+# Following variables are referenced in db_setup.py. Do not remove these
+sqlanywhere_core_file=
+cred_keystore_filename=
+
+# ################# DO NOT MODIFY ANY VARIABLES BELOW
#########################
+#
+# --- These deployment variables are not to be modified unless you understand
the full impact of the changes
+#
+################################################################################
+XAPOLICYMGR_DIR=$PWD
+app_home=$PWD/ews/webapp
+TMPFILE=$PWD/.fi_tmp
+LOGFILE=$PWD/logfile
+LOGFILES="$LOGFILE"
+
+JAVA_BIN='java'
+JAVA_VERSION_REQUIRED='1.8'
+
+ranger_admin_max_heap_size=1g
+#retry DB and Java patches after the given time in seconds.
+PATCH_RETRY_INTERVAL=120
+STALE_PATCH_ENTRY_HOLD_TIME=10
+
+hadoop_conf=
+authentication_method=UNIX
diff --git
a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties
b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties
new file mode 100644
index 0000000000..04c96989be
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties
@@ -0,0 +1,223 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This file provides a list of the deployment variables for the Ranger KMS Web
Application
+#
+
+PYTHON_COMMAND_INVOKER=python3
+DB_FLAVOR=MSSQL
+SQL_CONNECTOR_JAR=/usr/share/java/mssql.jar
+CONNECTION_STRING_ADDITIONAL_PARAMS="trustServerCertificate=true;"
+
+db_root_user=sa
+db_root_password=rangerR0cks!
+db_host=ranger-db
+
+db_name=rangerkms
+db_user=rangerkms
+db_password=rangerR0cks!
+
+# Following variables are referenced in db_setup.py. Do not remove these
+mysql_core_file=db/mysql/kms_core_db.sql
+postgres_core_file=db/postgres/kms_core_db_postgres.sql
+oracle_core_file=db/oracle/kms_core_db_oracle.sql
+sqlserver_core_file=db/sqlserver/kms_core_db_sqlserver.sql
+sqlanywhere_core_file=
+
+# For over-riding the jdbc url
+is_override_db_connection_string=true
+db_override_jdbc_connection_string="jdbc:sqlserver://ranger-db;databaseName=rangerkms;trustServerCertificate=true;"
+
+
+#SSL config
+db_ssl_enabled=false
+db_ssl_required=false
+db_ssl_verifyServerCertificate=false
+#db_ssl_auth_type=1-way|2-way, where 1-way represents standard one way ssl
authentication and 2-way represents mutual ssl authentication
+db_ssl_auth_type=2-way
+javax_net_ssl_keyStore=
+javax_net_ssl_keyStorePassword=
+javax_net_ssl_trustStore=
+javax_net_ssl_trustStorePassword=
+javax_net_ssl_trustStore_type=jks
+javax_net_ssl_keyStore_type=jks
+
+# For postgresql db
+db_ssl_certificate_file=
+
+#------------------------- DB CONFIG - END ----------------------------------
+#KMS Server config
+ranger_kms_http_enabled=true
+ranger_kms_https_keystore_file=
+ranger_kms_https_keystore_keyalias=rangerkms
+ranger_kms_https_keystore_password=
+
+#------------------------- RANGER KMS Install Dir ------------------
+COMPONENT_INSTALL_DIR_NAME=/opt/ranger/kms
+
+#------------------------- RANGER KMS Master Key Crypt Key ------------------
+KMS_MASTER_KEY_PASSWD=Str0ngPassw0rd
+
+#------------------------- Ranger KMS Kerberos Configuration
---------------------------
+kms_principal=
+kms_keytab=
+hadoop_conf=
+
+#------------------------- Ranger KMS HSM CONFIG ------------------------------
+HSM_TYPE=LunaProvider
+HSM_ENABLED=false
+HSM_PARTITION_NAME=par19
+HSM_PARTITION_PASSWORD=S@fenet123
+
+#------------------------- Ranger SAFENET KEYSECURE CONFIG
------------------------------
+KEYSECURE_ENABLED=false
+KEYSECURE_USER_PASSWORD_AUTHENTICATION=true
+KEYSECURE_MASTERKEY_NAME=safenetkeysecure
+KEYSECURE_USERNAME=user1
+KEYSECURE_PASSWORD=t1e2s3t4
+KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn
+KEYSECURE_MASTER_KEY_SIZE=256
+KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg
+
+#------------------------- Ranger Azure Key Vault
------------------------------
+AZURE_KEYVAULT_ENABLED=false
+AZURE_KEYVAULT_SSL_ENABLED=false
+AZURE_CLIENT_ID=50fd7ca6-fd4f-4785-a13f-1a6cc4e95e42
+AZURE_CLIENT_SECRET=<AzureKeyVaultPassword>
+AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx
+# Initialize below prop if your certificate file has any password
+#AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=certPass
+AZURE_MASTERKEY_NAME=RangerMasterKey
+# E.G. RSA, RSA_HSM, EC, EC_HSM, OCT
+AZURE_MASTER_KEY_TYPE=RSA
+# E.G. RSA_OAEP, RSA_OAEP_256, RSA1_5, RSA_OAEP
+ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP
+AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/
+
+#------------------------- Ranger Google Cloud HSM
------------------------------
+IS_GCP_ENABLED=false
+GCP_KEYRING_ID=
+GCP_CRED_JSON_FILE=/full/path/to/credfile.json
+GCP_PROJECT_ID=
+GCP_LOCATION_ID=
+GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt
+
+#------------------------- Ranger Tencent KMS ------------------------------
+TENCENT_KMS_ENABLED=false
+TENCENT_MASTERKEY_ID=b756b016-6e11-11ec-a735-525400fe0300
+TENCENT_CLIENT_ID=AKIDrXx6ybx2qNdiaBWaNs76pGQJvFJ6crpW
+TENCENT_CLIENT_SECRET=<TencentSecretKey>
+TENCENT_CLIENT_REGION=ap-beijing
+
+# ------- UNIX User CONFIG ----------------
+#
+unix_user=rangerkms
+unix_user_pwd=kms
+unix_group=ranger
+
+#
+# ------- UNIX User CONFIG - END ----------------
+#
+
+POLICY_MGR_URL=http://ranger:6080
+REPOSITORY_NAME=dev_kms
+
+# AUDIT configuration with V3 properties
+XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
+XAAUDIT.SUMMARY.ENABLE=true
+
+# Following properties are needed to get past installation script! Please
don't remove
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit
+XAAUDIT.HDFS.DESTINTATION_FILE=hive
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hive/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hive/audit/archive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
+XAAUDIT.SOLR.USER=NONE
+XAAUDIT.SOLR.PASSWORD=NONE
+XAAUDIT.SOLR.ZOOKEEPER=NONE
+XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hive/audit/solr/spool
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
+XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool
+
+XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
+XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
+
+XAAUDIT.LOG4J.ENABLE=false
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=false
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
+SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
+
+
+# Custom log directory path
+RANGER_KMS_LOG_DIR=/var/log/ranger/kms
+
+#PID file path
+RANGER_KMS_PID_DIR_PATH=/var/run/ranger_kms
+# ################# DO NOT MODIFY ANY VARIABLES BELOW
#########################
+#
+# --- These deployment variables are not to be modified unless you understand
the full impact of the changes
+#
+################################################################################
+KMS_DIR=$PWD
+app_home=$PWD/ews/webapp
+TMPFILE=$PWD/.fi_tmp
+LOGFILE=$PWD/logfile
+
+JAVA_BIN='java'
+JAVA_VERSION_REQUIRED='1.8'
+JAVA_ORACLE='Java(TM) SE Runtime Environment'
+
+cred_keystore_filename=$app_home/WEB-INF/classes/conf/.jceks/rangerkms.jceks
+
+KMS_BLACKLIST_DECRYPT_EEK=hdfs
diff --git a/kms/scripts/dba_script.py b/kms/scripts/dba_script.py
index 7e7b4e1ce3..5d19a7d215 100755
--- a/kms/scripts/dba_script.py
+++ b/kms/scripts/dba_script.py
@@ -825,19 +825,27 @@ def writeDrymodeCmd(self, xa_db_root_user,
xa_db_root_password, db_user, db_pass
class SqlServerConf(BaseDB):
# Constructor
- def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN):
+ def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN,
is_db_override_jdbc_connection_string, db_override_jdbc_connection_string):
self.host = host
self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR
self.JAVA_BIN = JAVA_BIN
+ self.is_db_override_jdbc_connection_string =
is_db_override_jdbc_connection_string
+ self.db_override_jdbc_connection_string =
db_override_jdbc_connection_string
def get_jisql_cmd(self, user, password, db_name):
#TODO: User array for forming command
path = RANGER_KMS_HOME
self.JAVA_BIN = self.JAVA_BIN.strip("'")
if is_unix:
- jisql_cmd = "%s -cp %s:%s/jisql/lib/*
org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring
jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name)
+ if self.is_db_override_jdbc_connection_string == 'true'
and self.db_override_jdbc_connection_string is not None and
len(self.db_override_jdbc_connection_string) > 0:
+ jisql_cmd = "%s -cp %s:%s/jisql/lib/*
org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring %s -noheader
-trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password,
self.db_override_jdbc_connection_string)
+ else:
+ jisql_cmd = "%s -cp %s:%s/jisql/lib/*
org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring
jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name)
elif os_name == "WINDOWS":
- jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\*
org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring
jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name)
+ if self.is_db_override_jdbc_connection_string == 'true'
and self.db_override_jdbc_connection_string is not None and
len(self.db_override_jdbc_connection_string) > 0:
+ jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\*
org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring %s
-noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password,
self.db_override_jdbc_connection_string)
+ else:
+ jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\*
org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring
jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name)
return jisql_cmd
def verify_user(self, root_user, db_root_password, db_user,dryMode):
@@ -1398,6 +1406,14 @@ def main(argv):
javax_net_ssl_keyStorePassword=''
javax_net_ssl_trustStore=''
javax_net_ssl_trustStorePassword=''
+
+ is_override_db_connection_string='false'
+ db_override_jdbc_connection_string=''
+ if 'is_override_db_connection_string' in globalDict:
+
is_override_db_connection_string=globalDict['is_override_db_connection_string'].lower()
+ if 'db_override_jdbc_connection_string' in globalDict:
+
db_override_jdbc_connection_string=globalDict['db_override_jdbc_connection_string'].strip()
+
if XA_DB_FLAVOR == "MYSQL" or XA_DB_FLAVOR == "POSTGRES":
if 'db_ssl_enabled' in globalDict:
db_ssl_enabled=globalDict['db_ssl_enabled'].lower()
@@ -1451,7 +1467,7 @@ def main(argv):
elif XA_DB_FLAVOR == "MSSQL":
SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR
- xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR,
JAVA_BIN)
+ xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR,
JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string)
xa_db_core_file =
os.path.join(RANGER_KMS_HOME,sqlserver_core_file)
elif XA_DB_FLAVOR == "SQLA":
diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh
index 9e31f5d347..b547b1751b 100755
--- a/kms/scripts/setup.sh
+++ b/kms/scripts/setup.sh
@@ -66,8 +66,10 @@ get_prop_or_default() {
}
PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE)
+
DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE)
SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE)
+CONNECTION_STRING_ADDITIONAL_PARAMS=$(get_prop
'CONNECTION_STRING_ADDITIONAL_PARAMS' $PROPFILE)
db_root_user=$(get_prop 'db_root_user' $PROPFILE)
db_root_password=$(get_prop 'db_root_password' $PROPFILE)
db_host=$(get_prop 'db_host' $PROPFILE)
@@ -605,7 +607,12 @@ update_properties() {
if [ "${DB_FLAVOR}" == "MSSQL" ]
then
propertyName=ranger.ks.jpa.jdbc.url
-
newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}"
+ if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ]
+ then
+
newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};${CONNECTION_STRING_ADDITIONAL_PARAMS}"
+ else
+
newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}"
+ fi
updatePropertyToFilePy $propertyName $newPropertyValue $to_file
propertyName=ranger.ks.jpa.jdbc.dialect
diff --git a/security-admin/scripts/dba_script.py
b/security-admin/scripts/dba_script.py
index 5f96ef26d5..8a334de063 100644
--- a/security-admin/scripts/dba_script.py
+++ b/security-admin/scripts/dba_script.py
@@ -1004,19 +1004,27 @@ def writeDrymodeCmd(self, xa_db_host, audit_db_host,
xa_db_root_user, xa_db_root
class SqlServerConf(BaseDB):
# Constructor
- def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN):
+ def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN,
is_db_override_jdbc_connection_string, db_override_jdbc_connection_string):
self.host = host
self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR
self.JAVA_BIN = JAVA_BIN
+ self.is_db_override_jdbc_connection_string =
is_db_override_jdbc_connection_string
+ self.db_override_jdbc_connection_string =
db_override_jdbc_connection_string
def get_jisql_cmd(self, user, password, db_name):
#TODO: User array for forming command
path = RANGER_ADMIN_HOME
self.JAVA_BIN = self.JAVA_BIN.strip("'")
if is_unix:
- jisql_cmd = "%s -cp %s:%s/jisql/lib/*
org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring
jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name)
+ if self.is_db_override_jdbc_connection_string == 'true'
and self.db_override_jdbc_connection_string is not None and
len(self.db_override_jdbc_connection_string) > 0:
+ jisql_cmd = "%s -cp %s:%s/jisql/lib/*
org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring %s -noheader
-trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password,
self.db_override_jdbc_connection_string)
+ else:
+ jisql_cmd = "%s -cp %s:%s/jisql/lib/*
org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring
jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name)
elif os_name == "WINDOWS":
- jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\*
org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring
jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name)
+ if self.is_db_override_jdbc_connection_string == 'true'
and self.db_override_jdbc_connection_string is not None and
len(self.db_override_jdbc_connection_string) > 0:
+ jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\*
org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring %s
-noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password,
self.db_override_jdbc_connection_string)
+ else:
+ jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\*
org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring
jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN,
self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name)
return jisql_cmd
def verify_user(self, root_user, db_root_password, db_user,dryMode):
@@ -1714,6 +1722,13 @@ def main(argv):
log("[E] Invalid ssl
keystore password!","error")
sys.exit(1)
+ is_override_db_connection_string='false'
+ db_override_jdbc_connection_string=''
+ if 'is_override_db_connection_string' in globalDict:
+
is_override_db_connection_string=globalDict['is_override_db_connection_string'].lower()
+ if 'db_override_jdbc_connection_string' in globalDict:
+
db_override_jdbc_connection_string=globalDict['db_override_jdbc_connection_string'].strip()
+
if XA_DB_FLAVOR == "MYSQL":
MYSQL_CONNECTOR_JAR=CONNECTOR_JAR
xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR,
JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type)
@@ -1740,7 +1755,7 @@ def main(argv):
elif XA_DB_FLAVOR == "MSSQL":
SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR
- xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR,
JAVA_BIN)
+ xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR,
JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string)
xa_db_version_file =
os.path.join(RANGER_ADMIN_HOME,sqlserver_dbversion_catalog)
xa_db_core_file =
os.path.join(RANGER_ADMIN_HOME,sqlserver_core_file)
xa_patch_file =
os.path.join(RANGER_ADMIN_HOME,sqlserver_patches)
@@ -1779,7 +1794,7 @@ def main(argv):
elif AUDIT_DB_FLAVOR == "MSSQL":
SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR
- audit_sqlObj = SqlServerConf(audit_db_host,
SQLSERVER_CONNECTOR_JAR, JAVA_BIN)
+ audit_sqlObj = SqlServerConf(audit_db_host,
SQLSERVER_CONNECTOR_JAR, JAVA_BIN, is_override_db_connection_string,
db_override_jdbc_connection_string)
audit_db_file =
os.path.join(RANGER_ADMIN_HOME,sqlserver_audit_file)
elif AUDIT_DB_FLAVOR == "SQLA":
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index b5eec25dd8..1fef99ee45 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -69,8 +69,10 @@ fi
LOGFILE=$(eval echo " $(get_prop 'LOGFILE' $PROPFILE)")
PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE)
+
DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE)
SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE)
+CONNECTION_STRING_ADDITIONAL_PARAMS=$(get_prop
'CONNECTION_STRING_ADDITIONAL_PARAMS' $PROPFILE)
db_root_user=$(get_prop 'db_root_user' $PROPFILE)
db_root_password=$(get_prop 'db_root_password' $PROPFILE)
db_host=$(get_prop 'db_host' $PROPFILE)
@@ -746,7 +748,12 @@ update_properties() {
if [ "${DB_FLAVOR}" == "MSSQL" ]
then
propertyName=ranger.jpa.jdbc.url
-
newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}"
+ if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ]
+ then
+
newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};${CONNECTION_STRING_ADDITIONAL_PARAMS}"
+ else
+
newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}"
+ fi
updatePropertyToFilePy $propertyName $newPropertyValue
$to_file_ranger
propertyName=ranger.jpa.jdbc.dialect
