Repository: incubator-ranger Updated Branches: refs/heads/tag-policy 892f6bf30 -> 3452ce857
RANGER-660: tagsync updated to use rangertagsync user while calling ranger-admin Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/3452ce85 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/3452ce85 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/3452ce85 Branch: refs/heads/tag-policy Commit: 3452ce8570d059f142c217c4b5f506f6d072769b Parents: 892f6bf Author: Abhay Kulkarni <[email protected]> Authored: Tue Oct 6 17:09:39 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Oct 14 11:54:59 2015 -0700 ---------------------------------------------------------------------- .../db/mysql/patches/012-createusersyncuser.sql | 2 +- .../db/mysql/patches/017-createtagsyncuser.sql | 45 ++++++ .../db/oracle/patches/017-createtagsyncuser.sql | 50 +++++++ .../db/postgres/xa_core_db_postgres.sql | 4 + .../db/sqlserver/xa_core_db_sqlserver.sql | 6 + .../ranger-tagsync-env-setup-hadoop-home.sh | 4 + .../conf/templates/installprop2xml.properties | 7 +- .../conf/templates/ranger-tagsync-template.xml | 12 +- tagsync/pom.xml | 5 + tagsync/scripts/install.properties | 11 +- tagsync/scripts/ranger-tagsync-services.sh | 2 +- tagsync/scripts/setup.py | 34 +++-- tagsync/scripts/updatetagadminpassword.py | 143 +++++++++++++++++++ .../ranger/tagsync/process/TagSyncConfig.java | 53 +++++-- .../tagsync/sink/tagadmin/TagRESTSink.java | 1 + .../main/resources/ranger-tagsync-default.xml | 20 +-- 16 files changed, 347 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/security-admin/db/mysql/patches/012-createusersyncuser.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/mysql/patches/012-createusersyncuser.sql b/security-admin/db/mysql/patches/012-createusersyncuser.sql index 9f3af62..0984621 100644 --- a/security-admin/db/mysql/patches/012-createusersyncuser.sql +++ b/security-admin/db/mysql/patches/012-createusersyncuser.sql @@ -17,7 +17,7 @@ drop procedure if exists create_user_sync; delimiter ;; create procedure create_user_sync() begin -DECLARE loginID varchar(1024); +DECLARE loginID bigint(20); /* check tables exist or not */ if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_portal_user') then if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_portal_user_role') then http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/security-admin/db/mysql/patches/017-createtagsyncuser.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/mysql/patches/017-createtagsyncuser.sql b/security-admin/db/mysql/patches/017-createtagsyncuser.sql new file mode 100644 index 0000000..02a5285 --- /dev/null +++ b/security-admin/db/mysql/patches/017-createtagsyncuser.sql @@ -0,0 +1,45 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists create_tag_sync; + +delimiter ;; +create procedure create_tag_sync() begin +DECLARE loginID bigint(20); + /* check tables exist or not */ + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_portal_user') then + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_portal_user_role') then + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_user') then + /* check record for login id rangertagsync exist or not */ + if not exists (select * from x_portal_user where login_id = 'rangertagsync') then + INSERT INTO x_portal_user(create_time,update_time,added_by_id,upd_by_id,first_name,last_name,pub_scr_name,login_id,password,email,status,user_src,notes) VALUES (UTC_TIMESTAMP(),UTC_TIMESTAMP(),NULL,NULL,'rangertagsync','','rangertagsync','rangertagsync','f5820e1229418dcf2575908f2c493da5','rangertagsync',1,0,NULL); + end if; + set loginID = (select id from x_portal_user where login_id = 'rangertagsync'); + if not exists (select * from x_portal_user_role where user_id =loginID ) then + INSERT INTO x_portal_user_role(create_time,update_time,added_by_id,upd_by_id,user_id,user_role,status) VALUES (UTC_TIMESTAMP(),UTC_TIMESTAMP(),NULL,NULL,loginID,'ROLE_SYS_ADMIN',1); + end if; + if not exists (select * from x_user where user_name = 'rangertagsync') then + INSERT INTO x_user(create_time,update_time,added_by_id,upd_by_id,user_name,descr,status) values (UTC_TIMESTAMP(), UTC_TIMESTAMP(),NULL,NULL,'rangertagsync','rangertagsync',0); + end if; + end if; + end if; + end if; + +end;; + +delimiter ; +call create_tag_sync(); + +drop procedure if exists create_tag_sync; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/security-admin/db/oracle/patches/017-createtagsyncuser.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/oracle/patches/017-createtagsyncuser.sql b/security-admin/db/oracle/patches/017-createtagsyncuser.sql new file mode 100644 index 0000000..7b72262 --- /dev/null +++ b/security-admin/db/oracle/patches/017-createtagsyncuser.sql @@ -0,0 +1,50 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +DECLARE + v_count number:=0; + loginID number:=0; + sql_stmt VARCHAR2(1000); + first_name VARCHAR2(20):='rangertagsync'; + scr_name VARCHAR2(20):='rangertagsync'; + login_name VARCHAR2(20):='rangertagsync'; + password VARCHAR2(50):='f5820e1229418dcf2575908f2c493da5'; + user_role VARCHAR2(50):='ROLE_SYS_ADMIN'; + email VARCHAR2(20):='rangertagsync'; +BEGIN + select count(*) into v_count from user_tables where table_name IN('X_PORTAL_USER','X_PORTAL_USER_ROLE','X_USER'); + if (v_count = 3) then + v_count:=0; + select count(*) into v_count from x_portal_user where login_id = login_name; + if (v_count = 0) then + sql_stmt := 'INSERT INTO x_portal_user(ID,CREATE_TIME,UPDATE_TIME,FIRST_NAME,LAST_NAME,PUB_SCR_NAME,LOGIN_ID,PASSWORD,EMAIL,STATUS,USER_SRC) VALUES (X_PORTAL_USER_SEQ.nextval,sys_extract_utc(systimestamp),sys_extract_utc(systimestamp),:1,NULL,:2,:3,:4,:5,1,0)'; + EXECUTE IMMEDIATE sql_stmt USING first_name,scr_name,login_name,password,email; + commit; + end if; + select id into loginID from x_portal_user where login_id = login_name; + if (loginID > 0) then + sql_stmt := 'INSERT INTO x_portal_user_role(id,create_time,update_time,user_id,user_role,status) VALUES (X_PORTAL_USER_ROLE_SEQ.nextval,sys_extract_utc(systimestamp),sys_extract_utc(systimestamp),:1,:2,1)'; + EXECUTE IMMEDIATE sql_stmt USING loginID,user_role; + commit; + end if; + v_count:=0; + select count(*) into v_count from x_user where user_name = login_name; + if (v_count = 0) then + sql_stmt := 'INSERT INTO x_user(id,create_time,update_time,user_name,descr,status) values (X_USER_SEQ.nextval,sys_extract_utc(systimestamp),sys_extract_utc(systimestamp),:1,:2,0)'; + EXECUTE IMMEDIATE sql_stmt USING login_name,login_name; + commit; + end if; + end if; +end;/ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/security-admin/db/postgres/xa_core_db_postgres.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/postgres/xa_core_db_postgres.sql b/security-admin/db/postgres/xa_core_db_postgres.sql index 5a76442..61f6cba 100644 --- a/security-admin/db/postgres/xa_core_db_postgres.sql +++ b/security-admin/db/postgres/xa_core_db_postgres.sql @@ -984,3 +984,7 @@ INSERT INTO x_portal_user(CREATE_TIME,UPDATE_TIME,FIRST_NAME,LAST_NAME,PUB_SCR_N INSERT INTO x_portal_user_role(CREATE_TIME,UPDATE_TIME,USER_ID,USER_ROLE,STATUS)VALUES(current_timestamp,current_timestamp,3,'ROLE_KEY_ADMIN',1); INSERT INTO x_user(CREATE_TIME,UPDATE_TIME,user_name,status,descr)VALUES(current_timestamp,current_timestamp,'keyadmin',0,'keyadmin'); COMMIT; +INSERT INTO x_portal_user(CREATE_TIME,UPDATE_TIME,FIRST_NAME,LAST_NAME,PUB_SCR_NAME,LOGIN_ID,PASSWORD,EMAIL,STATUS)VALUES(current_timestamp,current_timestamp,'rangertagsync','','rangertagsync','rangertagsync','f5820e1229418dcf2575908f2c493da5','rangertagsync',1); +INSERT INTO x_portal_user_role(CREATE_TIME,UPDATE_TIME,USER_ID,USER_ROLE,STATUS)VALUES(current_timestamp,current_timestamp,4,'ROLE_SYS_ADMIN',1); +INSERT INTO x_user(CREATE_TIME,UPDATE_TIME,user_name,status,descr)VALUES(current_timestamp,current_timestamp,'rangertagsync',0,'rangertagsync'); +COMMIT; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/security-admin/db/sqlserver/xa_core_db_sqlserver.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/sqlserver/xa_core_db_sqlserver.sql b/security-admin/db/sqlserver/xa_core_db_sqlserver.sql index a5b6559..8a1251d 100644 --- a/security-admin/db/sqlserver/xa_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/xa_core_db_sqlserver.sql @@ -2773,4 +2773,10 @@ GO insert into x_portal_user_role (CREATE_TIME,UPDATE_TIME,USER_ID,USER_ROLE,STATUS) values (CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,3,'ROLE_KEY_ADMIN',1); GO insert into x_user (CREATE_TIME,UPDATE_TIME,user_name,status,descr) values (CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,'keyadmin',0,'keyadmin'); +GO +insert into x_portal_user (CREATE_TIME,UPDATE_TIME,FIRST_NAME,LAST_NAME,PUB_SCR_NAME,LOGIN_ID,PASSWORD,EMAIL,STATUS) values (CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,'rangertagsync','','rangertagsync','rangertagsync','f5820e1229418dcf2575908f2c493da5','rangertagsync',1); +GO +insert into x_portal_user_role (CREATE_TIME,UPDATE_TIME,USER_ID,USER_ROLE,STATUS) values (CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,4,'ROLE_SYS_ADMIN',1); +GO +insert into x_user (CREATE_TIME,UPDATE_TIME,user_name,status,descr) values (CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,'rangertagsync',0,'rangertagsync'); exit http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/conf.dist/ranger-tagsync-env-setup-hadoop-home.sh ---------------------------------------------------------------------- diff --git a/tagsync/conf.dist/ranger-tagsync-env-setup-hadoop-home.sh b/tagsync/conf.dist/ranger-tagsync-env-setup-hadoop-home.sh new file mode 100644 index 0000000..012d862 --- /dev/null +++ b/tagsync/conf.dist/ranger-tagsync-env-setup-hadoop-home.sh @@ -0,0 +1,4 @@ +#!/bin/bash +if [ "$HADOOP_HOME" == "" ]; then + export HADOOP_HOME=/usr/hdp/current/hadoop-client +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/conf/templates/installprop2xml.properties ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties index 5d445ce..94618fc 100644 --- a/tagsync/conf/templates/installprop2xml.properties +++ b/tagsync/conf/templates/installprop2xml.properties @@ -24,8 +24,11 @@ TAGSYNC_ENABLED = ranger.tagsync.enabled TAGADMIN_ENDPOINT = ranger.tagsync.tagadmin.rest.url TAGADMIN_SSL_CONFIG_FILENAME = ranger.tagsync.tagadmin.rest.ssl.config.file -TAGADMIN_USERNAME = ranger.tagsync.tagadmin.basicauth.username -TAGADMIN_PASSWORD = ranger.tagsync.tagadmin.basicauth.password +#TAGADMIN_USERNAME = ranger.tagsync.tagadmin.basicauth.username +#TAGADMIN_PASSWORD = ranger.tagsync.tagadmin.basicauth.password + +TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.tagadmin.keystore + SYNC_INTERVAL = ranger.tagsync.sleeptimeinmillisbetweensynccycle http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/conf/templates/ranger-tagsync-template.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml index f9bc31b..ebee22d 100644 --- a/tagsync/conf/templates/ranger-tagsync-template.xml +++ b/tagsync/conf/templates/ranger-tagsync-template.xml @@ -36,19 +36,23 @@ <value></value> </property> <property> - <name>ranger.tagsync.tagadmin.basicauth.username</name> + <name>ranger.tagsync.filesource.filename</name> <value></value> </property> <property> - <name>ranger.tagsync.tagadmin.basicauth.password</name> + <name>ranger.tagsync.source.impl.class</name> <value></value> </property> <property> - <name>ranger.tagsync.filesource.filename</name> + <name>ranger.tagsync.tagadmin.keystore</name> <value></value> </property> <property> - <name>ranger.tagsync.source.impl.class</name> + <name>ranger.tagsync.tagadmin.alias</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.tagadmin.password</name> <value></value> </property> <property> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/pom.xml ---------------------------------------------------------------------- diff --git a/tagsync/pom.xml b/tagsync/pom.xml index d50374d..b800f61 100644 --- a/tagsync/pom.xml +++ b/tagsync/pom.xml @@ -97,6 +97,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>credentialbuilder</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <version>4.0</version> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/scripts/install.properties ---------------------------------------------------------------------- diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties index fb11ede..6b36846 100644 --- a/tagsync/scripts/install.properties +++ b/tagsync/scripts/install.properties @@ -23,6 +23,11 @@ unix_user=ranger unix_group=ranger +# +# The file where all credential is kept in cryptic format +# +TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks + # Logs are stored in logdir logdir = log @@ -32,12 +37,6 @@ TAGADMIN_ENDPOINT = http://localhost:6080 # SSL config file name for TagAdmin TAGADMIN_SSL_CONFIG_FILENAME = -# TagAdmin user name for basic authentication -TAGADMIN_USERNAME = admin - -# TagAdmin password for basic authentication -TAGADMIN_PASSWORD = admin - # Source for tags (either 'atlas' or 'file') TAG_SOURCE = atlas http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/scripts/ranger-tagsync-services.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh index 64622ce..e818d0d 100755 --- a/tagsync/scripts/ranger-tagsync-services.sh +++ b/tagsync/scripts/ranger-tagsync-services.sh @@ -42,7 +42,7 @@ if [ "${action}" == "START" ]; then . ${cdir}/conf/java_home.sh fi - for custom_env_script in `find ${cdir}/conf/ -name "ranger-tagsync-env*"`; do + for custom_env_script in `find ${cdir}/conf.dist/ -name "ranger-tagsync-env*"`; do if [ -f $custom_env_script ]; then . $custom_env_script fi http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/scripts/setup.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py index faa0af4..2721186 100755 --- a/tagsync/scripts/setup.py +++ b/tagsync/scripts/setup.py @@ -58,6 +58,9 @@ tagsyncBaseDirFullName = join(rangerBaseDirName, tagsyncBaseDirName) confFolderName = join(tagsyncBaseDirFullName, confBaseDirName) localConfFolderName = join(installPropDirName, confBaseDirName) +credUpdateClassName = 'org.apache.ranger.credentialapi.buildks' +defaultKeyStoreFileName = '/etc/ranger/tagsync/conf/rangertagsync.jceks' + unixUserProp = 'unix_user' unixGroupProp = 'unix_group' @@ -134,12 +137,8 @@ def getPropertiesKeyList(configFileName): def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName): tree = ET.parse(xmlTemplateFileName) root = tree.getroot() - prop_arr =[ "ranger.tagsync.keystore.password","ranger.tagsync.truststore.password","ranger.tagsync.policymgr"] for config in root.findall('property'): name = config.find('name').text - if name in prop_arr: - config.find('value').text = "_" - continue if (name in prop.keys()): if (name == TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING): # Expected value is 'clusterName,componentName,serviceName;clusterName,componentName,serviceName' ... @@ -168,6 +167,17 @@ def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName): archiveFile(xmlOutputFileName) tree.write(xmlOutputFileName) +def updatePropertyInJCKSFile(jcksFileName,propName,value): + fn = jcksFileName + if (value == ''): + value = ' ' + cmd = "java -cp './lib/*' %s create '%s' -value '%s' -provider jceks://file%s 2>&1" % (credUpdateClassName,propName,value,fn) + ret = os.system(cmd) + if (ret != 0): + print "ERROR: Unable update the JCKSFile(%s) for aliasName (%s)" % (fn,propName) + sys.exit(1) + return ret + def convertInstallPropsToXML(props): directKeyMap = getPropertiesConfigMap(join(installTemplateDirName,install2xmlMapFileName)) ret = {} @@ -358,11 +368,17 @@ def main(): initializeInitD() - if ('ranger.tagsync.tagadmin.basicauth.username' not in mergeProps): - mergeProps['ranger.tagsync.tagadmin.username'] = 'admin' - - if ('ranger.tagsync.tagadmin.basicauth.password' not in mergeProps): - mergeProps['ranger.tagsync.policymgr.password'] = 'admin' + tagsyncKSPath = mergeProps['ranger.tagsync.tagadmin.keystore'] + + if (tagsyncKSPath == ''): + mergeProps['ranger.tagsync.tagadmin.password'] = 'rangertagsync' + + else: + tagadminPasswd = 'rangertagsync' + tagadminAlias = 'tagadmin.user.password' + mergeProps['ranger.tagsync.tagadmin.alias'] = tagadminAlias + updatePropertyInJCKSFile(tagsyncKSPath,tagadminAlias,tagadminPasswd) + os.chown(tagsyncKSPath,ownerId,groupId) writeXMLUsingProperties(fn, mergeProps, outfn) http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/scripts/updatetagadminpassword.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/updatetagadminpassword.py b/tagsync/scripts/updatetagadminpassword.py new file mode 100644 index 0000000..ba0c911 --- /dev/null +++ b/tagsync/scripts/updatetagadminpassword.py @@ -0,0 +1,143 @@ +#!/usr/bin/python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +import os +import re +import sys +import errno +import shlex +import logging +import subprocess +import platform +import fileinput +import getpass +import shutil +from xml.etree import ElementTree as ET +from os.path import basename +from subprocess import Popen,PIPE +from datetime import date +from datetime import datetime +globalDict = {} + +os_name = platform.system() +os_name = os_name.upper() + +def check_output(query): + if os_name == "LINUX": + p = subprocess.Popen(shlex.split(query), stdout=subprocess.PIPE) + elif os_name == "WINDOWS": + p = subprocess.Popen(query, stdout=subprocess.PIPE, shell=True) + output = p.communicate ()[0] + return output + +def log(msg,type): + if type == 'info': + logging.info(" %s",msg) + if type == 'debug': + logging.debug(" %s",msg) + if type == 'warning': + logging.warning(" %s",msg) + if type == 'exception': + logging.exception(" %s",msg) + if type == 'error': + logging.error(" %s",msg) + +def import_properties_from_xml(xml_path, properties_from_xml=None): + print('getting values from file : ' + str(xml_path)) + if os.path.isfile(xml_path): + xml = ET.parse(xml_path) + root = xml.getroot() + if properties_from_xml is None: + properties_from_xml = dict() + for child in root.findall('property'): + name = child.find("name").text.strip() + value = child.find("value").text.strip() if child.find("value").text is not None else "" + properties_from_xml[name] = value + else: + print('XML file not found at path : ' + str(xml_path)) + return properties_from_xml + +def write_properties_to_xml(xml_path, property_name='', property_value=''): + if(os.path.isfile(xml_path)): + xml = ET.parse(xml_path) + root = xml.getroot() + for child in root.findall('property'): + name = child.find("name").text.strip() + if name == property_name: + child.find("value").text = property_value + xml.write(xml_path) + return 0 + else: + return -1 + +def main(): + global globalDict + FORMAT = '%(asctime)-15s %(message)s' + logging.basicConfig(format=FORMAT, level=logging.DEBUG) + + CFG_FILE=os.path.join(os.getcwd(),'conf','ranger-tagsync-site.xml') + if os.path.isfile(CFG_FILE): + pass + else: + log("[E] Required file not found: ["+CFG_FILE+"]","error") + sys.exit(1) + + if os.environ['JAVA_HOME'] == "": + log("[E] ---------- JAVA_HOME environment property not defined, aborting installation. ----------", "error") + sys.exit(1) + JAVA_BIN=os.path.join(os.environ['JAVA_HOME'],'bin','java') + if os_name == "WINDOWS" : + JAVA_BIN = JAVA_BIN+'.exe' + if os.path.isfile(JAVA_BIN): + pass + else: + while os.path.isfile(JAVA_BIN) == False: + log("Enter java executable path: :","info") + JAVA_BIN=raw_input() + log("[I] Using Java:" + str(JAVA_BIN),"info") + + globalDict=import_properties_from_xml(CFG_FILE,globalDict) + TAGSYNC_KEYSTORE_FILENAME=globalDict['ranger.tagsync.tagadmin.keystore'] + log("[I] TAGSYNC_KEYSTORE_FILENAME:" + str(TAGSYNC_KEYSTORE_FILENAME),"info") + TAGSYNC_TAGADMIN_ALIAS="tagadmin.user.password" + TAGSYNC_TAGADMIN_PASSWORD = '' + TAGSYNC_TAGADMIN_USERNAME = 'rangertagsync' + unix_user = "ranger" + unix_group = "ranger" + + while TAGSYNC_TAGADMIN_PASSWORD == "": + TAGSYNC_TAGADMIN_PASSWORD=getpass.getpass("Enter tagadmin user password:") + + if TAGSYNC_KEYSTORE_FILENAME != "" or TAGSYNC_TAGADMIN_ALIAS != "" or TAGSYNC_TAGADMIN_USERNAME != "" or TAGSYNC_TAGADMIN_PASSWORD != "": + log("[I] Storing tagadmin tagsync password in credential store:","info") + cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s -provider jceks://file%s" %(JAVA_BIN,TAGSYNC_TAGADMIN_ALIAS,TAGSYNC_TAGADMIN_PASSWORD,TAGSYNC_KEYSTORE_FILENAME) + ret=subprocess.call(shlex.split(cmd)) + if ret == 0: + cmd="chown %s:%s %s" %(unix_user,unix_group,TAGSYNC_KEYSTORE_FILENAME) + ret=subprocess.call(shlex.split(cmd)) + if ret == 0: + if os.path.isfile(CFG_FILE): + write_properties_to_xml(CFG_FILE,"ranger.tagsync.tagadmin.keystore",TAGSYNC_KEYSTORE_FILENAME) + write_properties_to_xml(CFG_FILE,"ranger.tagsync.tagadmin.alias", TAGSYNC_TAGADMIN_ALIAS) + else: + log("[E] Required file not found: ["+CFG_FILE+"]","error") + else: + log("[E] unable to execute command ["+cmd+"]","error") + else: + log("[E] unable to execute command ["+cmd+"]","error") + else: + log("[E] Input Error","error") + +main() http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index 7fe6bdb..bfd1b8b 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -26,6 +26,7 @@ import org.apache.log4j.Logger; import java.io.*; import java.net.URL; import java.util.Properties; +import org.apache.ranger.credentialapi.CredentialReader; public class TagSyncConfig extends Configuration { private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ; @@ -42,10 +43,6 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file"; - private static final String TAGSYNC_TAGADMIN_SSL_BASICAUTH_USERNAME_PROP = "ranger.tagsync.tagadmin.basicauth.username"; - - private static final String TAGSYNC_TAGADMIN_SSL_BASICAUTH_PASSWORD_PROP = "ranger.tagsync.tagadmin.basicauth.password"; - private static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.filesource.filename"; private static final String TAGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PROP = "ranger.tagsync.sleeptimeinmillisbetweensynccycle"; @@ -62,6 +59,11 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_"; + private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.tagadmin.keystore"; + private static final String TAGSYNC_TAGADMIN_ALIAS_PROP = "ranger.tagsync.tagadmin.alias"; + private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.tagadmin.password"; + private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync"; + private static volatile TagSyncConfig instance = null; public static TagSyncConfig getInstance() { @@ -210,16 +212,6 @@ public class TagSyncConfig extends Configuration { return val; } - static public String getTagAdminUserName(Properties prop) { - String val = prop.getProperty(TAGSYNC_TAGADMIN_SSL_BASICAUTH_USERNAME_PROP); - return val; - } - - static public String getTagAdminPassword(Properties prop) { - String val = prop.getProperty(TAGSYNC_TAGADMIN_SSL_BASICAUTH_PASSWORD_PROP); - return val; - } - static public String getTagSourceFileName(Properties prop) { String val = prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP); return val; @@ -230,6 +222,39 @@ public class TagSyncConfig extends Configuration { return val; } + static public String getTagAdminPassword(Properties prop) { + //update credential from keystore + String password = null; + if (prop != null && prop.containsKey(TAGSYNC_TAGADMIN_PASSWORD_PROP)) { + password = prop.getProperty(TAGSYNC_TAGADMIN_PASSWORD_PROP); + if (password != null && !password.isEmpty()) { + return password; + } + } + if (prop != null && prop.containsKey(TAGSYNC_TAGADMIN_KEYSTORE_PROP) && prop.containsKey(TAGSYNC_TAGADMIN_ALIAS_PROP)) { + String path = prop.getProperty(TAGSYNC_TAGADMIN_KEYSTORE_PROP); + String alias = prop.getProperty(TAGSYNC_TAGADMIN_ALIAS_PROP, "tagadmin.user.password"); + if (path != null && alias != null) { + if (!path.trim().isEmpty() && !alias.trim().isEmpty()) { + try { + password = CredentialReader.getDecryptedString(path.trim(), alias.trim()); + } catch (Exception ex) { + password = null; + } + if (password != null && !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")) { + prop.setProperty(TAGSYNC_TAGADMIN_PASSWORD_PROP, password); + return password; + } + } + } + } + return null; + } + + static public String getTagAdminUserName(Properties prop) { + return DEFAULT_TAGADMIN_USERNAME; + } + static public String getAtlasSslConfigFileName(Properties prop) { return ""; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagRESTSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagRESTSink.java b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagRESTSink.java index e1bcfbb..76bb62d 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagRESTSink.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagRESTSink.java @@ -23,6 +23,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.admin.client.datatype.RESTResponse; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3452ce85/tagsync/src/main/resources/ranger-tagsync-default.xml ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/ranger-tagsync-default.xml b/tagsync/src/main/resources/ranger-tagsync-default.xml index 5f754f9..b9e4512 100644 --- a/tagsync/src/main/resources/ranger-tagsync-default.xml +++ b/tagsync/src/main/resources/ranger-tagsync-default.xml @@ -23,7 +23,7 @@ </property> <property> <name>ranger.tagsync.logdir</name> - <value>./log</value> + <value>log</value> </property> <property> <name>ranger.tagsync.tagadmin.rest.url</name> @@ -36,16 +36,6 @@ <description></description> </property> <property> - <name>ranger.tagsync.tagadmin.basicauth.username</name> - <value>admin</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.tagadmin.basicauth.password</name> - <value>admin</value> - <description></description> - </property> - <property> <name>ranger.tagsync.sleeptimeinmillisbetweensynccycle</name> <value>60000</value> <description></description> @@ -66,13 +56,13 @@ <description></description> </property> <property> - <name>ranger.tagsync.atlassource.endpoint</name> - <value>http://localhost:21000/</value> + <name>ranger.tagsync.atlas.hive.instance.c1.ranger.service</name> + <value>cl1_hive</value> <description></description> </property> <property> - <name>ranger.tagsync.atlas.hive.instance.c1.ranger.service</name> - <value>cl1_hive</value> + <name>ranger.tagsync.atlassource.endpoint</name> + <value>http://localhost:21000/</value> <description></description> </property> </configuration>
