RANGER-660: updates to setup/install ranger-tagsync Signed-off-by: Madhan Neethiraj <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/44ddd597 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/44ddd597 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/44ddd597 Branch: refs/heads/master Commit: 44ddd59706e2d61678cb1f487276fcb06eb70ea5 Parents: e2075ff Author: Abhay Kulkarni <[email protected]> Authored: Mon Sep 28 12:09:57 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Sep 30 18:21:51 2015 -0700 ---------------------------------------------------------------------- .../ranger/rest/ServiceTagsProcessor.java | 2 + src/main/assembly/tagsync.xml | 13 +- tagsync/conf.dist/ranger-tagsync-site.xml | 67 --- .../conf/templates/installprop2xml.properties | 36 ++ .../conf/templates/ranger-tagsync-template.xml | 54 ++ tagsync/scripts/initd | 78 --- tagsync/scripts/install.properties | 58 ++ tagsync/scripts/ranger-tagsync-services.sh | 5 +- tagsync/scripts/ranger-tagsync.sh | 78 +++ tagsync/scripts/setup.py | 363 ++++++++++++ tagsync/scripts/setup.sh | 28 + .../java/org/apache/ranger/model/TagSink.java | 32 - .../java/org/apache/ranger/model/TagSource.java | 41 -- .../apache/ranger/process/TagSyncConfig.java | 280 --------- .../apache/ranger/process/TagSynchronizer.java | 211 ------- .../ranger/sink/policymgr/TagRESTSink.java | 511 ---------------- .../source/atlas/AtlasNotificationMapper.java | 391 ------------ .../ranger/source/atlas/TagAtlasSource.java | 588 ------------------ .../ranger/source/file/TagFileSource.java | 264 --------- .../apache/ranger/tagsync/model/TagSink.java | 31 + .../apache/ranger/tagsync/model/TagSource.java | 36 ++ .../ranger/tagsync/process/TagSyncConfig.java | 273 +++++++++ .../ranger/tagsync/process/TagSynchronizer.java | 211 +++++++ .../tagsync/sink/tagadmin/TagRESTSink.java | 511 ++++++++++++++++ .../source/atlas/AtlasNotificationMapper.java | 402 +++++++++++++ .../tagsync/source/atlas/TagAtlasSource.java | 589 +++++++++++++++++++ .../tagsync/source/file/TagFileSource.java | 264 +++++++++ .../src/main/resources/application.properties | 29 - .../main/resources/ranger-tagsync-default.xml | 35 +- .../ranger/process/TestTagSynchronizer.java | 93 --- .../tagsync/process/TestTagSynchronizer.java | 95 +++ 31 files changed, 3054 insertions(+), 2615 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java index 7f0a92c..da99b72 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java @@ -207,6 +207,8 @@ public class ServiceTagsProcessor { resource.setResourceSignature(resourceSignature); resourceInStore = tagStore.createServiceResource(resource); + } else if (StringUtils.isEmpty(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) { + resourceInStore = existing; } else { resource.setId(existing.getId()); resource.setGuid(existing.getGuid()); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/src/main/assembly/tagsync.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml index 4f43851..331dae0 100644 --- a/src/main/assembly/tagsync.xml +++ b/src/main/assembly/tagsync.xml @@ -122,10 +122,15 @@ <outputDirectory>/</outputDirectory> <directory>tagsync/scripts</directory> <excludes> - <exclude>*.properties</exclude> - <exclude>initd</exclude> + <exclude>ranger-tagsync.sh</exclude> </excludes> </fileSet> + <fileSet> + <directoryMode>755</directoryMode> + <fileMode>444</fileMode> + <outputDirectory>/templates</outputDirectory> + <directory>tagsync/conf/templates</directory> + </fileSet> <fileSet> <directoryMode>755</directoryMode> <outputDirectory>/</outputDirectory> @@ -138,9 +143,9 @@ </fileSets> <files> <file> - <source>tagsync/scripts/initd</source> + <source>tagsync/scripts/ranger-tagsync.sh</source> <outputDirectory>/</outputDirectory> - <destName>ranger-tagsync.sh</destName> + <destName>ranger-tagsync</destName> <fileMode>755</fileMode> </file> </files> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/conf.dist/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf.dist/ranger-tagsync-site.xml b/tagsync/conf.dist/ranger-tagsync-site.xml deleted file mode 100644 index 21accd4..0000000 --- a/tagsync/conf.dist/ranger-tagsync-site.xml +++ /dev/null @@ -1,67 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - - -<configuration> - <property> - <name>ranger.authentication.method</name> - <value>NONE</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.tagadmin.rest.url</name> - <value>http://localhost:6080</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name> - <value></value> - <description></description> - </property> - <property> - <name>ranger.tagsync.policymanager.basicauth.username</name> - <value>admin</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.policymanager.basicauth.password</name> - <value>admin</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.sleeptimeinmillisbetweensynccycle</name> - <value>60000</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.source.file</name> - <value>/etc/ranger/data/tags.json</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.source.impl.class</name> - <value>org.apache.ranger.source.file.TagFileSource</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.sink.impl.class</name> - <value>org.apache.ranger.sink.policymgr.TagRESTSink</value> - <description></description> - </property> - <property> - <name>atlas.endpoint</name> - <value>http://localhost:21000/</value> - <description></description> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/conf/templates/installprop2xml.properties ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties new file mode 100644 index 0000000..9fe1cac --- /dev/null +++ b/tagsync/conf/templates/installprop2xml.properties @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +unix_user = unix_user +unix_group = unix_group +TAG_SOURCE = TAG_SOURCE + +logdir = ranger.tagsync.logdir + +TAGSYNC_ENABLED = ranger.tagsync.enabled + +TAGADMIN_ENDPOINT = ranger.tagsync.tagadmin.rest.url +TAGADMIN_SSL_CONFIG_FILENAME = ranger.tagsync.tagadmin.rest.ssl.config.file +TAGADMIN_USERNAME = ranger.tagsync.tagadmin.basicauth.username +TAGADMIN_PASSWORD = ranger.tagsync.tagadmin.basicauth.password + +SYNC_INTERVAL = ranger.tagsync.sleeptimeinmillisbetweensynccycle + +TAGSYNC_FILESOURCE_FILENAME = ranger.tagsync.filesource.filename + +TAGSYNC_ATLAS_KAFKA_ENDPOINTS = atlas.notification.kafka.bootstrap.servers +TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = atlas.notification.kafka.zookeeper.connect +TAGSYNC_ATLAS_CONSUMER_GROUP = atlas.notification.kafka.group.id http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/conf/templates/ranger-tagsync-template.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml new file mode 100644 index 0000000..5f9f3b5 --- /dev/null +++ b/tagsync/conf/templates/ranger-tagsync-template.xml @@ -0,0 +1,54 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<configuration> + <property> + <name>ranger.tagsync.enabled</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.logdir</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.tagadmin.rest.url</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.sleeptimeinmillisbetweensynccycle</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.tagadmin.basicauth.username</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.tagadmin.basicauth.password</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.filesource.filename</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.source.impl.class</name> + <value></value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/scripts/initd ---------------------------------------------------------------------- diff --git a/tagsync/scripts/initd b/tagsync/scripts/initd deleted file mode 100644 index 9349fae..0000000 --- a/tagsync/scripts/initd +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -### BEGIN INIT INFO -# Provides: ranger-tagsync -# Required-Start: $local_fs $remote_fs $network $named $syslog $time -# Required-Stop: $local_fs $remote_fs $network $named $syslog $time -# Default-Start: 2 3 4 5 -# Default-Stop: -# Short-Description: Start/Stop Ranger tagsync -### END INIT INFO - -LINUX_USER=ranger -BIN_PATH=/usr/bin -MOD_NAME=ranger-tagsync-services.sh -pidf=/var/run/ranger/tagsync.pid -pid="" -if [ -f ${pidf} ] -then - pid=`cat $pidf` -fi - -case $1 in - start) - if [ "${pid}" != "" ] - then - echo "Ranger tagsync Service is already running" - exit 1 - else - echo "Starting Ranger tagsync." - /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} start" - fi - ;; - stop) - if [ "${pid}" != "" ] - then - echo "Stopping Ranger tagsync." - /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} stop" - else - echo "Ranger tagsync Service is NOT running" - exit 1 - fi - ;; - restart) - if [ "${pid}" != "" ] - then - echo "Stopping Ranger tagsync." - /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} stop" - sleep 10 - fi - echo "Starting Ranger tagsync." - /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} start" - ;; - status) - if [ "${pid}" != "" ] - then - echo "Ranger tagsync Service is running [pid={$pid}]" - else - echo "Ranger tagsync Service is NOT running." - fi - ;; - *) - echo "Invalid argument [$1]; Only start | stop | restart | status, are supported." - exit 1 - esac http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/scripts/install.properties ---------------------------------------------------------------------- diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties new file mode 100644 index 0000000..82477d3 --- /dev/null +++ b/tagsync/scripts/install.properties @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file provides list of deployment variables for the Ranger KMS Web Application +# +# +# + +#User and group for the usersync process +unix_user=ranger +unix_group=ranger + +# Logs are stored in logdir +logdir = log + +# URL for TagAdmin +TAGADMIN_ENDPOINT = http://localhost:6080 + +# SSL config file name for TagAdmin +TAGADMIN_SSL_CONFIG_FILENAME = + +# TagAdmin user name for basic authentication +TAGADMIN_USERNAME = admin + +# TagAdmin password for basic authentication +TAGADMIN_PASSWORD = admin + +# Source for tags (either 'atlas' or 'file') + +TAG_SOURCE = atlas + +# File name to be used for reading tags information if TAG_SOURCE = file + +TAGSYNC_FILESOURCE_FILENAME = /etc/ranger/data/tags.json + + +# Interval for checking the source for any changes in case of TAG_SOURCE = file +# Also used for periodicity of checking if the process needs to be shut down +SYNC_INTERVAL = 60000 + +# Endpoint specifications needed by Atlas + +TAGSYNC_ATLAS_KAFKA_ENDPOINTS = localhost:6667 +TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = localhost:2181 +TAGSYNC_ATLAS_CONSUMER_GROUP = entityConsumer \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/scripts/ranger-tagsync-services.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh index 2f4b255..64622ce 100755 --- a/tagsync/scripts/ranger-tagsync-services.sh +++ b/tagsync/scripts/ranger-tagsync-services.sh @@ -56,9 +56,10 @@ if [ "${action}" == "START" ]; then if [ ! -d $logdir ]; then mkdir -p $logdir + chmod 777 $logdir fi - cp="${cdir}/conf.dist:${cdir}/dist/*:${cdir}/lib/*" + cp="${cdir}/conf:${cdir}/dist/*:${cdir}/lib/*" if [ -f $pidf ]; then PID=`cat $pidf` @@ -73,7 +74,7 @@ if [ "${action}" == "START" ]; then cd ${cdir} umask 0077 - nohup java -Dproc_rangertagsync ${JAVA_OPTS} -Dlogdir="${logdir}" -cp "${cp}" org.apache.ranger.process.TagSynchronizer > ${logdir}/tagsync.log 2>&1 & + nohup java -Dproc_rangertagsync ${JAVA_OPTS} -Dlogdir="${logdir}" -cp "${cp}" org.apache.ranger.tagsync.process.TagSynchronizer > ${logdir}/tagsync.out 2>&1 & echo $! > ${pidf} chown ranger ${pidf} sleep 5 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/scripts/ranger-tagsync.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync.sh b/tagsync/scripts/ranger-tagsync.sh new file mode 100644 index 0000000..9349fae --- /dev/null +++ b/tagsync/scripts/ranger-tagsync.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: ranger-tagsync +# Required-Start: $local_fs $remote_fs $network $named $syslog $time +# Required-Stop: $local_fs $remote_fs $network $named $syslog $time +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Start/Stop Ranger tagsync +### END INIT INFO + +LINUX_USER=ranger +BIN_PATH=/usr/bin +MOD_NAME=ranger-tagsync-services.sh +pidf=/var/run/ranger/tagsync.pid +pid="" +if [ -f ${pidf} ] +then + pid=`cat $pidf` +fi + +case $1 in + start) + if [ "${pid}" != "" ] + then + echo "Ranger tagsync Service is already running" + exit 1 + else + echo "Starting Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} start" + fi + ;; + stop) + if [ "${pid}" != "" ] + then + echo "Stopping Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} stop" + else + echo "Ranger tagsync Service is NOT running" + exit 1 + fi + ;; + restart) + if [ "${pid}" != "" ] + then + echo "Stopping Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} stop" + sleep 10 + fi + echo "Starting Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} start" + ;; + status) + if [ "${pid}" != "" ] + then + echo "Ranger tagsync Service is running [pid={$pid}]" + else + echo "Ranger tagsync Service is NOT running." + fi + ;; + *) + echo "Invalid argument [$1]; Only start | stop | restart | status, are supported." + exit 1 + esac http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/scripts/setup.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py new file mode 100755 index 0000000..383964a --- /dev/null +++ b/tagsync/scripts/setup.py @@ -0,0 +1,363 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import StringIO +import xml.etree.ElementTree as ET +import ConfigParser +import os,errno,sys,getopt +from os import listdir +from os.path import isfile, join, dirname, basename +from urlparse import urlparse +from time import gmtime, strftime, localtime +from xml import etree +import shutil +import pwd, grp + +if (not 'JAVA_HOME' in os.environ): + print "ERROR: JAVA_HOME environment variable is not defined. Please define JAVA_HOME before running this script" + sys.exit(1) + +debugLevel = 1 +generateXML = 0 +installPropDirName = '.' +pidFolderName = '/var/run/ranger' +logFolderName = '/var/log/ranger' +initdDirName = '/etc/init.d' + +rangerBaseDirName = '/etc/ranger' +tagsyncBaseDirName = 'tagsync' +confBaseDirName = 'conf' +confDistBaseDirName = 'conf.dist' + +outputFileName = 'ranger-tagsync-site.xml' +installPropFileName = 'install.properties' +log4jFileName = 'log4j.xml' +install2xmlMapFileName = 'installprop2xml.properties' +templateFileName = 'ranger-tagsync-template.xml' +initdProgramName = 'ranger-tagsync' +atlasApplicationPropFileName = 'application.properties' + +installTemplateDirName = join(installPropDirName,'templates') +confDistDirName = join(installPropDirName, confDistBaseDirName) +tagsyncLogFolderName = join(logFolderName, 'tagsync') +tagsyncBaseDirFullName = join(rangerBaseDirName, tagsyncBaseDirName) +confFolderName = join(tagsyncBaseDirFullName, confBaseDirName) +localConfFolderName = join(installPropDirName, confBaseDirName) + +unixUserProp = 'unix_user' +unixGroupProp = 'unix_group' + +logFolderPermMode = 0777 +rootOwnerId = 0 +initPrefixList = ['S99', 'K00'] + +TAG_SOURCE_KEY = 'TAG_SOURCE' +SYNC_INTERVAL_NEW_KEY = 'ranger.tagsync.sleeptimeinmillisbetweensynccycle' +TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAGSYNC_ATLAS_KAFKA_ENDPOINTS' +TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT' +TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAGSYNC_ATLAS_CONSUMER_GROUP' + +TAG_SOURCE_ATLAS = 'atlas' +TAG_SOURCE_FILE = 'file' +TAG_SOURCE_LIST = [ TAG_SOURCE_ATLAS, TAG_SOURCE_FILE ] + +def archiveFile(originalFileName): + archiveDir = dirname(originalFileName) + archiveFileName = "." + basename(originalFileName) + "." + (strftime("%d%m%Y%H%M%S", localtime())) + movedFileName = join(archiveDir,archiveFileName) + print "INFO: moving [%s] to [%s] ......." % (originalFileName,movedFileName) + os.rename(originalFileName, movedFileName) + +def getXMLConfigKeys(xmlFileName): + ret = [] + tree = ET.parse(xmlFileName) + root = tree.getroot() + for config in root.iter('property'): + name = config.find('name').text + ret.append(name) + return ret + +def getXMLConfigMap(xmlFileName): + ret = {} + tree = ET.parse(xmlFileName) + root = tree.getroot() + for config in root.findall('property'): + name = config.find('name').text + val = config.find('value').text + ret[name] = val + return ret + + +def getPropertiesConfigMap(configFileName): + ret = {} + config = StringIO.StringIO() + config.write('[dummysection]\n') + config.write(open(configFileName).read()) + config.seek(0,os.SEEK_SET) + fcp = ConfigParser.ConfigParser() + fcp.optionxform = str + fcp.readfp(config) + for k,v in fcp.items('dummysection'): + ret[k] = v + return ret + +def getPropertiesKeyList(configFileName): + ret = [] + config = StringIO.StringIO() + config.write('[dummysection]\n') + config.write(open(configFileName).read()) + config.seek(0,os.SEEK_SET) + fcp = ConfigParser.ConfigParser() + fcp.optionxform = str + fcp.readfp(config) + for k,v in fcp.items('dummysection'): + ret.append(k) + return ret + +def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName): + tree = ET.parse(xmlTemplateFileName) + root = tree.getroot() + prop_arr =[ "ranger.tagsync.keystore.password","ranger.tagsync.truststore.password","ranger.tagsync.policymgr"] + for config in root.findall('property'): + name = config.find('name').text + if name in prop_arr: + config.find('value').text = "_" + continue + if (name in prop.keys()): + config.find('value').text = str(prop[name]) + #else: + # print "ERROR: key not found: %s" % (name) + if isfile(xmlOutputFileName): + archiveFile(xmlOutputFileName) + tree.write(xmlOutputFileName) + +def convertInstallPropsToXML(props): + directKeyMap = getPropertiesConfigMap(join(installTemplateDirName,install2xmlMapFileName)) + ret = {} + atlasOutFn = join(confFolderName, atlasApplicationPropFileName) + + atlasOutFile = file(atlasOutFn, "w") + + for k,v in props.iteritems(): + if (k in directKeyMap.keys()): + newKey = directKeyMap[k] + if (k == TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY): + atlasOutFile.write(newKey + "=" + v + "\n") + elif (k == TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY): + atlasOutFile.write(newKey + "=" + v + "\n") + elif (k == TAGSYNC_ATLAS_CONSUMER_GROUP_KEY): + atlasOutFile.write(newKey + "=" + v + "\n") + else: + ret[newKey] = v + else: + print "Direct Key not found:%s" % (k) + + ret['ranger.tagsync.sink.impl.class'] = 'org.apache.ranger.sink.policymgr.TagRESTSink' + if (TAG_SOURCE_KEY in ret): + tagSource = ret[TAG_SOURCE_KEY] + if (tagSource == TAG_SOURCE_ATLAS): + ret['ranger.tagsync.source.impl.class'] = 'atlas' + elif (tagSource == TAG_SOURCE_FILE): + ret['ranger.tagsync.source.impl.class'] = 'file' + else: + print "ERROR: Invalid value (%s) defined for %s in install.properties. Only valid values are %s" % (tagSource, TAG_SOURCE_ATLAS,TAG_SOURCE_FILE) + sys.exit(1) + del ret['TAG_SOURCE'] + else: + print "ERROR: No value defined for TAG_SOURCE in install.properties. valid values are %s" % (TAG_SOURCE_ATLAS, TAG_SOURCE_FILE) + sys.exit(1) + + atlasOutFile.close() + + return ret + +def createUser(username,groupname): + cmd = "useradd -g %s %s -m" % (groupname,username) + ret = os.system(cmd) + if (ret != 0): + print "ERROR: os command execution (%s) failed. error code = %d " % (cmd, ret) + sys.exit(1) + try: + ret = pwd.getpwnam(username).pw_uid + return ret + except KeyError, e: + print "ERROR: Unable to create a new user account: %s with group %s - error [%s]" % (username,groupname,e) + sys.exit(1) + +def createGroup(groupname): + cmd = "groupadd %s" % (groupname) + ret = os.system(cmd) + if (ret != 0): + print "ERROR: os command execution (%s) failed. error code = %d " % (cmd, ret) + sys.exit(1) + try: + ret = grp.getgrnam(groupname).gr_gid + return ret + except KeyError, e: + print "ERROR: Unable to create a new group: %s" % (groupname,e) + sys.exit(1) + +def initializeInitD(): + if (os.path.isdir(initdDirName)): + fn = join(installPropDirName,initdProgramName) + initdFn = join(initdDirName,initdProgramName) + shutil.copy(fn, initdFn) + os.chmod(initdFn,0550) + rcDirList = [ "/etc/rc2.d", "/etc/rc3.d", "/etc/rc.d/rc2.d", "/etc/rc.d/rc3.d" ] + for rcDir in rcDirList: + if (os.path.isdir(rcDir)): + for prefix in initPrefixList: + scriptFn = prefix + initdProgramName + scriptName = join(rcDir, scriptFn) + if isfile(scriptName): + os.remove(scriptName) + #print "+ ln -sf %s %s" % (initdFn, scriptName) + os.symlink(initdFn,scriptName) + tagSyncScriptName = "ranger-tagsync-services.sh" + localScriptName = os.path.abspath(join(installPropDirName,tagSyncScriptName)) + ubinScriptName = join("/usr/bin",tagSyncScriptName) + if isfile(ubinScriptName): + os.remove(ubinScriptName) + os.symlink(localScriptName,ubinScriptName) + + +def main(): + + print "\nINFO: Installing ranger-tagsync .....\n" + + dirList = [ rangerBaseDirName, tagsyncBaseDirFullName, confFolderName ] + for dir in dirList: + if (not os.path.isdir(dir)): + os.makedirs(dir,0755) + + defFileList = [ log4jFileName ] + for defFile in defFileList: + fn = join(confDistDirName, defFile) + if ( isfile(fn) ): + shutil.copy(fn,join(confFolderName,defFile)) + + # + # Create JAVA_HOME setting in confFolderName + # + java_home_setter_fn = join(confFolderName, 'java_home.sh') + if isfile(java_home_setter_fn): + archiveFile(java_home_setter_fn) + jhf = open(java_home_setter_fn, 'w') + str = "export JAVA_HOME=%s\n" % os.environ['JAVA_HOME'] + jhf.write(str) + jhf.close() + os.chmod(java_home_setter_fn,0750) + + + if (not os.path.isdir(localConfFolderName)): + os.symlink(confFolderName, localConfFolderName) + + installProps = getPropertiesConfigMap(join(installPropDirName,installPropFileName)) + modifiedInstallProps = convertInstallPropsToXML(installProps) + + mergeProps = {} + mergeProps.update(modifiedInstallProps) + + localLogFolderName = mergeProps['ranger.tagsync.logdir'] + if (not os.path.isdir(localLogFolderName)): + if (localLogFolderName != tagsyncLogFolderName): + os.symlink(tagsyncLogFolderName, localLogFolderName) + + fn = join(installTemplateDirName,templateFileName) + outfn = join(confFolderName, outputFileName) + + atlasOutFn = join(confFolderName, atlasApplicationPropFileName) + + atlasOutFile = file(atlasOutFn, "a") + + atlasOutFile.write("atlas.notification.embedded=false" + "\n") + atlasOutFile.write("atlas.notification.kafka.acks=1" + "\n") + atlasOutFile.write("atlas.notification.kafka.data=${sys:atlas.home}/data/kafka" + "\n") + + atlasOutFile.close() + + + if ( os.path.isdir(logFolderName) ): + logStat = os.stat(logFolderName) + logStat.st_uid + logStat.st_gid + ownerName = pwd.getpwuid(logStat.st_uid).pw_name + groupName = pwd.getpwuid(logStat.st_uid).pw_name + else: + os.makedirs(logFolderName,logFolderPermMode) + + if (not os.path.isdir(pidFolderName)): + os.makedirs(pidFolderName,logFolderPermMode) + + if (not os.path.isdir(tagsyncLogFolderName)): + os.makedirs(tagsyncLogFolderName,logFolderPermMode) + + if (unixUserProp in mergeProps): + ownerName = mergeProps[unixUserProp] + else: + mergeProps[unixUserProp] = "ranger" + ownerName = mergeProps[unixUserProp] + + if (unixGroupProp in mergeProps): + groupName = mergeProps[unixGroupProp] + else: + mergeProps[unixGroupProp] = "ranger" + groupName = mergeProps[unixGroupProp] + + try: + ownerId = pwd.getpwnam(ownerName).pw_uid + except KeyError, e: + ownerId = createUser(ownerName, groupName) + + try: + groupId = grp.getgrnam(groupName).gr_gid + except KeyError, e: + groupId = createGroup(groupId) + + os.chown(logFolderName,ownerId,groupId) + os.chown(tagsyncLogFolderName,ownerId,groupId) + os.chown(pidFolderName,ownerId,groupId) + os.chown(rangerBaseDirName,ownerId,groupId) + + initializeInitD() + + if ('ranger.tagsync.tagadmin.basicauth.username' not in mergeProps): + mergeProps['ranger.tagsync.tagadmin.username'] = 'admin' + + if ('ranger.tagsync.tagadmin.basicauth.password' not in mergeProps): + mergeProps['ranger.tagsync.policymgr.password'] = 'admin' + + writeXMLUsingProperties(fn, mergeProps, outfn) + + fixPermList = [ ".", tagsyncBaseDirName, confFolderName ] + + for dir in fixPermList: + for root, dirs, files in os.walk(dir): + os.chown(root, ownerId, groupId) + os.chmod(root,0755) + for obj in dirs: + dn = join(root,obj) + os.chown(dn, ownerId, groupId) + os.chmod(dn, 0755) + for obj in files: + fn = join(root,obj) + os.chown(fn, ownerId, groupId) + os.chmod(fn, 0755) + + print "\nINFO: Completed ranger-tagsync installation.....\n" + +main() http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/scripts/setup.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/setup.sh b/tagsync/scripts/setup.sh new file mode 100755 index 0000000..d2a4f1b --- /dev/null +++ b/tagsync/scripts/setup.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "${JAVA_HOME}" == "" ] +then + echo "JAVA_HOME environment property not defined, aborting installation." + exit 1 +else + export JAVA_HOME + PATH="${JAVA_HOME}/bin:${PATH}" + export PATH +fi + +./setup.py http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/src/main/java/org/apache/ranger/model/TagSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java deleted file mode 100644 index 5256570..0000000 --- a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.model; - -import org.apache.ranger.plugin.store.TagStore; -import org.apache.ranger.plugin.util.ServiceTags; - -import java.util.Map; -import java.util.Properties; - - -public interface TagSink extends TagStore { - boolean initialize(Properties properties); - void uploadServiceTags(ServiceTags serviceTags) throws Exception; -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/src/main/java/org/apache/ranger/model/TagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java deleted file mode 100644 index 0ccceae..0000000 --- a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.model; - -import org.apache.ranger.plugin.model.RangerTagDef; -import org.apache.ranger.plugin.model.RangerTagResourceMap; - -import java.util.List; -import java.util.Map; -import java.util.Properties; - -public interface TagSource { - - boolean initialize(Properties properties); - - void setTagSink(TagSink sink); - - void updateSink() throws Exception; - - Thread start(); - - boolean isChanged(); - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java deleted file mode 100644 index c80eff9..0000000 --- a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.process; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.log4j.Logger; - -import java.io.*; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Properties; - -public class TagSyncConfig extends Configuration { - private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ; - - public static final String CONFIG_FILE = "ranger-tagsync-site.xml"; - - public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml"; - - public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled" ; - - public static final String TAGSYNC_PORT_PROP = "ranger.tagsync.port" ; - - public static final String TAGSYNC_SSL_PROP = "ranger.tagsync.ssl" ; - - public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir" ; - - private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.tagadmin.rest.url"; - - private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file"; - - private static final String TAGSYNC_PM_SSL_BASICAUTH_USERNAME_PROP = "ranger.tagsync.policymanager.basicauth.username"; - - private static final String TAGSYNC_PM_SSL_BASICAUTH_PASSWORD_PROP = "ranger.tagsync.policymanager.basicauth.password"; - - private static final String TAGSYNC_SOURCE_FILE_PROP = "ranger.tagsync.source.file"; - - private static final String TAGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PROP = "ranger.tagsync.sleeptimeinmillisbetweensynccycle"; - - private static final String TAGSYNC_SOURCE_CLASS_PROP = "ranger.tagsync.source.impl.class"; - - private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.sink.impl.class"; - - private static final String TAGSYNC_SOURCE_ATLAS_PROP = "atlas.endpoint"; - - private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas."; - - private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service"; - - private static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_"; - - private static volatile TagSyncConfig instance = null; - - public static TagSyncConfig getInstance() { - /* - TagSyncConfig ret = instance; - if (ret == null) { - synchronized(TagSyncConfig.class) { - if (ret == null) { - ret = instance = new TagSyncConfig(); - LOG.debug("TagSyncConfig = {" + ret + "}"); - } - } - } - */ - TagSyncConfig newConfig = new TagSyncConfig(); - return newConfig; - } - - public Properties getProperties() { - return getProps(); - } - - public static InputStream getFileInputStream(String path) throws FileNotFoundException { - - InputStream ret = null; - - File f = new File(path); - - if (f.exists() && f.isFile() && f.canRead()) { - ret = new FileInputStream(f); - } else { - ret = TagSyncConfig.class.getResourceAsStream(path); - - if (ret == null) { - if (! path.startsWith("/")) { - ret = TagSyncConfig.class.getResourceAsStream("/" + path); - } - } - - if (ret == null) { - ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path) ; - if (ret == null) { - if (! path.startsWith("/")) { - ret = ClassLoader.getSystemResourceAsStream("/" + path); - } - } - } - } - - return ret; - } - - public static String getResourceFileName(String path) { - - String ret = null; - - if (StringUtils.isNotBlank(path)) { - - File f = new File(path); - - if (f.exists() && f.isFile() && f.canRead()) { - ret = path; - } else { - - URL fileURL = TagSyncConfig.class.getResource(path); - if (fileURL == null) { - if (!path.startsWith("/")) { - fileURL = TagSyncConfig.class.getResource("/" + path); - } - } - - if (fileURL == null) { - fileURL = ClassLoader.getSystemClassLoader().getResource(path); - if (fileURL == null) { - if (!path.startsWith("/")) { - fileURL = ClassLoader.getSystemClassLoader().getResource("/" + path); - } - } - } - - if (fileURL != null) { - try { - ret = fileURL.getFile(); - } catch (Exception exception) { - LOG.error(path + " is not a file", exception); - } - } else { - LOG.warn("URL not found for " + path + " or no privilege for reading file " + path); - } - } - } - - return ret; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - - sb.append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append(", ") - .append("CONFIG_FILE=").append(CONFIG_FILE).append("\n\n"); - - return sb.toString() + super.toString(); - } - - static public boolean isTagSyncEnabled(Properties prop) { - String val = prop.getProperty(TAGSYNC_ENABLED_PROP); - return !(val != null && val.trim().equalsIgnoreCase("falae")); - } - - static public String getTagSyncPort(Properties prop) { - String val = prop.getProperty(TAGSYNC_PORT_PROP); - return val; - } - - static public boolean isTagSyncSsl(Properties prop) { - String val = prop.getProperty(TAGSYNC_SSL_PROP); - return (val != null && val.trim().equalsIgnoreCase("true")); - } - - static public String getTagSyncLogdir(Properties prop) { - String val = prop.getProperty(TAGSYNC_LOGDIR_PROP); - return val; - } - - static public long getSleepTimeInMillisBetweenCycle(Properties prop) { - String val = prop.getProperty(TAGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PROP); - return Long.valueOf(val); - } - - static public String getTagSourceClassName(Properties prop) { - String val = prop.getProperty(TAGSYNC_SOURCE_CLASS_PROP); - return val; - } - - static public String getTagSinkClassName(Properties prop) { - String val = prop.getProperty(TAGSYNC_SINK_CLASS_PROP); - return val; - } - - static public String getTagAdminRESTUrl(Properties prop) { - String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP); - return val; - } - - static public String getTagAdminRESTSslConfigFile(Properties prop) { - String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP); - return val; - } - - static public String getPolicyMgrUserName(Properties prop) { - String val = prop.getProperty(TAGSYNC_PM_SSL_BASICAUTH_USERNAME_PROP); - return val; - } - - static public String getPolicyMgrPassword(Properties prop) { - String val = prop.getProperty(TAGSYNC_PM_SSL_BASICAUTH_PASSWORD_PROP); - return val; - } - - static public String getTagSourceFileName(Properties prop) { - String val = prop.getProperty(TAGSYNC_SOURCE_FILE_PROP); - return val; - } - - static public String getAtlasEndpoint(Properties prop) { - String val = prop.getProperty(TAGSYNC_SOURCE_ATLAS_PROP); - return val; - } - - static public String getAtlasSslConfigFileName(Properties prop) { - return ""; - } - - static public String getServiceName(String componentName, String instanceName, Properties prop) { - String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName - + ".instance." + instanceName - + TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX; - String val = prop.getProperty(propName); - if (StringUtils.isBlank(val)) { - val = instanceName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + componentName; - } - return val; - } - - private TagSyncConfig() { - super(false); - init() ; - } - - private void init() { - readConfigFile(DEFAULT_CONFIG_FILE); - readConfigFile(CONFIG_FILE); - } - - private void readConfigFile(String fileName) { - - if (StringUtils.isNotBlank(fileName)) { - String fName = getResourceFileName(fileName); - if (StringUtils.isBlank(fName)) { - LOG.warn("Cannot find configuration file " + fileName + " in the classpath"); - } else { - LOG.info("Loading configuration from " + fName); - addResource(fileName); - } - } else { - LOG.error("Configuration fileName is null"); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java deleted file mode 100644 index 8ab2a1a..0000000 --- a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.process; - -import org.apache.commons.collections.MapUtils; -import org.apache.log4j.Logger; -import org.apache.ranger.model.TagSink; -import org.apache.ranger.model.TagSource; - -import java.util.Map; -import java.util.Properties; - -public class TagSynchronizer implements Runnable { - - private static final Logger LOG = Logger.getLogger(TagSynchronizer.class); - - private final static int MAX_INIT_RETRIES = 5; - - private boolean shutdownFlag = false; - private TagSink tagSink = null; - private TagSource tagSource = null; - private Properties properties = null; - - - public static void main(String[] args) { - - TagSyncConfig config = TagSyncConfig.getInstance(); - Properties props = config.getProperties(); - - TagSynchronizer tagSynchronizer = new TagSynchronizer(props); - - tagSynchronizer.run(); - } - - public TagSynchronizer(Properties properties) { - if (properties == null || MapUtils.isEmpty(properties)) { - LOG.error("TagSynchronizer initialized with null properties!"); - this.properties = new Properties(); - } else { - this.properties = properties; - } - } - - public TagSink getTagSink() { - return tagSink; - } - - public TagSource getTagSource() { - return tagSource; - } - - @Override - public void run() { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagSynchronizer.run()"); - } - try { - long sleepTimeBetweenCycleInMillis = TagSyncConfig.getSleepTimeInMillisBetweenCycle(properties); - - boolean initDone = initLoop(); - - if (initDone) { - - Thread tagSourceThread = tagSource.start(); - - if (tagSourceThread != null) { - while (!shutdownFlag) { - try { - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); - Thread.sleep(sleepTimeBetweenCycleInMillis); - } catch (InterruptedException e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e); - } - } - if (shutdownFlag) { - LOG.info("Interrupting tagSourceThread..."); - tagSourceThread.interrupt(); - try { - tagSourceThread.join(); - } catch (InterruptedException interruptedException) { - LOG.error("tagSourceThread.join() was interrupted"); - } - } - } else { - LOG.error("Could not start tagSource monitoring thread"); - } - } else { - LOG.error("Failed to initialize TagSynchonizer after " + MAX_INIT_RETRIES + " retries. Exiting thread"); - } - - } catch (Throwable t) { - LOG.error("tag-sync thread got an error", t); - } finally { - LOG.error("Shutting down the tag-sync thread"); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagSynchronizer.run()"); - } - } - - public boolean initLoop() { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagSynchronizer.initLoop()"); - } - boolean ret = false; - - long sleepTimeBetweenCycleInMillis = TagSyncConfig.getSleepTimeInMillisBetweenCycle(properties); - - for (int initRetries = 0; initRetries < MAX_INIT_RETRIES && !ret; initRetries++) { - - printConfigurationProperties(); - - ret = init(); - - if (!ret) { - LOG.error("Failed to initialize TAG source/sink. Will retry after " + sleepTimeBetweenCycleInMillis + " milliseconds."); - try { - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); - Thread.sleep(sleepTimeBetweenCycleInMillis); - properties = TagSyncConfig.getInstance().getProperties(); - } catch (Exception e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize tag source/sink", e); - } - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagSynchronizer.initLoop()"); - } - return ret; - } - - public boolean init() { - - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagSynchronizer.init()"); - } - boolean ret = false; - try { - LOG.info("Initializing TAG source and sink"); - // Initialize tagSink and tagSource - String tagSourceClassName = TagSyncConfig.getTagSourceClassName(properties); - String tagSinkClassName = TagSyncConfig.getTagSinkClassName(properties); - - if (LOG.isDebugEnabled()) { - LOG.debug("tagSourceClassName=" + tagSourceClassName + ", tagSinkClassName=" + tagSinkClassName); - } - - Class<TagSource> tagSourceClass = (Class<TagSource>) Class.forName(tagSourceClassName); - Class<TagSink> tagSinkClass = (Class<TagSink>) Class.forName(tagSinkClassName); - - tagSink = tagSinkClass.newInstance(); - tagSource = tagSourceClass.newInstance(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Created instance of " + tagSourceClassName + ", " + tagSinkClassName); - } - - ret = tagSink.initialize(properties) && tagSource.initialize(properties); - - tagSource.setTagSink(tagSink); - - LOG.info("Done initializing TAG source and sink"); - } catch (Throwable t) { - LOG.error("Failed to initialize TAG source/sink. Error details: ", t); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagSynchronizer.init(), result=" + ret); - } - - return ret; - } - - public void shutdown(String reason) { - LOG.info("Received shutdown(), reason=" + reason); - this.shutdownFlag = true; - } - - public void printConfigurationProperties() { - LOG.info("--------------------------------"); - LOG.info(""); - LOG.info("Ranger-TagSync Configuration: {\n"); - if (MapUtils.isNotEmpty(properties)) { - for (Map.Entry<Object, Object> entry : properties.entrySet()) { - LOG.info("\tProperty-Name:" + entry.getKey()); - LOG.info("\tProperty-Value:" + entry.getValue()); - LOG.info("\n"); - } - } - LOG.info("\n}"); - LOG.info(""); - LOG.info("--------------------------------"); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java deleted file mode 100644 index e934ce4..0000000 --- a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java +++ /dev/null @@ -1,511 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.sink.policymgr; - -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; - -import org.apache.commons.collections.MapUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.ranger.admin.client.datatype.RESTResponse; -import org.apache.ranger.model.TagSink; -import org.apache.ranger.plugin.model.*; -import org.apache.ranger.plugin.store.PList; -import org.apache.ranger.plugin.store.ServiceStore; -import org.apache.ranger.plugin.util.RangerRESTClient; -import org.apache.ranger.plugin.util.SearchFilter; -import org.apache.ranger.plugin.util.ServiceTags; -import org.apache.ranger.process.TagSyncConfig; - -import java.util.List; -import java.util.Map; -import java.util.Properties; - -public class TagRESTSink implements TagSink { - private static final Log LOG = LogFactory.getLog(TagRESTSink.class); - - private static final String REST_PREFIX = "/service"; - private static final String MODULE_PREFIX = "/tags"; - - private static final String REST_MIME_TYPE_JSON = "application/json" ; - private static final String REST_URL_TAGDEFS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagdefs/" ; - private static final String REST_URL_TAGDEF_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagdef/" ; - private static final String REST_URL_SERVICERESOURCES_RESOURCE = REST_PREFIX + MODULE_PREFIX + "resources/" ; - private static final String REST_URL_SERVICERESOURCE_RESOURCE = REST_PREFIX + MODULE_PREFIX + "resource/" ; - private static final String REST_URL_TAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tags/" ; - private static final String REST_URL_TAG_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tag/" ; - private static final String REST_URL_TAGRESOURCEMAP_IDS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagresourcemapids/"; - private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/"; - - private RangerRESTClient tagRESTClient = null; - - @Override - public void init() {} - - @Override - public boolean initialize(Properties properties) { - if(LOG.isDebugEnabled()) { - LOG.debug("==> TagRESTSink.initialize()"); - } - - boolean ret = false; - - String restUrl = TagSyncConfig.getTagAdminRESTUrl(properties); - String sslConfigFile = TagSyncConfig.getTagAdminRESTSslConfigFile(properties); - String userName = TagSyncConfig.getPolicyMgrUserName(properties); - String password = TagSyncConfig.getPolicyMgrPassword(properties); - - if (LOG.isDebugEnabled()) { - LOG.debug("restUrl=" + restUrl); - LOG.debug("sslConfigFile=" + sslConfigFile); - LOG.debug("userName=" + userName); - LOG.debug("password=" + password); - } - tagRESTClient = new RangerRESTClient(restUrl, sslConfigFile); - if (tagRESTClient != null) { - tagRESTClient.setBasicAuthInfo(userName, password); - ret = true; - } else { - LOG.error("Could not create RangerRESTClient"); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== TagRESTSink.initialize(), result=" + ret); - } - return ret; - } - - @Override - public void setServiceStore(ServiceStore svcStore) { - - } - - @Override - public RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> createTagDef(" + tagDef + ")"); - } - - RangerTagDef ret = null; - - WebResource webResource = createWebResource(REST_URL_TAGDEFS_RESOURCE); - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, tagRESTClient.toJson(tagDef)); - - if(response != null && response.getStatus() == 200) { - ret = response.getEntity(RangerTagDef.class); - } else { - LOG.error("RangerAdmin REST call returned with response={" + response +"}"); - RESTResponse resp = RESTResponse.fromClientResponse(response); - - throw new Exception(resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== createTagDef(" + tagDef + "): " + ret); - } - - return ret; - } - - @Override - public RangerTagDef updateTagDef(RangerTagDef TagDef) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public void deleteTagDefByName(String name) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public void deleteTagDef(Long id) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> deleteTagDef(" + id + ")"); - } - WebResource webResource = createWebResource(REST_URL_TAGDEF_RESOURCE + Long.toString(id)); - - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).delete(ClientResponse.class); - - if(response != null && response.getStatus() == 204) { - } else { - LOG.error("RangerAdmin REST call returned with response={" + response + "}"); - - RESTResponse resp = RESTResponse.fromClientResponse(response); - - throw new Exception(resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== deleteTagDef(" + id + ")"); - } - } - - @Override - public RangerTagDef getTagDef(Long id) throws Exception { - throw new Exception("Not implemented"); - - } - - @Override - public RangerTagDef getTagDefByGuid(String guid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public RangerTagDef getTagDefByName(String name) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTagDef> getTagDefs(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public PList<RangerTagDef> getPaginatedTagDefs(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<String> getTagTypes() throws Exception { - // TODO Auto-generated method stub - return null; - } - - - @Override - public RangerTag createTag(RangerTag tag) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> createTag(" + tag + ")"); - } - - RangerTag ret = null; - - WebResource webResource = createWebResource(REST_URL_TAGS_RESOURCE); - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, tagRESTClient.toJson(tag)); - - if(response != null && response.getStatus() == 200) { - ret = response.getEntity(RangerTag.class); - } else { - LOG.error("RangerAdmin REST call returned with response={" + response +"}"); - RESTResponse resp = RESTResponse.fromClientResponse(response); - - throw new Exception(resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== createTag(" + tag + "): " + ret); - } - - return ret; - } - - @Override - public RangerTag updateTag(RangerTag tag) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public void deleteTag(Long id) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> deleteTag(" + id + ")"); - } - WebResource webResource = createWebResource(REST_URL_TAG_RESOURCE + Long.toString(id)); - - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).delete(ClientResponse.class); - - if(response != null && response.getStatus() == 204) { - } else { - LOG.error("RangerAdmin REST call returned with response={" + response + "}"); - - RESTResponse resp = RESTResponse.fromClientResponse(response); - - throw new Exception(resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== deleteTag(" + id + ")"); - } - } - - @Override - public RangerTag getTag(Long id) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public RangerTag getTagByGuid(String guid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTag> getTagsByType(String name) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTag> getTagsForResourceGuid(String resourceGuid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTag> getTags(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public PList<RangerTag> getPaginatedTags(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - - @Override - public RangerServiceResource createServiceResource(RangerServiceResource resource) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> createServiceResource(" + resource + ")"); - } - - RangerServiceResource ret = null; - - WebResource webResource = createWebResource(REST_URL_SERVICERESOURCES_RESOURCE); - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, tagRESTClient.toJson(resource)); - - if(response != null && response.getStatus() == 200) { - ret = response.getEntity(RangerServiceResource.class); - } else { - LOG.error("RangerAdmin REST call returned with response={" + response +"}"); - - RESTResponse resp = RESTResponse.fromClientResponse(response); - - throw new Exception(resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== createServiceResource(" + resource + "): " + ret); - } - - return ret; - } - - @Override - public RangerServiceResource updateServiceResource(RangerServiceResource resource) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public void deleteServiceResource(Long id) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> deleteServiceResource(" + id + ")"); - } - WebResource webResource = createWebResource(REST_URL_SERVICERESOURCE_RESOURCE + Long.toString(id)); - - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).delete(ClientResponse.class); - - if(response != null && response.getStatus() == 204) { - } else { - LOG.error("RangerAdmin REST call returned with response={" + response + "}"); - - RESTResponse resp = RESTResponse.fromClientResponse(response); - - throw new Exception(resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== deleteServiceResource(" + id + ")"); - } - } - - @Override - public RangerServiceResource getServiceResource(Long id) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public RangerServiceResource getServiceResourceByGuid(String guid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerServiceResource> getServiceResourcesByService(String serviceName) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public RangerServiceResource getServiceResourceByResourceSignature(String resourceSignature) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerServiceResource> getServiceResources(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public PList<RangerServiceResource> getPaginatedServiceResources(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - - @Override - public RangerTagResourceMap createTagResourceMap(RangerTagResourceMap tagResourceMap) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> createTagResourceMap(" + tagResourceMap + ")"); - } - - RangerTagResourceMap ret = null; - - WebResource webResource = createWebResource(REST_URL_TAGRESOURCEMAP_IDS_RESOURCE) - .queryParam("tag-id", Long.toString(tagResourceMap.getTagId())) - .queryParam("resource-id", Long.toString(tagResourceMap.getResourceId())); - - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class); - - if(response != null && response.getStatus() == 200) { - ret = response.getEntity(RangerTagResourceMap.class); - } else { - LOG.error("RangerAdmin REST call returned with response={" + response +"}"); - - RESTResponse resp = RESTResponse.fromClientResponse(response); - - throw new Exception(resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== createTagResourceMap(" + tagResourceMap + "): " + ret); - } - - return ret; - } - - @Override - public void deleteTagResourceMap(Long id) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public void uploadServiceTags(ServiceTags serviceTags) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> uploadServiceTags()"); - } - WebResource webResource = createWebResource(REST_URL_IMPORT_SERVICETAGS_RESOURCE); - - ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).put(ClientResponse.class, tagRESTClient.toJson(serviceTags)); - - if(response != null && response.getStatus() == 204) { - } else { - LOG.error("RangerAdmin REST call returned with response={" + response + "}"); - - RESTResponse resp = RESTResponse.fromClientResponse(response); - - LOG.error("Upload of service-tags failed with message " + resp.getMessage()); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== uploadServiceTags()"); - } - } - - @Override - public RangerTagResourceMap getTagResourceMap(Long id) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public RangerTagResourceMap getTagResourceMapByGuid(String guid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTagResourceMap> getTagResourceMapsForTagId(Long tagId) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTagResourceMap> getTagResourceMapsForTagGuid(String tagGuid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTagResourceMap> getTagResourceMapsForResourceId(Long resourceId) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTagResourceMap> getTagResourceMapsForResourceGuid(String resourceGuid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public RangerTagResourceMap getTagResourceMapForTagAndResourceId(Long tagId, Long resourceId) throws Exception { - throw new Exception("Not implemented"); - } - - - @Override - public RangerTagResourceMap getTagResourceMapForTagAndResourceGuid(String tagGuid, String resourceGuid) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public List<RangerTagResourceMap> getTagResourceMaps(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - @Override - public PList<RangerTagResourceMap> getPaginatedTagResourceMaps(SearchFilter filter) throws Exception { - throw new Exception("Not implemented"); - } - - - @Override - public ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception { - throw new Exception("Not implemented"); - } - - private WebResource createWebResource(String url) { - return createWebResource(url, null); - } - - private WebResource createWebResource(String url, SearchFilter filter) { - WebResource ret = tagRESTClient.getResource(url); - - if(filter != null && !MapUtils.isEmpty(filter.getParams())) { - for(Map.Entry<String, String> e : filter.getParams().entrySet()) { - String name = e.getKey(); - String value = e.getValue(); - - ret.queryParam(name, value); - } - } - - return ret; - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44ddd597/tagsync/src/main/java/org/apache/ranger/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/source/atlas/AtlasNotificationMapper.java deleted file mode 100644 index fd92ce1..0000000 --- a/tagsync/src/main/java/org/apache/ranger/source/atlas/AtlasNotificationMapper.java +++ /dev/null @@ -1,391 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.source.atlas; - -import org.apache.atlas.notification.entity.EntityNotification; -import org.apache.atlas.typesystem.api.Entity; -import org.apache.atlas.typesystem.api.Trait; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerServiceResource; -import org.apache.ranger.plugin.model.RangerTag; -import org.apache.ranger.plugin.model.RangerTagDef; -import org.apache.ranger.plugin.util.ServiceTags; -import org.apache.ranger.process.TagSyncConfig; - -import java.util.*; - -class AtlasNotificationMapper { - private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class); - - public static final String ENTITY_TYPE_HIVE_DB = "hive_db"; - public static final String ENTITY_TYPE_HIVE_TABLE = "hive_table"; - public static final String ENTITY_TYPE_HIVE_COLUMN = "hive_column"; - - public static final String RANGER_TYPE_HIVE_DB = "database"; - public static final String RANGER_TYPE_HIVE_TABLE = "table"; - public static final String RANGER_TYPE_HIVE_COLUMN = "column"; - - public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; - public static final String QUALIFIED_NAME_FORMAT_DELIMITER_STRING = "."; - - - private static Properties properties = null; - - public static ServiceTags processEntityNotification(EntityNotification entityNotification, Properties props) { - - ServiceTags ret = null; - properties = props; - - try { - if (isEntityMappable(entityNotification.getEntity())) { - ret = createServiceTags(entityNotification); - } - } catch (Exception exception) { - LOG.error("createServiceTags() failed!! ", exception); - } - return ret; - } - - static private boolean isEntityMappable(Entity entity) { - boolean ret = false; - - String entityTypeName = entity.getTypeName(); - - if (StringUtils.isNotBlank(entityTypeName)) { - if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB) || - StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE) || - StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) { - ret = true; - } - } - return ret; - } - - static private ServiceTags createServiceTags(EntityNotification entityNotification) throws Exception { - - ServiceTags ret = null; - - EntityNotification.OperationType opType = entityNotification.getOperationType(); - Entity entity = entityNotification.getEntity(); - - String opName = entityNotification.getOperationType().name(); - switch (opType) { - case ENTITY_CREATED: { - ret = getServiceTags(entity, opType); - break; - } - case ENTITY_UPDATED: { - ret = handleEntityUpdate(entity); - break; - } - case TRAIT_ADDED: { - ret = getServiceTags(entity, opType); - break; - } - case TRAIT_DELETED: { - ret = handleTraitDelete(entity); - break; - } - default: - LOG.error("Unknown notification received. Will not be handled, notificationType=" + opName); - } - - return ret; - } - - static private ServiceTags getServiceTags(Entity entity, EntityNotification.OperationType opType) throws Exception { - ServiceTags ret = null; - - - List<RangerServiceResource> serviceResources = new ArrayList<RangerServiceResource>(); - - RangerServiceResource serviceResource = getServiceResource(entity, opType); - serviceResources.add(serviceResource); - - Map<Long, RangerTag> tags = getTags(entity); - - Map<Long, RangerTagDef> tagDefs = getTagDefs(tags, EntityNotification.OperationType.ENTITY_CREATED); - - Map<Long, List<Long>> resourceIdToTagIds = null; - - if (MapUtils.isNotEmpty(tags)) { - resourceIdToTagIds = new HashMap<Long, List<Long>>(); - - List<Long> tagList = new ArrayList<Long>(); - for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) { - tagList.add(entry.getKey()); - } - resourceIdToTagIds.put(1L, tagList); - } - - ret = new ServiceTags(); - - ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); - ret.setServiceName(serviceResource.getServiceName()); - ret.setServiceResources(serviceResources); - ret.setTagDefinitions(tagDefs); - ret.setTags(tags); - ret.setResourceToTagIds(resourceIdToTagIds); - - return ret; - } - - - static private RangerServiceResource getServiceResource(Entity entity, EntityNotification.OperationType opType) throws Exception { - - RangerServiceResource ret = null; - - Map<String, RangerPolicy.RangerPolicyResource> elements = null; - String serviceName = null; - - if (opType == EntityNotification.OperationType.ENTITY_CREATED) { - - elements = new HashMap<String, RangerPolicy.RangerPolicyResource>(); - - //String[] components = getQualifiedNameComponents(entity); - String[] components = getTempNameComponents(entity); - // components should contain qualifiedName, instanceName, dbName, tableName, columnName in that order - - - String entityTypeName = entity.getTypeName(); - - String instanceName, dbName, tableName, columnName; - - if (components.length > 1) { - instanceName = components[1]; - serviceName = getServiceName(instanceName, entityTypeName); - } - - if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) { - if (components.length > 2) { - dbName = components[2]; - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - - } else { - LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]); - } - } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) { - if (components.length > 3) { - dbName = components[2]; - tableName = components[3]; - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); - elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); - } else { - LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]); - } - } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) { - LOG.error("HIVE_COLUMN creation is not handled."); - throw new Exception("HIVE_COLUMN entity-creation not implemented"); - /* - if (components.length > 4) { - dbName = components[2]; - tableName = components[3]; - columnName = components[4]; - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); - elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); - RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(tableName); - elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource); - } else { - LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]); - } - */ - } - } - - ret = new RangerServiceResource(); - ret.setGuid(entity.getId().getGuid()); - ret.setId(1L); - ret.setServiceName(serviceName); - ret.setResourceElements(elements); - - return ret; - } - - static private Map<Long, RangerTag> getTags(Entity entity) { - Map<Long, RangerTag> ret = null; - - Map<String, ? extends Trait> traits = entity.getTraits(); - - if (MapUtils.isNotEmpty(traits)) { - ret = new HashMap<Long, RangerTag>(); - long index = 1; - - for (Map.Entry<String, ? extends Trait> entry : traits.entrySet()) { - String traitName = entry.getKey(); - Trait trait = entry.getValue(); - - Map<String, Object> attrValues = trait.getValues(); - - Map<String, String> tagAttrValues = new HashMap<String, String>(); - - for (Map.Entry<String, Object> attrValueEntry : attrValues.entrySet()) { - String attrName = attrValueEntry.getKey(); - Object attrValue = attrValueEntry.getValue(); - try { - String strValue = String.class.cast(attrValue); - tagAttrValues.put(attrName, strValue); - } catch (ClassCastException exception) { - LOG.error("Cannot cast attribute-value to String, skipping... attrName=" + attrName); - } - } - - RangerTag tag = new RangerTag(); - - tag.setGuid(entity.getId().getGuid() + "-" + traitName); - tag.setType(traitName); - tag.setAttributes(tagAttrValues); - - ret.put(index++, tag); - } - } - - return ret; - } - - static private Map<Long, RangerTagDef> getTagDefs(Map<Long, RangerTag> tags, EntityNotification.OperationType opType) { - - Map<Long, RangerTagDef> ret = null; - - if (opType == EntityNotification.OperationType.ENTITY_CREATED || opType == EntityNotification.OperationType.TRAIT_ADDED) { - if (MapUtils.isNotEmpty(tags)) { - ret = new HashMap<Long, RangerTagDef>(); - for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) { - RangerTagDef tagDef = new RangerTagDef(); - tagDef.setName(entry.getValue().getType()); - tagDef.setId(entry.getKey()); - ret.put(entry.getKey(), tagDef); - } - } - } - - return ret; - } - - static private String[] getQualifiedNameComponents(Entity entity) { - String ret[] = new String[5]; - - String qualifiedName = getAttribute(entity.getValues(), ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); - - String nameHierarchy[] = qualifiedName.split(QUALIFIED_NAME_FORMAT_DELIMITER_STRING); - - int hierarchyLevels = nameHierarchy.length; - - if (LOG.isDebugEnabled()) { - LOG.debug("----- Entity-Id:" + entity.getId().getGuid()); - LOG.debug("----- Entity-Type-Name:" + entity.getTypeName()); - LOG.debug("----- Entity-Qualified-Name:" + qualifiedName); - LOG.debug("----- Entity-Qualified-Name-Components -----"); - for (int i = 0; i < hierarchyLevels; i++) { - LOG.debug("----- Index:" + i + " Value:" + nameHierarchy[i]); - } - } - - int i; - for (i = 0; i < ret.length; i++) { - ret[i] = null; - } - ret[0] = qualifiedName; - - for (i = 0; i < hierarchyLevels; i++) { - ret[i+1] = nameHierarchy[i]; - } - return ret; - } - - static private String getServiceName(String instanceName, String entityTypeName) { - // Parse entityTypeName to get the Apache-component Name - String apacheComponents[] = entityTypeName.split("_"); - String apacheComponent = null; - if (apacheComponents.length > 0) { - apacheComponent = apacheComponents[0].toLowerCase(); - } - - return TagSyncConfig.getServiceName(apacheComponent, instanceName, properties); - } - - static private <T> T getAttribute(Map<String, Object> map, String name, Class<T> type) { - return type.cast(map.get(name)); - } - - // Temporary stuff, until qualifiedName is implemented by Atlas - static private String[] getTempNameComponents(Entity entity) { - String ret[] = new String[4]; - if (StringUtils.equals(entity.getTypeName(), ENTITY_TYPE_HIVE_DB)) { - ret[1] = getAttribute(entity.getValues(), "clusterName", String.class); - ret[2] = getAttribute(entity.getValues(), "name", String.class); - ret[3] = null; - ret[0] = ret[1] + "@" + ret[2]; - } else if (StringUtils.equals(entity.getTypeName(), ENTITY_TYPE_HIVE_TABLE)) { - String qualifiedName = getAttribute(entity.getValues(), "name", String.class); - String nameHierarchy[] = qualifiedName.split(".@"); - - int hierarchyLevels = nameHierarchy.length; - - if (LOG.isDebugEnabled()) { - LOG.debug("----- Entity-Id:" + entity.getId().getGuid()); - LOG.debug("----- Entity-Type-Name:" + entity.getTypeName()); - LOG.debug("----- Entity-Qualified-Name:" + qualifiedName); - LOG.debug("----- Entity-Qualified-Name-Components -----"); - for (int i = 0; i < hierarchyLevels; i++) { - LOG.debug("----- Index:" + i + " Value:" + nameHierarchy[i]); - } - } - - int i; - for (i = 0; i < ret.length; i++) { - ret[i] = null; - } - ret[0] = qualifiedName; - if (hierarchyLevels > 2) { - ret[1] = nameHierarchy[2]; - } - if (hierarchyLevels > 1) { - ret[2] = nameHierarchy[1]; - } - if (hierarchyLevels > 0) { - ret[3] = nameHierarchy[0]; - } - - - } - return ret; - } - - - static private ServiceTags handleEntityUpdate(Entity entity) throws Exception { - - throw new Exception("Not implemented"); - - } - - static private ServiceTags handleTraitDelete(Entity entity) throws Exception { - - throw new Exception("Not implemented"); - } -}
