Repository: incubator-ranger Updated Branches: refs/heads/tag-policy 3852419c3 -> 001ec8837
RANGER-660: TagSync process implementation - initial version Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/001ec883 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/001ec883 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/001ec883 Branch: refs/heads/tag-policy Commit: 001ec88376cf51d43174aa3631e18b512bdd1333 Parents: 3852419 Author: Abhay Kulkarni <[email protected]> Authored: Tue Sep 22 15:55:56 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Sep 22 17:23:34 2015 -0700 ---------------------------------------------------------------------- pom.xml | 2 + src/main/assembly/tagsync.xml | 105 ++++ tagsync/conf.dist/log4j.xml | 42 ++ .../conf.dist/ranger-tagsync-default-site.xml | 36 ++ tagsync/pom.xml | 112 ++++ tagsync/scripts/initd | 78 +++ tagsync/scripts/ranger-tagsync-services.sh | 110 ++++ .../java/org/apache/ranger/model/TagSink.java | 35 ++ .../java/org/apache/ranger/model/TagSource.java | 52 ++ .../apache/ranger/process/TagSyncConfig.java | 326 ++++++++++++ .../apache/ranger/process/TagSynchronizer.java | 187 +++++++ .../ranger/sink/policymgr/TagRESTSink.java | 515 +++++++++++++++++++ .../ranger/source/file/TagFileSource.java | 427 +++++++++++++++ .../src/main/resources/application.properties | 29 ++ .../main/resources/etc/ranger/data/tags.json | 75 +++ .../src/main/resources/ranger-tagsync-site.xml | 67 +++ .../ranger/process/TestTagSynchronizer.java | 96 ++++ tagsync/src/test/resources/log4j.properties | 35 ++ 18 files changed, 2329 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 84ecf24..aa40882 100644 --- a/pom.xml +++ b/pom.xml @@ -96,6 +96,7 @@ <module>unixauthservice</module> <module>ranger-util</module> <module>plugin-kms</module> + <module>tagsync</module> </modules> <properties> <javac.source.version>1.7</javac.source.version> @@ -388,6 +389,7 @@ <descriptor>src/main/assembly/plugin-solr.xml</descriptor> <descriptor>src/main/assembly/admin-web.xml</descriptor> <descriptor>src/main/assembly/usersync.xml</descriptor> + <descriptor>src/main/assembly/tagsync.xml</descriptor> <descriptor>src/main/assembly/migration-util.xml</descriptor> <descriptor>src/main/assembly/kms.xml</descriptor> <descriptor>src/main/assembly/ranger-src.xml</descriptor> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/src/main/assembly/tagsync.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml new file mode 100644 index 0000000..d7262ac --- /dev/null +++ b/src/main/assembly/tagsync.xml @@ -0,0 +1,105 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>tagsync</id> + <formats> + <format>tar.gz</format> + <format>zip</format> + </formats> + <baseDirectory>${project.name}-${project.version}-tagsync</baseDirectory> + <includeBaseDirectory>true</includeBaseDirectory> + <moduleSets> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <outputDirectory>/lib</outputDirectory> + <includes> + <include>com.google.code.gson:gson</include> + <include>com.sun.jersey:jersey-bundle</include> + <include>log4j:log4j</include> + <include>commons-cli:commons-cli</include> + <include>commons-collections:commons-collections</include> + <include>commons-configuration:commons-configuration</include> + <include>commons-lang:commons-lang</include> + <include>commons-logging:commons-logging</include> + <include>com.google.guava:guava</include> + <include>org.apache.hadoop:hadoop-auth</include> + <include>org.slf4j:slf4j-api</include> + <include>org.apache.hadoop:hadoop-common</include> + <include>org.apache.commons:commons-csv</include> + <include>org.apache.ranger:credentialbuilder</include> + <include>org.apache.ranger:ranger-util</include> + <include>commons-io:commons-io:jar:${commons.io.version}</include> + <include>org.apache.htrace:htrace-core</include> + <include>org.codehaus.jackson:jackson-core-asl</include> + <include>org.codehaus.jackson:jackson-jaxrs</include> + <include>org.codehaus.jackson:jackson-mapper-asl</include> + <include>org.codehaus.jackson:jackson-xc</include> + <include>security_plugins.ranger-plugins-common:ranger-plugins-common</include> + </includes> + <unpack>false</unpack> + </dependencySet> + </dependencySets> + <outputDirectory>/dist</outputDirectory> + </binaries> + <includes> + <include>org.apache.ranger:ranger-tagsync</include> + </includes> + </moduleSet> + </moduleSets> + <fileSets> + <fileSet> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <outputDirectory>/conf.dist</outputDirectory> + <directory>tagsync/conf.dist</directory> + </fileSet> + <fileSet> + <directoryMode>755</directoryMode> + <fileMode>544</fileMode> + <outputDirectory>/</outputDirectory> + <directory>tagsync/scripts</directory> + <excludes> + <exclude>*.properties</exclude> + <exclude>initd</exclude> + </excludes> + </fileSet> + <fileSet> + <directoryMode>755</directoryMode> + <outputDirectory>/</outputDirectory> + <directory>${project.build.directory}</directory> + <includes> + <include>version</include> + </includes> + <fileMode>444</fileMode> + </fileSet> + </fileSets> + <files> + <file> + <source>tagsync/scripts/initd</source> + <outputDirectory>/</outputDirectory> + <destName>ranger-usersync.sh</destName> + <fileMode>755</fileMode> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/conf.dist/log4j.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf.dist/log4j.xml b/tagsync/conf.dist/log4j.xml new file mode 100644 index 0000000..fb6986f --- /dev/null +++ b/tagsync/conf.dist/log4j.xml @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"; debug="true"> + + <appender name="logFile" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="file" value="${logdir}/tagsync.log" /> + <param name="DatePattern" value="'.'yyyy-MM-dd" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/> + </layout> + </appender> + + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/> + </layout> + </appender> + + <root> + <priority value ="info" /> + <appender-ref ref="logFile" /> + </root> + +</log4j:configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/conf.dist/ranger-tagsync-default-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf.dist/ranger-tagsync-default-site.xml b/tagsync/conf.dist/ranger-tagsync-default-site.xml new file mode 100644 index 0000000..b098740 --- /dev/null +++ b/tagsync/conf.dist/ranger-tagsync-default-site.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>ranger.tagsync.port</name> + <value>6161</value> + </property> + <property> + <name>ranger.tagsync.ssl</name> + <value>true</value> + </property> + <property> + <name>ranger.tagsync.enabled</name> + <value>true</value> + </property> + <property> + <name>ranger.tagsync.logdir</name> + <value>./log</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/pom.xml ---------------------------------------------------------------------- diff --git a/tagsync/pom.xml b/tagsync/pom.xml new file mode 100644 index 0000000..83011d2 --- /dev/null +++ b/tagsync/pom.xml @@ -0,0 +1,112 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + <artifactId>ranger-tagsync</artifactId> + <name>Tag Synchronizer</name> + <description>Tag Synchronizer Java Process</description> + <packaging>jar</packaging> + + <parent> + <artifactId>ranger</artifactId> + <groupId>org.apache.ranger</groupId> + <version>0.5.0</version> + </parent> + + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>${log4j.version}</version> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + <version>${jersey-bundle.version}</version> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>${commons.cli.version}</version> + </dependency> + <dependency> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + <version>${commons.collections.version}</version> + </dependency> + <dependency> + <groupId>commons-configuration</groupId> + <artifactId>commons-configuration</artifactId> + <version>${commons.configuration.version}</version> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>${commons.lang.version}</version> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>${commons.logging.version}</version> + </dependency> + <dependency> + <groupId>security_plugins.ranger-plugins-common</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${project.version}</version> + </dependency> + <!-- + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <version>4.0</version> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-notification</artifactId> + <version>0.6-incubating-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-typesystem</artifactId> + <version>0.6-incubating-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-client</artifactId> + <version>0.6-incubating-SNAPSHOT</version> + </dependency> + --> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/scripts/initd ---------------------------------------------------------------------- diff --git a/tagsync/scripts/initd b/tagsync/scripts/initd new file mode 100644 index 0000000..9349fae --- /dev/null +++ b/tagsync/scripts/initd @@ -0,0 +1,78 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: ranger-tagsync +# Required-Start: $local_fs $remote_fs $network $named $syslog $time +# Required-Stop: $local_fs $remote_fs $network $named $syslog $time +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Start/Stop Ranger tagsync +### END INIT INFO + +LINUX_USER=ranger +BIN_PATH=/usr/bin +MOD_NAME=ranger-tagsync-services.sh +pidf=/var/run/ranger/tagsync.pid +pid="" +if [ -f ${pidf} ] +then + pid=`cat $pidf` +fi + +case $1 in + start) + if [ "${pid}" != "" ] + then + echo "Ranger tagsync Service is already running" + exit 1 + else + echo "Starting Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} start" + fi + ;; + stop) + if [ "${pid}" != "" ] + then + echo "Stopping Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} stop" + else + echo "Ranger tagsync Service is NOT running" + exit 1 + fi + ;; + restart) + if [ "${pid}" != "" ] + then + echo "Stopping Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} stop" + sleep 10 + fi + echo "Starting Ranger tagsync." + /bin/su --login $LINUX_USER -c "${BIN_PATH}/${MOD_NAME} start" + ;; + status) + if [ "${pid}" != "" ] + then + echo "Ranger tagsync Service is running [pid={$pid}]" + else + echo "Ranger tagsync Service is NOT running." + fi + ;; + *) + echo "Invalid argument [$1]; Only start | stop | restart | status, are supported." + exit 1 + esac http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/scripts/ranger-tagsync-services.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh new file mode 100755 index 0000000..5ca5b95 --- /dev/null +++ b/tagsync/scripts/ranger-tagsync-services.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z $1 ]]; then + echo "Invalid argument [$1];" + echo "Usage: Only start | stop | restart | version, are supported." + exit; +fi +action=$1 +action=`echo $action | tr '[:lower:]' '[:upper:]'` +realScriptPath=`readlink -f $0` +realScriptDir=`dirname $realScriptPath` +cd $realScriptDir +cdir=`pwd` + +pidf=/var/run/ranger/tagsync.pid + + +if [ "${action}" == "START" ]; then + + #Export JAVA_HOME + if [ -f ${cdir}/conf/java_home.sh ]; then + . ${cdir}/conf/java_home.sh + fi + + for custom_env_script in `find ${cdir}/conf/ -name "ranger-tagsync-env*"`; do + if [ -f $custom_env_script ]; then + . $custom_env_script + fi + done + + if [ "$JAVA_HOME" != "" ]; then + export PATH=$JAVA_HOME/bin:$PATH + fi + + logdir=/var/log/ranger/tagsync + + cp="${cdir}/dist/*:${cdir}/lib/*:${cdir}/conf" + + if [ -f $pidf ]; then + PID=`cat $pidf` + if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then + rm -f ${pidf} + else + kill -9 ${PID} > /dev/null 2>&1 + rm -f ${pidf} + echo "Ranger Tagsync Service [pid = ${PID}] has been stopped." + fi + fi + + cd ${cdir} + umask 0077 + nohup java -Dproc_rangertagsync ${JAVA_OPTS} -Dlogdir="${logdir}" -cp "${cp}" org.apache.ranger.process.TagSynchronizer > ${logdir}/tagsync.log 2>&1 & + echo $! > ${pidf} + chown ranger ${pidf} + sleep 5 + pid=`cat $pidf` + + if [ "${pid}" != "" ] + then + echo "Ranger Tagsync Service has started successfully." + else + echo "Ranger Tagsync Service failed to start. Please refer to log files under ${logdir} for further details." + fi + exit; + +elif [ "${action}" == "STOP" ]; then + + if [ -f $pidf ]; then + pidf=/var/run/ranger/tagsync.pid + PID=`cat $pidf` > /dev/null 2>&1 + kill -9 $PID > /dev/null 2>&1 + rm -f $pidf + echo "Ranger Tagsync Service [pid = ${PID}] has been stopped." + else + echo "Ranger Tagsync Service not running" + fi + + exit; + +elif [ "${action}" == "RESTART" ]; then + echo "Stopping Ranger Tagsync" + ${cdir}/ranger-tagsync-services.sh stop + echo "Starting Apache Ranger Tagsync" + ${cdir}/ranger-tagsync-services.sh start + exit; +elif [ "${action}" == "VERSION" ]; then + cd ${cdir}/lib + java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo + exit +else + echo "Invalid argument [$1];" + echo "Usage: Only start | stop | restart | version, are supported." + exit; +fi + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/java/org/apache/ranger/model/TagSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java new file mode 100644 index 0000000..74ea5cf --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.model; + +import org.apache.ranger.plugin.store.TagStore; +import org.apache.ranger.plugin.util.ServiceTags; + +import java.util.Map; +import java.util.Properties; + + +/** + * Created by akulkarni on 9/10/15. + */ +public interface TagSink extends TagStore { + boolean initialize(Properties properties); + void uploadServiceTags(ServiceTags serviceTags) throws Exception; +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/java/org/apache/ranger/model/TagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java new file mode 100644 index 0000000..568d3c7 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.model; + +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.model.RangerTagResourceMap; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Created by akulkarni on 9/10/15. + */ +public interface TagSource { + + boolean initialize(Properties properties); + + void setTagSink(TagSink sink); + + void updateSink() throws Exception; + + void start(); + + boolean isChanged(); + + List<RangerTagDef> fetchAllTagDefs(String syncSentinel) throws Exception; + + List<RangerTagDef> receiveUpdatesToTagDefs() throws Exception; + + List<RangerTagResourceMap> fetchAllTaggedEntities() throws Exception; + + List<RangerTagResourceMap> receiveUpdatesToTaggedEntities() throws Exception; + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java new file mode 100644 index 0000000..1c995b1 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.process; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.*; +import java.net.URL; +import java.util.Properties; + +/** + * Created by akulkarni on 9/11/15. + */ +public class TagSyncConfig { + private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ; + + public static final String CONFIG_FILE = "ranger-tagsync-site.xml"; + + public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default-site.xml"; + + public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled" ; + + public static final String TAGSYNC_PORT_PROP = "ranger.tagsync.port" ; + + public static final String TAGSYNC_SSL_PROP = "ranger.tagsync.ssl" ; + + public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir" ; + + private static final String TAGSYNC_PM_URL_PROP = "ranger.tagsync.policymanager.baseURL"; + + private static final String TAGSYNC_PM_SSL_CONFIG_FILE_PROP = "ranger.tagsync.policymanager.ssl.config.file"; + + private static final String TAGSYNC_PM_SSL_BASICAUTH_USERNAME_PROP = "ranger.tagsync.policymanager.basicauth.username"; + + private static final String TAGSYNC_PM_SSL_BASICAUTH_PASSWORD_PROP = "ranger.tagsync.policymanager.basicauth.password"; + + private static final String TAGSYNC_SOURCE_FILE_PROP = "ranger.tagsync.source.file"; + + private static final String TAGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PROP = "ranger.tagsync.sleeptimeinmillisbetweensynccycle"; + + private static final String TAGSYNC_SOURCE_CLASS_PROP = "ranger.tagsync.source.impl.class"; + + private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.sink.impl.class"; + + private static final String TAGSYNC_SOURCE_ATLAS_PROP = "atlas.endpoint"; + + private static volatile TagSyncConfig instance = null; + + private Properties prop = new Properties() ; + + public static TagSyncConfig getInstance() { + /* + TagSyncConfig ret = instance; + if (ret == null) { + synchronized(TagSyncConfig.class) { + if (ret == null) { + ret = instance = new TagSyncConfig(); + LOG.debug("TagSyncConfig = {" + ret + "}"); + } + } + } + */ + TagSyncConfig newConfig = new TagSyncConfig(); + newConfig.init(); + return newConfig; + } + + public Properties getProperties() { + return prop; + } + + public static InputStream getFileInputStream(String path) throws FileNotFoundException { + + InputStream ret = null; + + File f = new File(path); + + if (f.exists() && f.isFile() && f.canRead()) { + ret = new FileInputStream(f); + } else { + ret = TagSyncConfig.class.getResourceAsStream(path); + + if (ret == null) { + if (! path.startsWith("/")) { + ret = TagSyncConfig.class.getResourceAsStream("/" + path); + } + } + + if (ret == null) { + ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path) ; + if (ret == null) { + if (! path.startsWith("/")) { + ret = ClassLoader.getSystemResourceAsStream("/" + path); + } + } + } + } + + return ret; + } + + public static String getResourceFileName(String path) { + + String ret = null; + + if (StringUtils.isNotBlank(path)) { + + File f = new File(path); + + if (f.exists() && f.isFile() && f.canRead()) { + ret = path; + } else { + + URL fileURL = TagSyncConfig.class.getResource(path); + if (fileURL == null) { + if (!path.startsWith("/")) { + fileURL = TagSyncConfig.class.getResource("/" + path); + } + } + + if (fileURL == null) { + fileURL = ClassLoader.getSystemClassLoader().getResource(path); + if (fileURL == null) { + if (!path.startsWith("/")) { + fileURL = ClassLoader.getSystemClassLoader().getResource("/" + path); + } + } + } + + if (fileURL != null) { + try { + ret = fileURL.getFile(); + } catch (Exception exception) { + LOG.error(path + " is not a file", exception); + } + } else { + LOG.error("URL not found for " + path + " or no privilege for reading file " + path); + } + } + } + + return ret; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + + sb.append("CONFIG_FILE=").append(CONFIG_FILE).append(", ") + .append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append("\n"); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(outputStream); + prop.list(printStream); + printStream.close(); + sb.append(outputStream.toString()); + + return sb.toString(); + } + + static public boolean isTagSyncEnabled(Properties prop) { + String val = prop.getProperty(TAGSYNC_ENABLED_PROP); + return !(val != null && val.trim().equalsIgnoreCase("falae")); + } + + static public String getTagSyncPort(Properties prop) { + String val = prop.getProperty(TAGSYNC_PORT_PROP); + return val; + } + + static public boolean isTagSyncSsl(Properties prop) { + String val = prop.getProperty(TAGSYNC_SSL_PROP); + return (val != null && val.trim().equalsIgnoreCase("true")); + } + + static public String getTagSyncLogdir(Properties prop) { + String val = prop.getProperty(TAGSYNC_LOGDIR_PROP); + return val; + } + + static public long getSleepTimeInMillisBetweenCycle(Properties prop) { + String val = prop.getProperty(TAGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PROP); + return Long.valueOf(val); + } + + static public String getTagSourceClassName(Properties prop) { + String val = prop.getProperty(TAGSYNC_SOURCE_CLASS_PROP); + return val; + } + + static public String getTagSinkClassName(Properties prop) { + String val = prop.getProperty(TAGSYNC_SINK_CLASS_PROP); + return val; + } + + static public String getPolicyMgrUrl(Properties prop) { + String val = prop.getProperty(TAGSYNC_PM_URL_PROP); + return val; + } + + static public String getPolicyMgrSslConfigFile(Properties prop) { + String val = prop.getProperty(TAGSYNC_PM_SSL_CONFIG_FILE_PROP); + return val; + } + + static public String getPolicyMgrUserName(Properties prop) { + String val = prop.getProperty(TAGSYNC_PM_SSL_BASICAUTH_USERNAME_PROP); + return val; + } + + static public String getPolicyMgrPassword(Properties prop) { + String val = prop.getProperty(TAGSYNC_PM_SSL_BASICAUTH_PASSWORD_PROP); + return val; + } + + static public String getTagSourceFileName(Properties prop) { + String val = prop.getProperty(TAGSYNC_SOURCE_FILE_PROP); + return val; + } + + static public String getAtlasEndpoint(Properties prop) { + String val = prop.getProperty(TAGSYNC_SOURCE_ATLAS_PROP); + return val; + } + + static public String getAtlasSslConfigFileName(Properties prop) { + return ""; + } + + private TagSyncConfig() { + init() ; + } + + private void init() { + readConfigFile(CONFIG_FILE); + readConfigFile(DEFAULT_CONFIG_FILE); + } + + private void readConfigFile(String fileName) { + try { + InputStream in = getFileInputStream(fileName); + if (in != null) { + try { + DocumentBuilderFactory xmlDocumentBuilderFactory = DocumentBuilderFactory + .newInstance(); + xmlDocumentBuilderFactory.setIgnoringComments(true); + xmlDocumentBuilderFactory.setNamespaceAware(true); + DocumentBuilder xmlDocumentBuilder = xmlDocumentBuilderFactory + .newDocumentBuilder(); + Document xmlDocument = xmlDocumentBuilder.parse(in); + xmlDocument.getDocumentElement().normalize(); + + NodeList nList = xmlDocument + .getElementsByTagName("property"); + + for (int temp = 0; temp < nList.getLength(); temp++) { + + Node nNode = nList.item(temp); + + if (nNode.getNodeType() == Node.ELEMENT_NODE) { + + Element eElement = (Element) nNode; + + String propertyName = ""; + String propertyValue = ""; + if (eElement.getElementsByTagName("name").item( + 0) != null) { + propertyName = eElement + .getElementsByTagName("name") + .item(0).getTextContent().trim(); + } + if (eElement.getElementsByTagName("value") + .item(0) != null) { + propertyValue = eElement + .getElementsByTagName("value") + .item(0).getTextContent().trim(); + } + + if (prop.get(propertyName) != null) { + prop.remove(propertyName) ; + } + + prop.put(propertyName, propertyValue); + + } + } + } + finally { + try { + in.close() ; + } + catch(IOException ioe) { + // Ignore IOE when closing stream + } + } + } + } catch (Throwable e) { + throw new RuntimeException("Unable to load configuration file [" + CONFIG_FILE + "]", e) ; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java new file mode 100644 index 0000000..c7e8d6e --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.process; + +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.model.TagSink; +import org.apache.ranger.model.TagSource; + +import java.util.Properties; + +/** + * Created by akulkarni on 9/11/15. + */ +public class TagSynchronizer implements Runnable { + + private static final Logger LOG = Logger.getLogger(TagSynchronizer.class); + + private final static int MAX_INIT_RETRIES = 1; + + private boolean shutdownFlag = false; + private TagSink tagSink = null; + private TagSource tagSource = null; + private Properties properties = null; + + + public static void main(String[] args) { + + TagSyncConfig config = TagSyncConfig.getInstance(); + Properties props = config.getProperties(); + + LOG.info("--------------------------------"); + LOG.info(""); + LOG.info("Ranger-TagSync Configuration: {\n" + config + "\n}"); + LOG.info(""); + LOG.info("--------------------------------"); + + TagSynchronizer tagSynchronizer = new TagSynchronizer(props); + + tagSynchronizer.run(); + } + + public TagSynchronizer(Properties properties) { + if (properties == null || MapUtils.isEmpty(properties)) { + LOG.error("TagSynchronizer initialized with null properties!"); + this.properties = new Properties(); + } else { + this.properties = properties; + } + } + + public TagSink getTagSink() { + return tagSink; + } + + public TagSource getTagSource() { + return tagSource; + } + + @Override + public void run() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagSynchronizer.run()"); + } + try { + long sleepTimeBetweenCycleInMillis = TagSyncConfig.getSleepTimeInMillisBetweenCycle(properties); + + boolean initDone = initLoop(); + + if (initDone) { + + tagSource.start(); + + while (!shutdownFlag) { + try { + LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + Thread.sleep(sleepTimeBetweenCycleInMillis); + } catch (InterruptedException e) { + LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e); + } + } + } else { + LOG.error("Failed to initialize TagSynchonizer after " + MAX_INIT_RETRIES + " retries. Exiting thread"); + } + + } catch (Throwable t) { + LOG.error("tag-sync thread got an error", t); + } finally { + LOG.error("Shutting down the tag-sync thread"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagSynchronizer.run()"); + } + } + + public boolean initLoop() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagSynchronizer.initLoop()"); + } + boolean ret = false; + + long sleepTimeBetweenCycleInMillis = TagSyncConfig.getSleepTimeInMillisBetweenCycle(properties); + + for (int initRetries = 0; initRetries < MAX_INIT_RETRIES && !ret; initRetries++) { + + ret = init(); + + if (!ret) { + LOG.error("Failed to initialize TAG source/sink. Will retry after " + sleepTimeBetweenCycleInMillis + " milliseconds."); + try { + LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + Thread.sleep(sleepTimeBetweenCycleInMillis); + } catch (Exception e) { + LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize tag source/sink", e); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagSynchronizer.initLoop()"); + } + return ret; + } + + public boolean init() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagSynchronizer.init()"); + } + boolean ret = false; + try { + LOG.info("Initializing TAG source and sink"); + // Initialize tagSink and tagSource + String tagSourceClassName = TagSyncConfig.getTagSourceClassName(properties); + String tagSinkClassName = TagSyncConfig.getTagSinkClassName(properties); + + if (LOG.isDebugEnabled()) { + LOG.debug("tagSourceClassName=" + tagSourceClassName + ", tagSinkClassName=" + tagSinkClassName); + } + + Class<TagSource> tagSourceClass = (Class<TagSource>) Class.forName(tagSourceClassName); + Class<TagSink> tagSinkClass = (Class<TagSink>) Class.forName(tagSinkClassName); + + tagSink = tagSinkClass.newInstance(); + tagSource = tagSourceClass.newInstance(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Created instance of " + tagSourceClassName + ", " + tagSinkClassName); + } + + ret = tagSink.initialize(properties) && tagSource.initialize(properties); + + tagSource.setTagSink(tagSink); + + LOG.info("Done initializing TAG source and sink"); + } catch (Throwable t) { + LOG.error("Failed to initialize TAG source/sink. Error details: ", t); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagSynchronizer.init(), result=" + ret); + } + + return ret; + } + + public void shutdown(String reason) { + LOG.error("Received shutdown(), reason=" + reason); + this.shutdownFlag = true; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java new file mode 100644 index 0000000..0695cd3 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java @@ -0,0 +1,515 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.sink.policymgr; + +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.admin.client.datatype.RESTResponse; +import org.apache.ranger.model.TagSink; +import org.apache.ranger.plugin.model.*; +import org.apache.ranger.plugin.store.PList; +import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.process.TagSyncConfig; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Created by akulkarni on 9/11/15. + */ +public class TagRESTSink implements TagSink { + private static final Log LOG = LogFactory.getLog(TagRESTSink.class); + + private static final String REST_PREFIX = "/service"; + private static final String MODULE_PREFIX = "/tags"; + + private static final String REST_MIME_TYPE_JSON = "application/json" ; + private static final String REST_URL_TAGDEFS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagdefs/" ; + private static final String REST_URL_TAGDEF_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagdef/" ; + private static final String REST_URL_SERVICERESOURCES_RESOURCE = REST_PREFIX + MODULE_PREFIX + "resources/" ; + private static final String REST_URL_SERVICERESOURCE_RESOURCE = REST_PREFIX + MODULE_PREFIX + "resource/" ; + private static final String REST_URL_TAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tags/" ; + private static final String REST_URL_TAG_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tag/" ; + private static final String REST_URL_TAGRESOURCEMAP_IDS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagresourcemapids/"; + private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/"; + public static final String REST_URL_IMPORT_SERVICETAGS_PARAM = "op"; + + + private RangerRESTClient tagRESTClient = null; + + @Override + public void init() {} + + @Override + public boolean initialize(Properties properties) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> TagRESTSink.initialize()"); + } + + boolean ret = false; + + String restUrl = TagSyncConfig.getPolicyMgrUrl(properties); + String sslConfigFile = TagSyncConfig.getPolicyMgrSslConfigFile(properties); + String userName = TagSyncConfig.getPolicyMgrUserName(properties); + String password = TagSyncConfig.getPolicyMgrPassword(properties); + + if (LOG.isDebugEnabled()) { + LOG.debug("restUrl=" + restUrl); + LOG.debug("sslConfigFile=" + sslConfigFile); + LOG.debug("userName=" + userName); + LOG.debug("password=" + password); + } + tagRESTClient = new RangerRESTClient(restUrl, sslConfigFile); + if (tagRESTClient != null) { + tagRESTClient.setBasicAuthInfo(userName, password); + ret = true; + } else { + LOG.error("Could not create RangerRESTClient"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== TagRESTSink.initialize(), result=" + ret); + } + return ret; + } + + @Override + public void setServiceStore(ServiceStore svcStore) { + + } + + @Override + public RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> createTagDef(" + tagDef + ")"); + } + + RangerTagDef ret = null; + + WebResource webResource = createWebResource(REST_URL_TAGDEFS_RESOURCE); + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, tagRESTClient.toJson(tagDef)); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(RangerTagDef.class); + } else { + LOG.error("RangerAdmin REST call returned with response={" + response +"}"); + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== createTagDef(" + tagDef + "): " + ret); + } + + return ret; + } + + @Override + public RangerTagDef updateTagDef(RangerTagDef TagDef) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public void deleteTagDefByName(String name) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public void deleteTagDef(Long id) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> deleteTagDef(" + id + ")"); + } + WebResource webResource = createWebResource(REST_URL_TAGDEF_RESOURCE + Long.toString(id)); + + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).delete(ClientResponse.class); + + if(response != null && response.getStatus() == 204) { + } else { + LOG.error("RangerAdmin REST call returned with response={" + response + "}"); + + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== deleteTagDef(" + id + ")"); + } + } + + @Override + public RangerTagDef getTagDef(Long id) throws Exception { + throw new Exception("Not implemented"); + + } + + @Override + public RangerTagDef getTagDefByGuid(String guid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public RangerTagDef getTagDefByName(String name) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagDef> getTagDefs(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public PList<RangerTagDef> getPaginatedTagDefs(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + + @Override + public RangerTag createTag(RangerTag tag) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> createTag(" + tag + ")"); + } + + RangerTag ret = null; + + WebResource webResource = createWebResource(REST_URL_TAGS_RESOURCE); + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, tagRESTClient.toJson(tag)); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(RangerTag.class); + } else { + LOG.error("RangerAdmin REST call returned with response={" + response +"}"); + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== createTag(" + tag + "): " + ret); + } + + return ret; + } + + @Override + public RangerTag updateTag(RangerTag tag) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public void deleteTag(Long id) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> deleteTag(" + id + ")"); + } + WebResource webResource = createWebResource(REST_URL_TAG_RESOURCE + Long.toString(id)); + + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).delete(ClientResponse.class); + + if(response != null && response.getStatus() == 204) { + } else { + LOG.error("RangerAdmin REST call returned with response={" + response + "}"); + + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== deleteTag(" + id + ")"); + } + } + + @Override + public RangerTag getTag(Long id) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public RangerTag getTagByGuid(String guid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTag> getTagsByType(String name) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTag> getTagsForResourceGuid(String resourceGuid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTag> getTags(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public PList<RangerTag> getPaginatedTags(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + + @Override + public RangerServiceResource createServiceResource(RangerServiceResource resource) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> createServiceResource(" + resource + ")"); + } + + RangerServiceResource ret = null; + + WebResource webResource = createWebResource(REST_URL_SERVICERESOURCES_RESOURCE); + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, tagRESTClient.toJson(resource)); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(RangerServiceResource.class); + } else { + LOG.error("RangerAdmin REST call returned with response={" + response +"}"); + + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== createServiceResource(" + resource + "): " + ret); + } + + return ret; + } + + @Override + public RangerServiceResource updateServiceResource(RangerServiceResource resource) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public void deleteServiceResource(Long id) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> deleteServiceResource(" + id + ")"); + } + WebResource webResource = createWebResource(REST_URL_SERVICERESOURCE_RESOURCE + Long.toString(id)); + + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).delete(ClientResponse.class); + + if(response != null && response.getStatus() == 204) { + } else { + LOG.error("RangerAdmin REST call returned with response={" + response + "}"); + + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== deleteServiceResource(" + id + ")"); + } + } + + @Override + public RangerServiceResource getServiceResource(Long id) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public RangerServiceResource getServiceResourceByGuid(String guid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerServiceResource> getServiceResourcesByService(String serviceName) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public RangerServiceResource getServiceResourceByResourceSignature(String resourceSignature) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerServiceResource> getServiceResources(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public PList<RangerServiceResource> getPaginatedServiceResources(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + + @Override + public RangerTagResourceMap createTagResourceMap(RangerTagResourceMap tagResourceMap) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> createTagResourceMap(" + tagResourceMap + ")"); + } + + RangerTagResourceMap ret = null; + + WebResource webResource = createWebResource(REST_URL_TAGRESOURCEMAP_IDS_RESOURCE) + .queryParam("tag-id", Long.toString(tagResourceMap.getTagId())) + .queryParam("resource-id", Long.toString(tagResourceMap.getResourceId())); + + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(RangerTagResourceMap.class); + } else { + LOG.error("RangerAdmin REST call returned with response={" + response +"}"); + + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== createTagResourceMap(" + tagResourceMap + "): " + ret); + } + + return ret; + } + + @Override + public void deleteTagResourceMap(Long id) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public void uploadServiceTags(ServiceTags serviceTags) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> uploadServiceTags()"); + } + WebResource webResource = createWebResource(REST_URL_IMPORT_SERVICETAGS_RESOURCE); + + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).put(ClientResponse.class, tagRESTClient.toJson(serviceTags)); + + if(response != null && response.getStatus() == 204) { + } else { + LOG.error("RangerAdmin REST call returned with response={" + response + "}"); + + RESTResponse resp = RESTResponse.fromClientResponse(response); + + LOG.error("Upload of service-tags failed with message " + resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== uploadServiceTags()"); + } + } + + @Override + public RangerTagResourceMap getTagResourceMap(Long id) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public RangerTagResourceMap getTagResourceMapByGuid(String guid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagResourceMap> getTagResourceMapsForTagId(Long tagId) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagResourceMap> getTagResourceMapsForTagGuid(String tagGuid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagResourceMap> getTagResourceMapsForResourceId(Long resourceId) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagResourceMap> getTagResourceMapsForResourceGuid(String resourceGuid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public RangerTagResourceMap getTagResourceMapForTagAndResourceId(Long tagId, Long resourceId) throws Exception { + throw new Exception("Not implemented"); + } + + + @Override + public RangerTagResourceMap getTagResourceMapForTagAndResourceGuid(String tagGuid, String resourceGuid) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagResourceMap> getTagResourceMaps(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public PList<RangerTagResourceMap> getPaginatedTagResourceMaps(SearchFilter filter) throws Exception { + throw new Exception("Not implemented"); + } + + + @Override + public ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<String> getTagTypes(String serviceName) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<String> lookupTagTypes(String serviceName, String pattern) throws Exception { + throw new Exception("Not implemented"); + } + + private WebResource createWebResource(String url) { + return createWebResource(url, null); + } + + private WebResource createWebResource(String url, SearchFilter filter) { + WebResource ret = tagRESTClient.getResource(url); + + if(filter != null && !MapUtils.isEmpty(filter.getParams())) { + for(Map.Entry<String, String> e : filter.getParams().entrySet()) { + String name = e.getKey(); + String value = e.getValue(); + + ret.queryParam(name, value); + } + } + + return ret; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/java/org/apache/ranger/source/file/TagFileSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/source/file/TagFileSource.java b/tagsync/src/main/java/org/apache/ranger/source/file/TagFileSource.java new file mode 100644 index 0000000..2952edb --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/source/file/TagFileSource.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.source.file; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.model.TagSink; +import org.apache.ranger.model.TagSource; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.model.RangerTagResourceMap; +import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.process.TagSyncConfig; + +import java.io.*; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Created by akulkarni on 9/11/15. + */ +public class TagFileSource implements TagSource, Runnable { + private static final Log LOG = LogFactory.getLog(TagFileSource.class); + + public static final String CREATE_OR_UPDATE_SERVICETAGS_OP = "CREATE_OR_UPDATE"; + public static final String DELETE_SERVICETAGS_OP = "DELETE"; + + private String sourceFileName; + private long lastModifiedTimeInMillis = 0L; + + private Gson gson; + private TagSink tagSink; + private Properties properties; + + @Override + public boolean initialize(Properties properties) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.initialize()"); + } + + if (properties == null || MapUtils.isEmpty(properties)) { + LOG.error("No properties specified for TagFileSource initialization"); + this.properties = new Properties(); + } else { + this.properties = properties; + } + + boolean ret = true; + + if (ret) { + + sourceFileName = TagSyncConfig.getTagSourceFileName(properties); + + if (LOG.isDebugEnabled()) { + LOG.debug("Provided sourceFileName=" + sourceFileName); + } + + String realFileName = TagSyncConfig.getResourceFileName(sourceFileName); + if (realFileName != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Real sourceFileName=" + realFileName); + } + sourceFileName = realFileName; + } else { + LOG.error(sourceFileName + " is not a file or is not readable"); + ret = false; + } + } + + if (ret) { + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); + } catch (Throwable excp) { + LOG.fatal("failed to create GsonBuilder object", excp); + ret = false; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.initialize(): sourceFileName=" + sourceFileName + ", result=" + ret); + } + + return ret; + } + + @Override + public void setTagSink(TagSink sink) { + if (sink == null) { + LOG.error("Sink is null!!!"); + } else { + this.tagSink = sink; + } + } + + @Override + public void start() { + (new Thread(this)).start(); + } + + @Override + public void run() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.run()"); + } + long sleepTimeBetweenCycleInMillis = TagSyncConfig.getSleepTimeInMillisBetweenCycle(properties); + boolean shutdownFlag = false; + + while (!shutdownFlag) { + + try { + if (isChanged()) { + LOG.info("Begin: update tags from source==>sink"); + if (TagSyncConfig.isTagSyncEnabled(properties)) { + updateSink(); + LOG.info("End: update tags from source==>sink"); + } else { + LOG.info("Tag-sync is not enabled."); + } + } else { + LOG.debug("TagFileSource: no change found for synchronization."); + } + + LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + + Thread.sleep(sleepTimeBetweenCycleInMillis); + } + catch (InterruptedException e) { + LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e); + shutdownFlag = true; + } + catch (Throwable t) { + LOG.error("tag-sync thread got an error", t); + shutdownFlag = true; + } + + } + + LOG.error("Shutting down the Tag-file-source thread"); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.run()"); + } + } + + @Override + public void updateSink() throws Exception { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.updateSink()"); + } + ServiceTags serviceTags = readFromFile(); + + if (serviceTags != null) { + + tagSink.uploadServiceTags(serviceTags); + + } else { + LOG.error("Could not read ServiceTags from file"); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.updateSink()"); + } + } + + /* + private void createTagObjects(ServiceTags serviceTags) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.createTagObjects()"); + } + + Map<Long, RangerTagDef> tagDefsMap = serviceTags.getTagDefinitions(); + if (MapUtils.isNotEmpty(tagDefsMap)) { + for (Map.Entry<Long, RangerTagDef> entry : tagDefsMap.entrySet()) { + RangerTagDef tagDef = entry.getValue(); + try { + tagSink.createTagDef(tagDef); + } catch (Exception exception) { + // Ignore and continue + LOG.error("createTagDef failed, tagDef=" + tagDef, exception); + } + } + } + + List<RangerServiceResource> serviceResources = serviceTags.getServiceResources(); + if (CollectionUtils.isNotEmpty(serviceResources)) { + for (RangerServiceResource serviceResource : serviceResources) { + try { + tagSink.createServiceResource(serviceResource); + } catch (Exception exception) { + // Ignore and continue + LOG.error("createServiceResource failed, serviceResource=" + serviceResource, exception); + } + } + } + + Map<Long, RangerTag> tagsMap = serviceTags.getTags(); + if (MapUtils.isNotEmpty(tagsMap)) { + for (Map.Entry<Long, RangerTag> entry : tagsMap.entrySet()) { + RangerTag tag = entry.getValue(); + try { + tagSink.createTag(tag); + } catch (Exception exception) { + // Ignore and continue + LOG.error("createTag failed, tag=" + tag, exception); + } + } + } + + Map<Long, List<Long>> resourceTagIdsMap = serviceTags.getResourceToTagIds(); + if (MapUtils.isNotEmpty(resourceTagIdsMap)) { + for (Map.Entry<Long, List<Long>> entry : resourceTagIdsMap.entrySet()) { + Long resourceId = entry.getKey(); + List<Long> tagIds = entry.getValue(); + for (Long tagId : tagIds) { + RangerTagResourceMap tagResourceMap = new RangerTagResourceMap(); + tagResourceMap.setResourceId(resourceId); + tagResourceMap.setTagId(tagId); + try { + tagSink.createTagResourceMap(tagResourceMap); + } catch (Exception exception) { + LOG.error("createTagResourceMap failed, resourceId=" + resourceId + ", tagId=" + tagId, exception); + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.createTagObjects()"); + } + } + */ + /* + private void deleteTagObjects(ServiceTags serviceTags) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.deleteTagObjects()"); + } + + Map<Long, List<Long>> resourceTagIdsMap = serviceTags.getResourceToTagIds(); + if (MapUtils.isNotEmpty(resourceTagIdsMap)) { + for (Map.Entry<Long, List<Long>> entry : resourceTagIdsMap.entrySet()) { + Long resourceId = entry.getKey(); + List<Long> tagIds = entry.getValue(); + for (Long tagId : tagIds) { + try { + tagSink.deleteTagResourceMap(tagId, resourceId); + } catch (Exception exception) { + LOG.error("deleteTagResourceMap failed, resourceId=" + resourceId + ", tagId=" + tagId +")", exception); + } + } + } + } + + List<RangerServiceResource> serviceResources = serviceTags.getServiceResources(); + if (CollectionUtils.isNotEmpty(serviceResources)) { + for (RangerServiceResource serviceResource : serviceResources) { + try { + tagSink.deleteServiceResource(serviceResource.getId()); + } catch (Exception exception) { + // Ignore and continue + LOG.error("deleteServiceResource failed, serviceResource=" + serviceResource, exception); + } + } + } + + Map<Long, RangerTag> tagsMap = serviceTags.getTags(); + if (MapUtils.isNotEmpty(tagsMap)) { + for (Map.Entry<Long, RangerTag> entry : tagsMap.entrySet()) { + Long tagId = entry.getKey(); + try { + tagSink.deleteTag(tagId); + } catch (Exception exception) { + // Ignore and continue + LOG.error("deleteTag failed, tagId=" + tagId, exception); + } + } + } + + Map<Long, RangerTagDef> tagDefsMap = serviceTags.getTagDefinitions(); + if (MapUtils.isNotEmpty(tagDefsMap)) { + for (Map.Entry<Long, RangerTagDef> entry : tagDefsMap.entrySet()) { + Long tagDefId = entry.getKey(); + try { + tagSink.deleteTagDef(tagDefId); + } catch (Exception exception) { + // Ignore and continue + LOG.error("deleteTagDef failed, tagDefId=" + tagDefId, exception); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.deleteTagObjects()"); + } + } + */ + + @Override + public boolean isChanged() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.isChanged()"); + } + boolean ret = false; + + long modificationTime = getModificationTime(); + + if (modificationTime > lastModifiedTimeInMillis) { + if (LOG.isDebugEnabled()) { + Date modifiedDate = new Date(modificationTime); + Date lastModifiedDate = new Date(lastModifiedTimeInMillis); + LOG.debug("File modified at " + modifiedDate + "last-modified at " + lastModifiedDate); + } + lastModifiedTimeInMillis = modificationTime; + ret = true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.isChanged(): result=" + ret); + } + return ret; + } + + @Override + public List<RangerTagDef> fetchAllTagDefs(String syncSentinel) throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagDef> receiveUpdatesToTagDefs() throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagResourceMap> fetchAllTaggedEntities() throws Exception { + throw new Exception("Not implemented"); + } + + @Override + public List<RangerTagResourceMap> receiveUpdatesToTaggedEntities() throws Exception { + throw new Exception("Not implemented"); + } + + private ServiceTags readFromFile() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.readFromFile(): sourceFileName=" + sourceFileName); + } + + ServiceTags ret = null; + + Reader reader = null; + try { + + reader = new InputStreamReader(TagSyncConfig.getFileInputStream(sourceFileName)); + + ret = gson.fromJson(reader, ServiceTags.class); + + } + catch (FileNotFoundException exception) { + LOG.warn("Tag-source file does not exist or not readble '" + sourceFileName + "'"); + } + catch (Exception excp) { + LOG.error("failed to load service-tags from Tag-source file " + sourceFileName, excp); + } + finally { + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("error while closing opened Tag-source file " + sourceFileName, excp); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.readFromFile(): sourceFileName=" + sourceFileName); + } + + return ret; + } + + private long getModificationTime() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileSource.getLastModificationTime(): sourceFileName=" + sourceFileName); + } + long ret = 0L; + + File sourceFile = new File(sourceFileName); + + if (sourceFile.exists() && sourceFile.isFile() && sourceFile.canRead()) { + ret = sourceFile.lastModified(); + } else { + ret = new Date().getTime(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileSource.lastModificationTime(): sourceFileName=" + sourceFileName + " result=" + new Date(ret)); + } + + return ret; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/resources/application.properties ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/application.properties b/tagsync/src/main/resources/application.properties new file mode 100644 index 0000000..7c874b6 --- /dev/null +++ b/tagsync/src/main/resources/application.properties @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file is used currently to satisfy needs of Injection of EntityChangeConsumer and its +# initialization. +# +# Basic configuration required to create EntityChangeConsumer +# +atlas.notification.kafka.bootstrap.servers=ranger-tag-policy-akulkarni-1:6667 +atlas.notification.kafka.zookeeper.connect=ranger-tag-policy-akulkarni-1:2181 + +# +# These properties seem to be internal to Atlas. They probably are used for generating notifications. +atlas.notification.embedded=false +atlas.notification.kafka.acks=1 +atlas.notification.kafka.data=${sys:atlas.home}/data/kafka http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/resources/etc/ranger/data/tags.json ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/etc/ranger/data/tags.json b/tagsync/src/main/resources/etc/ranger/data/tags.json new file mode 100644 index 0000000..28e7922 --- /dev/null +++ b/tagsync/src/main/resources/etc/ranger/data/tags.json @@ -0,0 +1,75 @@ +{ + "op":"create_or_update", + "serviceName": "cl1_hive", + "tagVersion": 24, + "tagUpdateTime": "20150901-20:03:17.000-+0000", + "tagDefinitions": { + "1": { + "name": "EXPIRES_ON", + "source": "Internal", + "attributeDefs": [ + { + "name": "expiry_date", + "type": "datetime" + } + ], + "id": 1, + "guid": "1441137512654_323_77", + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin", + "createTime": "20150901-19:58:33.000-+0000", + "updateTime": "20150901-19:58:33.000-+0000" + } + }, + "tags": { + "1": { + "type": "EXPIRES_ON", + "attributes": { + "expiry_date": "2014/12/31" + }, + "id": 1, + "guid": "1441137512698_844_80", + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin", + "createTime": "20150901-19:58:33.000-+0000", + "updateTime": "20150901-20:03:17.000-+0000" + } + }, + "serviceResources": [ + { + "serviceName": "cl1_hive", + "resourceElements": { + "table": { + "values": [ + "tax_2010" + ], + "isExcludes": false, + "isRecursive": false + }, + "database": { + "values": [ + "finance" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "resourceSignature": "c1114679b35a65d28e0dca4fdffc27d6", + "id": 1, + "guid": "1441137512756_887_83", + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin", + "createTime": "20150901-19:58:33.000-+0000", + "updateTime": "20150901-19:58:33.000-+0000" + } + ], + "resourceToTagIds": { + "1": [ + 1 + ] + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/main/resources/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml new file mode 100644 index 0000000..63b9727 --- /dev/null +++ b/tagsync/src/main/resources/ranger-tagsync-site.xml @@ -0,0 +1,67 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + + +<configuration> + <property> + <name>ranger.authentication.method</name> + <value>NONE</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.policymanager.baseURL</name> + <value>http://ranger-tag-policy-akulkarni-1:6080</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.policymanager.ssl.config.file</name> + <value></value> + <description></description> + </property> + <property> + <name>ranger.tagsync.policymanager.basicauth.username</name> + <value>admin</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.policymanager.basicauth.password</name> + <value>admin</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.sleeptimeinmillisbetweensynccycle</name> + <value>60000</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.source.file</name> + <value>/etc/ranger/data/tags.txt</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.source.impl.class</name> + <value>org.apache.ranger.source.file.TagFileSource</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.sink.impl.class</name> + <value>org.apache.ranger.sink.policymgr.TagRESTSink</value> + <description></description> + </property> + <property> + <name>atlas.endpoint</name> + <value>http://ranger-tag-policy-akulkarni-1:21000/</value> + <description></description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/test/java/org/apache/ranger/process/TestTagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/test/java/org/apache/ranger/process/TestTagSynchronizer.java b/tagsync/src/test/java/org/apache/ranger/process/TestTagSynchronizer.java new file mode 100644 index 0000000..d810048 --- /dev/null +++ b/tagsync/src/test/java/org/apache/ranger/process/TestTagSynchronizer.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.process; + + +import org.apache.ranger.model.TagSource; +//import org.apache.ranger.source.atlas.TagAtlasSource; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.*; +import java.util.Properties; + +import static org.junit.Assert.*; + + +public class TestTagSynchronizer { + + private static TagSynchronizer tagSynchronizer; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + System.out.println("setUpBeforeClass() called"); + + TagSyncConfig config = TagSyncConfig.getInstance(); + + Properties props = config.getProperties(); + + System.out.println("Tester is configured with following properties--"); + System.out.println("--------------------"); + + config.toString(); + + System.out.println("--------------------"); + + tagSynchronizer = new TagSynchronizer(props); + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + System.out.println("tearDownAfterClass() called"); + + } + + @Test + public void testTagSynchronizer() { + + System.out.println("testTagSynchronizer() called"); + + //tagSynchronizer.run(); + + tagSynchronizer.shutdown("From testTagSynchronizer: time=up"); + + System.out.println("Exiting test"); + + + } + + @Test + public void testTagDownload() { + + boolean initDone = tagSynchronizer.init(); + + System.out.println("TagSynchronizer initialization result=" + initDone); + + TagSource tagSource = tagSynchronizer.getTagSource(); + + try { + //TagAtlasSource tagAtlasSource = (TagAtlasSource) tagSource; + //tagAtlasSource.printAllEntities(); + } catch (ClassCastException exception) { + System.err.println("TagSource is not of TagAtlasSource"); + } + + System.out.println("Exiting testTagDownload()"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/001ec883/tagsync/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/tagsync/src/test/resources/log4j.properties b/tagsync/src/test/resources/log4j.properties new file mode 100644 index 0000000..57118de --- /dev/null +++ b/tagsync/src/test/resources/log4j.properties @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##-- To prevent junits from cluttering the build run by default all test runs send output to null appender +log4j.appender.devnull=org.apache.log4j.varia.NullAppender +# ranger.root.logger=FATAL,devnull + +##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console +ranger.root.logger=DEBUG,console + +log4j.rootLogger=${ranger.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n
