Repository: incubator-ranger Updated Branches: refs/heads/master a6a0d94cb -> 1db9a5590
RANGER-807: added script to make one-time tags upload simpler Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/1db9a559 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/1db9a559 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/1db9a559 Branch: refs/heads/master Commit: 1db9a55906b1ebb539c2a742aad0beab2c78c866 Parents: a6a0d94 Author: Abhay Kulkarni <[email protected]> Authored: Mon Jan 11 17:18:43 2016 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Jan 11 18:03:42 2016 -0800 ---------------------------------------------------------------------- tagsync/scripts/ranger-tagsync-sync.sh | 66 -------- tagsync/scripts/ranger-tagsync-upload.sh | 70 ++++++++ .../ranger/tagsync/process/TagSyncConfig.java | 22 ++- .../ranger/tagsync/process/TagSynchronizer.java | 159 +++++++++---------- .../source/atlasrest/AtlasRESTTagSource.java | 26 +++ .../tagsync/source/file/FileTagSource.java | 33 ++++ 6 files changed, 222 insertions(+), 154 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/1db9a559/tagsync/scripts/ranger-tagsync-sync.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync-sync.sh b/tagsync/scripts/ranger-tagsync-sync.sh deleted file mode 100755 index f8990e3..0000000 --- a/tagsync/scripts/ranger-tagsync-sync.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -z $1 ]]; then - echo "No argument provided.." - echo "Usage: $0 {FILE | ATLAS_DOWNLOAD}" - exit -1; -fi -action=$1 -action=`echo $action | tr '[:lower:]' '[:upper:]'` -realScriptPath=`readlink -f $0` -realScriptDir=`dirname $realScriptPath` -cd $realScriptDir -cdir=`pwd` - -if [ "${action}" == "FILE" ]; then - action=file -elif [ "${action}" == "ATLAS_DOWNLOAD" ]; then - action=atlasrest -else - echo "Invalid argument [$action];" - echo "Usage: $0 {FILE | ATLAS_DOWNLOAD}" - exit -1; -fi - -if [ -f ${cdir}/conf/java_home.sh ]; then - . ${cdir}/conf/java_home.sh -fi - -for custom_env_script in `find ${cdir}/conf.dist/ -name "ranger-tagsync-env*"`; do - if [ -f $custom_env_script ]; then - . $custom_env_script - fi -done - -if [ "$JAVA_HOME" != "" ]; then - export PATH=$JAVA_HOME/bin:$PATH -fi - -logdir=/var/log/ranger/tagsync-$action - -if [ ! -d $logdir ]; then - mkdir -p $logdir - chmod 777 $logdir -fi - -cp="${cdir}/conf:${cdir}/dist/*:${cdir}/lib/*" - -cd ${cdir} -umask 0077 -java -Dproc_rangertagsync-$action ${JAVA_OPTS} -Dlogdir="${logdir}" -cp "${cp}" org.apache.ranger.tagsync.process.TagSynchronizer $action > ${logdir}/tagsync.out 2>&1 - http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/1db9a559/tagsync/scripts/ranger-tagsync-upload.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync-upload.sh b/tagsync/scripts/ranger-tagsync-upload.sh new file mode 100755 index 0000000..99f8dad --- /dev/null +++ b/tagsync/scripts/ranger-tagsync-upload.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z $1 ]]; then + echo "No argument provided.." + echo "Usage: $0 tag-source-class-name [<more-arguments>]" + exit -1; +fi +action=$1 +className= +realScriptPath=`readlink -f $0` +realScriptDir=`dirname $realScriptPath` +cd $realScriptDir +cdir=`pwd` + +if [ "${action}" == "file" ]; then + action=file + className=org.apache.ranger.tagsync.source.file.FileTagSource + +elif [ "${action}" == "atlasrest" ]; then + action=atlasrest + className=org.apache.ranger.tagsync.source.atlasrest.AtlasRESTTagSource +else + className=${action} +fi + +shift + +if [ -f ${cdir}/conf/java_home.sh ]; then + . ${cdir}/conf/java_home.sh +fi + +for custom_env_script in `find ${cdir}/conf.dist/ -name "ranger-tagsync-env*"`; do + if [ -f $custom_env_script ]; then + . $custom_env_script + fi +done + +if [ "$JAVA_HOME" != "" ]; then + export PATH=$JAVA_HOME/bin:$PATH +fi + +logdir=/var/log/ranger/tagsync-$action + +if [ ! -d $logdir ]; then + mkdir -p $logdir + chmod 777 $logdir +fi + +cp="${cdir}/conf:${cdir}/dist/*:${cdir}/lib/*" + +cd ${cdir} +umask 0077 + +java -Dproc_rangertagsync-${action} ${JAVA_OPTS} -Dlogdir="${logdir}" -cp "${cp}" ${className} $* > ${logdir}/tagsync.out 2>&1 + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/1db9a559/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index 0dfea25..b7ea2ad 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -25,6 +25,7 @@ import org.apache.log4j.Logger; import java.io.*; import java.net.URL; +import java.util.Enumeration; import java.util.Properties; import org.apache.ranger.credentialapi.CredentialReader; @@ -43,7 +44,7 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file"; - private static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.filesource.filename"; + public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.filesource.filename"; private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.filesource.modtime.check.interval"; @@ -71,13 +72,15 @@ public class TagSyncConfig extends Configuration { private static final long DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL = 900000; + private Properties props; + public static TagSyncConfig getInstance() { TagSyncConfig newConfig = new TagSyncConfig(); return newConfig; } public Properties getProperties() { - return getProps(); + return props; } public static InputStream getFileInputStream(String path) throws FileNotFoundException { @@ -289,8 +292,23 @@ public class TagSyncConfig extends Configuration { } private void init() { + readConfigFile(DEFAULT_CONFIG_FILE); readConfigFile(CONFIG_FILE); + + props = getProps(); + + @SuppressWarnings("unchecked") + Enumeration<String> propertyNames = (Enumeration<String>)props.propertyNames(); + + while (propertyNames.hasMoreElements()) { + String propertyName = propertyNames.nextElement(); + String systemPropertyValue = System.getProperty(propertyName); + if (systemPropertyValue != null) { + props.setProperty(propertyName, systemPropertyValue); + } + } + } private void readConfigFile(String fileName) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/1db9a559/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index c1a1c39..696ce5e 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -41,28 +41,20 @@ public class TagSynchronizer { TagSynchronizer tagSynchronizer = new TagSynchronizer(); TagSyncConfig config = TagSyncConfig.getInstance(); + Properties props = config.getProperties(); tagSynchronizer.setProperties(props); - String tagSourceName = null; - if (args.length > 0) { - tagSourceName = args[0]; - LOG.info("TagSource is set to " + args[0]); - } - - boolean tagSynchronizerInitialized = tagSynchronizer.initialize(tagSourceName); + boolean tagSynchronizerInitialized = tagSynchronizer.initialize(); if (tagSynchronizerInitialized) { - if (StringUtils.isNotBlank(tagSourceName)) { - tagSynchronizer.getFirstTagSource().synchUp(); - } else { - tagSynchronizer.run(); - } + tagSynchronizer.run(); } else { LOG.error("TagSynchronizer failed to initialize correctly, exiting.."); System.exit(-1); } + } public TagSynchronizer() { @@ -81,28 +73,27 @@ public class TagSynchronizer { } } - public boolean initialize(String source) { + public boolean initialize() { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagSynchronizer.initialize(" + source + ")"); + LOG.debug("==> TagSynchronizer.initialize()"); } - printConfigurationProperties(); + printConfigurationProperties(properties); boolean ret = false; - String tagSourceNames = StringUtils.isNotBlank(source) ? source : - TagSyncConfig.getTagSource(properties); + String tagSourceNames = TagSyncConfig.getTagSource(properties); if (StringUtils.isNotBlank(tagSourceNames)) { LOG.info("Initializing TAG source and sink"); - TagSink tagSink = initializeTagSink(); + TagSink tagSink = initializeTagSink(properties); if (tagSink != null) { - tagSources = initializeTagSources(tagSourceNames); + tagSources = initializeTagSources(tagSourceNames, properties); if (CollectionUtils.isNotEmpty(tagSources)) { for (TagSource tagSource : tagSources) { @@ -122,7 +113,68 @@ public class TagSynchronizer { return ret; } - public TagSink initializeTagSink() { + public void run() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagSynchronizer.run()"); + } + + long shutdownCheckIntervalInMs = 60*1000; + + boolean tagSourcesStarted = true; + + try { + for (TagSource tagSource : tagSources) { + tagSourcesStarted = tagSourcesStarted && tagSource.start(); + } + + if (tagSourcesStarted) { + while (!shutdownFlag) { + try { + LOG.debug("Sleeping for [" + shutdownCheckIntervalInMs + "] milliSeconds"); + Thread.sleep(shutdownCheckIntervalInMs); + } catch (InterruptedException e) { + LOG.error("Failed to wait for [" + shutdownCheckIntervalInMs + "] milliseconds before attempting to synchronize tag information ", e); + break; + } + } + } + } catch (Throwable t) { + LOG.error("tag-sync main thread got an error", t); + } finally { + LOG.info("Stopping all tagSources"); + + for (TagSource tagSource : tagSources) { + tagSource.stop(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagSynchronizer.run()"); + } + } + + public void shutdown(String reason) { + LOG.info("Received shutdown(), reason=" + reason); + this.shutdownFlag = true; + } + + static public void printConfigurationProperties(Properties properties) { + LOG.info("--------------------------------"); + LOG.info(""); + LOG.info("Ranger-TagSync Configuration: {\n"); + if (MapUtils.isNotEmpty(properties)) { + for (Map.Entry<Object, Object> entry : properties.entrySet()) { + LOG.info("\tProperty-Name:" + entry.getKey()); + LOG.info("\tProperty-Value:" + entry.getValue()); + LOG.info("\n"); + } + } + LOG.info("\n}"); + LOG.info(""); + LOG.info("--------------------------------"); + } + + static public TagSink initializeTagSink(Properties properties) { if (LOG.isDebugEnabled()) { LOG.debug("==> TagSynchronizer.initializeTagSink()"); } @@ -156,7 +208,7 @@ public class TagSynchronizer { return ret; } - public List<TagSource> initializeTagSources(String tagSourceNames) { + static public List<TagSource> initializeTagSources(String tagSourceNames, Properties properties) { if (LOG.isDebugEnabled()) { LOG.debug("==> TagSynchronizer.initializeTagSources(" + tagSourceNames + ")"); } @@ -203,69 +255,4 @@ public class TagSynchronizer { return ret; } - - public void run() { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagSynchronizer.run()"); - } - - long shutdownCheckIntervalInMs = 60*1000; - - boolean tagSourcesStarted = true; - - try { - for (TagSource tagSource : tagSources) { - tagSourcesStarted = tagSourcesStarted && tagSource.start(); - } - - if (tagSourcesStarted) { - while (!shutdownFlag) { - try { - LOG.debug("Sleeping for [" + shutdownCheckIntervalInMs + "] milliSeconds"); - Thread.sleep(shutdownCheckIntervalInMs); - } catch (InterruptedException e) { - LOG.error("Failed to wait for [" + shutdownCheckIntervalInMs + "] milliseconds before attempting to synchronize tag information ", e); - break; - } - } - } - } catch (Throwable t) { - LOG.error("tag-sync main thread got an error", t); - } finally { - LOG.info("Stopping all tagSources"); - - for (TagSource tagSource : tagSources) { - tagSource.stop(); - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagSynchronizer.run()"); - } - } - - public void shutdown(String reason) { - LOG.info("Received shutdown(), reason=" + reason); - this.shutdownFlag = true; - } - - public TagSource getFirstTagSource() { - return tagSources.get(0); - } - - public void printConfigurationProperties() { - LOG.info("--------------------------------"); - LOG.info(""); - LOG.info("Ranger-TagSync Configuration: {\n"); - if (MapUtils.isNotEmpty(properties)) { - for (Map.Entry<Object, Object> entry : properties.entrySet()) { - LOG.info("\tProperty-Name:" + entry.getKey()); - LOG.info("\tProperty-Value:" + entry.getValue()); - LOG.info("\n"); - } - } - LOG.info("\n}"); - LOG.info(""); - LOG.info("--------------------------------"); - } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/1db9a559/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index c8ed948..9d9f25d 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -31,7 +31,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.process.TagSyncConfig; +import org.apache.ranger.tagsync.process.TagSynchronizer; import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits; import org.apache.ranger.tagsync.source.atlas.AtlasNotificationMapper; import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil; @@ -44,6 +46,30 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { private String atlasEndpoint; private long sleepTimeBetweenCycleInMillis; + public static void main(String[] args) { + + AtlasRESTTagSource atlasRESTTagSource = new AtlasRESTTagSource(); + + TagSyncConfig config = TagSyncConfig.getInstance(); + + Properties props = config.getProperties(); + + TagSynchronizer.printConfigurationProperties(props); + + TagSink tagSink = TagSynchronizer.initializeTagSink(props); + + if (tagSink != null) { + + atlasRESTTagSource.initialize(props); + atlasRESTTagSource.setTagSink(tagSink); + atlasRESTTagSource.synchUp(); + + } else { + LOG.error("TagSink initialialization failed, exiting."); + System.exit(1); + } + + } @Override public boolean initialize(Properties properties) { if (LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/1db9a559/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java index 43eb3b5..e339d5e 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java @@ -27,7 +27,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.process.TagSyncConfig; +import org.apache.ranger.tagsync.process.TagSynchronizer; import java.io.*; import java.net.MalformedURLException; @@ -47,6 +49,37 @@ public class FileTagSource extends AbstractTagSource implements Runnable { private Properties properties; private long fileModTimeCheckIntervalInMs; + public static void main(String[] args) { + + FileTagSource fileTagSource = new FileTagSource(); + + TagSyncConfig config = TagSyncConfig.getInstance(); + + Properties props = config.getProperties(); + + if (args.length > 0) { + String tagSourceFileName = args[0]; + LOG.info("TagSourceFileName is set to " + args[0]); + props.setProperty(TagSyncConfig.TAGSYNC_FILESOURCE_FILENAME_PROP, tagSourceFileName); + } + + TagSynchronizer.printConfigurationProperties(props); + + TagSink tagSink = TagSynchronizer.initializeTagSink(props); + + if (tagSink != null) { + + fileTagSource.initialize(props); + fileTagSource.setTagSink(tagSink); + fileTagSource.synchUp(); + + } else { + LOG.error("TagSink initialialization failed, exiting."); + System.exit(1); + } + + } + @Override public boolean initialize(Properties props) {
