Repository: incubator-ranger Updated Branches: refs/heads/master a3a472d99 -> cd7f5b57a
Normalized configuration property names Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/cd7f5b57 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/cd7f5b57 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/cd7f5b57 Branch: refs/heads/master Commit: cd7f5b57a465b2162f4e732f69638149eee79d7b Parents: a3a472d Author: Abhay Kulkarni <[email protected]> Authored: Fri Apr 22 13:52:33 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Sun Apr 24 00:09:59 2016 -0700 ---------------------------------------------------------------------- .../conf/templates/installprop2xml.properties | 48 +++--- .../conf/templates/ranger-tagsync-template.xml | 36 +++-- tagsync/scripts/install.properties | 74 +++++---- tagsync/scripts/setup.py | 52 +++---- tagsync/scripts/updatetagadminpassword.py | 7 +- .../ranger/tagsync/process/TagSyncConfig.java | 92 ++++------- .../ranger/tagsync/process/TagSynchronizer.java | 153 +++++++++++++------ .../source/atlas/AtlasHiveResourceMapper.java | 11 +- .../source/atlas/AtlasResourceMapper.java | 12 +- .../tagsync/source/file/FileTagSource.java | 7 +- .../main/resources/ranger-tagsync-default.xml | 33 +--- .../src/main/resources/ranger-tagsync-site.xml | 72 +++++++-- .../tagsync/process/TestTagSynchronizer.java | 37 +---- 13 files changed, 341 insertions(+), 293 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/conf/templates/installprop2xml.properties ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties index 8f0ea75..27d3203 100644 --- a/tagsync/conf/templates/installprop2xml.properties +++ b/tagsync/conf/templates/installprop2xml.properties @@ -14,33 +14,43 @@ # See the License for the specific language governing permissions and # limitations under the License. -unix_user = unix_user -unix_group = unix_group -TAG_SOURCE = TAG_SOURCE - -logdir = ranger.tagsync.logdir TAGSYNC_ENABLED = ranger.tagsync.enabled -TAGADMIN_ENDPOINT = ranger.tagsync.tagadmin.rest.url -TAGADMIN_SSL_CONFIG_FILENAME = ranger.tagsync.tagadmin.rest.ssl.config.file -#TAGADMIN_USERNAME = ranger.tagsync.tagadmin.basicauth.username -#TAGADMIN_PASSWORD = ranger.tagsync.tagadmin.basicauth.password +TAG_DEST_RANGER_ENDPOINT = ranger.tagsync.dest.ranger.endpoint +TAG_DEST_RANGER_SSL_CONFIG_FILENAME = ranger.tagsync.dest.ranger.ssl.config.filename + +TAG_SOURCE_ATLAS_ENABLED = ranger.tagsync.source.atlas + +TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = atlas.kafka.bootstrap.servers +TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = atlas.kafka.zookeeper.connect +TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = atlas.kafka.entities.group.id + +TAG_SOURCE_ATLASREST_ENABLED = ranger.tagsync.source.atlasrest + +TAG_SOURCE_ATLASREST_ENDPOINT = ranger.tagsync.source.atlasrest.endpoint +TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = ranger.tagsync.source.atlasrest.download.interval.millis -TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.tagadmin.keystore +TAG_SOURCE_FILE_ENABLED = ranger.tagsync.source.file -TAGSYNC_FILESOURCE_FILENAME = ranger.tagsync.filesource.filename -TAG_SOURCE_FILE_MOD_TIME_CHECK_INTERVAL = ranger.tagsync.filesource.modtime.check.interval +TAG_SOURCE_FILE_FILENAME = ranger.tagsync.source.file.filename +TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = ranger.tagsync.source.file.check.interval.millis -TAG_SOURCE_ATLAS_REST_URL = ranger.tagsync.atlasrestsource.endpoint -TAG_SOURCE_ATLAS_REST_DOWNLOAD_INTERVAL = ranger.tagsync.atlasrestsource.download.interval +TAG_SOURCE_NONE_ENABLED = ranger.tagsync.source.none -TAGSYNC_ATLAS_KAFKA_ENDPOINTS = atlas.kafka.bootstrap.servers -TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = atlas.kafka.zookeeper.connect -TAGSYNC_ATLAS_CONSUMER_GROUP = atlas.kafka.entities.group.id +TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.ranger.service.mapping +TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.atlas.custom.resource.mappers -TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.service.mapping -TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.source.atlas.custom.resource.mappers +TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.keystore.filename + +unix_user = unix_user +unix_group = unix_group + +logdir = ranger.tagsync.logdir tagsync_principal = ranger.tagsync.kerberos.principal tagsync_keytab = ranger.tagsync.kerberos.keytab + +# TODO - What property in ranger-tagsync-site.xml should hadoop_conf map to?? +hadoop_conf = hadoop_conf + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/conf/templates/ranger-tagsync-template.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml index d82b6d1..03bb3fa 100644 --- a/tagsync/conf/templates/ranger-tagsync-template.xml +++ b/tagsync/conf/templates/ranger-tagsync-template.xml @@ -24,53 +24,57 @@ <value></value> </property> <property> - <name>ranger.tagsync.tagadmin.rest.url</name> + <name>ranger.tagsync.dest.ranger.endpoint</name> <value></value> </property> <property> - <name>ranger.tagsync.filesource.modtime.check.interval</name> + <name>ranger.tagsync.dest.ranger.ssl.config.filename</name> <value></value> </property> <property> - <name>ranger.tagsync.atlasrestsource.download.interval</name> + <name>ranger.tagsync.source.atlas</name> <value></value> </property> <property> - <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name> + <name>ranger.tagsync.source.atlasrest</name> <value></value> </property> <property> - <name>ranger.tagsync.filesource.filename</name> + <name>ranger.tagsync.source.atlasrest.endpoint</name> <value></value> </property> <property> - <name>ranger.tagsync.source.impl.class</name> + <name>ranger.tagsync.source.atlasrest.download.interval.millis</name> <value></value> </property> <property> - <name>ranger.tagsync.tagadmin.keystore</name> + <name>ranger.tagsync.source.file</name> <value></value> </property> <property> - <name>ranger.tagsync.tagadmin.alias</name> + <name>ranger.tagsync.source.file.filename</name> <value></value> </property> <property> - <name>ranger.tagsync.tagadmin.password</name> + <name>ranger.tagsync.source.file.check.interval.millis</name> <value></value> </property> <property> - <name>ranger.tagsync.atlas.to.service.mapping</name> + <name>ranger.tagsync.source.none</name> <value></value> </property> <property> - <name>ranger.tagsync.atlasrestsource.endpoint</name> + <name>ranger.tagsync.atlas.to.ranger.service.mapping</name> <value></value> </property> <property> - <name>ranger.tagsync.source.atlas.custom.resource.mappers</name> + <name>ranger.tagsync.atlas.custom.resource.mappers</name> <value></value> - </property> + </property> + <property> + <name>ranger.tagsync.keystore.filename</name> + <value></value> + </property> <property> <name>ranger.tagsync.kerberos.keytab</name> <value></value> @@ -78,5 +82,9 @@ <property> <name>ranger.tagsync.kerberos.principal</name> <value></value> - </property> + </property> + <property> + <name>hadoop_conf</name> + <value></value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/scripts/install.properties ---------------------------------------------------------------------- diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties index 695371d..90878d8 100644 --- a/tagsync/scripts/install.properties +++ b/tagsync/scripts/install.properties @@ -12,57 +12,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - # -# This file provides list of deployment variables for the Ranger KMS Web Application # +# This file provides list of deployment variables for the Ranger KMS Web Application # # - -#User and group for the usersync process -unix_user=ranger -unix_group=ranger - # -# The file where all credential is kept in cryptic format # -TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks - -# Logs are stored in logdir -logdir = log - -# URL for TagAdmin -TAGADMIN_ENDPOINT = http://localhost:6080 - -#Set to run in kerberos environment -tagsync_principal= -tagsync_keytab= -hadoop_conf=/etc/hadoop/conf - -# SSL config file name for TagAdmin -TAGADMIN_SSL_CONFIG_FILENAME = - -# Source for tags (either 'atlas' or 'file' or 'atlasrest') +# URL for tag destination - Ranger +TAG_DEST_RANGER_ENDPOINT = http://localhost:6080 -TAG_SOURCE = atlas -TAG_SOURCE_ATLAS_REST_URL = http://localhost:21000 +# SSL config file name for HTTPS messages to tag destination - Ranger +TAG_DEST_RANGER_SSL_CONFIG_FILENAME = -# Interval for checking the source for any changes in case of TAG_SOURCE = atlasrest -TAG_SOURCE_ATLAS_REST_DOWNLOAD_INTERVAL = 900000 +TAG_SOURCE_ATLAS_ENABLED = true -# File name to be used for reading tags information if TAG_SOURCE = file +# Endpoint specifications needed by Atlas +TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = localhost:6667 +TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = localhost:2181 +TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = ranger_entities_consumer -TAGSYNC_FILESOURCE_FILENAME = /etc/ranger/data/tags.json +TAG_SOURCE_ATLASREST_ENABLED = false +TAG_SOURCE_ATLASREST_ENDPOINT = http://localhost:21000 +TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = 900000 -# Interval for checking the source for any changes in case of TAG_SOURCE = file -TAG_SOURCE_FILE_MOD_TIME_CHECK_INTERVAL = 60000 +TAG_SOURCE_FILE_ENABLED = false -# Endpoint specifications needed by Atlas +TAG_SOURCE_FILE_FILENAME = /etc/ranger/data/tags.json +TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = 60000 -TAGSYNC_ATLAS_KAFKA_ENDPOINTS = localhost:6667 -TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = localhost:2181 -TAGSYNC_ATLAS_CONSUMER_GROUP = ranger_entities_consumer +TAG_SOURCE_NONE_ENABLED = false # Mapping from Atlas hive cluster-name to Ranger service-name # this needs to be in format clusterName,componentType,serviceName;clusterName2,componentType2,serviceName2 @@ -87,4 +67,22 @@ TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING= # RangerServiceResource structures are specified here. If there are no custom mappers, # then it can be left blank -TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS= +TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS= + +# +# The file where all credential is kept in cryptic format +# +TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks + +#User and group for the tagsync process +unix_user=ranger +unix_group=ranger + +# Logs are stored in logdir +logdir = log + +#Set to run in kerberos environment +tagsync_principal= +tagsync_keytab= + +hadoop_conf=/etc/hadoop/conf http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/scripts/setup.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py index 1e74a47..4404cdc 100755 --- a/tagsync/scripts/setup.py +++ b/tagsync/scripts/setup.py @@ -68,17 +68,22 @@ logFolderPermMode = 0777 rootOwnerId = 0 initPrefixList = ['S99', 'K00'] -TAG_SOURCE_KEY = 'TAG_SOURCE' -TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAGSYNC_ATLAS_KAFKA_ENDPOINTS' -TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT' -TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAGSYNC_ATLAS_CONSUMER_GROUP' +TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS' +TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT' +TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID' TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.service.mapping' TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING = 'ranger.tagsync.atlas.' TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = '.instance.' TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING = '.ranger.service' -TAG_SOURCE_ATLAS = 'atlas' -TAG_SOURCE_ATLASREST = 'atlasrest' -TAG_SOURCE_FILE = 'file' + +TAG_SOURCE_ATLAS_ENABLED_KEY = 'TAG_SOURCE_ATLAS_ENABLED' +TAG_SOURCE_ATLAS_ENABLED = 'ranger.tagsync.source.atlas' + +TAG_SOURCE_ATLASREST_ENABLED_KEY = 'TAG_SOURCE_ATLASREST_ENABLED' +TAG_SOURCE_ATLASREST_ENABLED = 'ranger.tagsync.source.atlasrest' + +TAG_SOURCE_FILE_ENABLED_KEY = 'TAG_SOURCE_FILE_ENABLED' +TAG_SOURCE_FILE_ENABLED = 'ranger.tagsync.source.file' hadoopConfFileName = 'core-site.xml' ENV_HADOOP_CONF_FILE = "ranger-tagsync-env-hadoopconfdir.sh" @@ -218,13 +223,19 @@ def convertInstallPropsToXML(props): else: print "Direct Key not found:%s" % (k) - ret['ranger.tagsync.sink.impl.class'] = 'org.apache.ranger.tagsync.sink.tagadmin.TagAdminRESTSink' + atlasOutFile.close() - if (TAG_SOURCE_KEY in ret): - ret['ranger.tagsync.source.impl.class'] = ret[TAG_SOURCE_KEY] - del ret[TAG_SOURCE_KEY] + if (TAG_SOURCE_ATLAS_ENABLED_KEY in ret): + ret[TAG_SOURCE_ATLAS_ENABLED] = ret[TAG_SOURCE_ATLAS_ENABLED_KEY] + del ret[TAG_SOURCE_ATLAS_ENABLED_KEY] - atlasOutFile.close() + if (TAG_SOURCE_ATLASREST_ENABLED_KEY in ret): + ret[TAG_SOURCE_ATLASREST_ENABLED] = ret[TAG_SOURCE_ATLASREST_ENABLED_KEY] + del ret[TAG_SOURCE_ATLASREST_ENABLED_KEY] + + if (TAG_SOURCE_FILE_ENABLED_KEY in ret): + ret[TAG_SOURCE_FILE_ENABLED] = ret[TAG_SOURCE_FILE_ENABLED_KEY] + del ret[TAG_SOURCE_FILE_ENABLED_KEY] return ret @@ -333,18 +344,6 @@ def main(): fn = join(installTemplateDirName,templateFileName) outfn = join(confFolderName, outputFileName) - atlasOutFn = join(confFolderName, atlasApplicationPropFileName) - - atlasOutFile = file(atlasOutFn, "a") - - atlasOutFile.write("atlas.notification.embedded=false" + "\n") - atlasOutFile.write("atlas.kafka.acks=1" + "\n") - atlasOutFile.write("atlas.kafka.data=${sys:atlas.home}/data/kafka" + "\n") - atlasOutFile.write("atlas.kafka.hook.group.id=atlas" + "\n") - - atlasOutFile.close() - - if ( os.path.isdir(logFolderName) ): logStat = os.stat(logFolderName) logStat.st_uid @@ -389,15 +388,14 @@ def main(): initializeInitD() - tagsyncKSPath = mergeProps['ranger.tagsync.tagadmin.keystore'] + tagsyncKSPath = mergeProps['ranger.tagsync.keystore.filename'] if (tagsyncKSPath == ''): - mergeProps['ranger.tagsync.tagadmin.password'] = 'rangertagsync' + mergeProps['ranger.tagsync.dest.ranger.password'] = 'rangertagsync' else: tagadminPasswd = 'rangertagsync' tagadminAlias = 'tagadmin.user.password' - mergeProps['ranger.tagsync.tagadmin.alias'] = tagadminAlias updatePropertyInJCKSFile(tagsyncKSPath,tagadminAlias,tagadminPasswd) os.chown(tagsyncKSPath,ownerId,groupId) http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/scripts/updatetagadminpassword.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/updatetagadminpassword.py b/tagsync/scripts/updatetagadminpassword.py index ba0c911..445d89b 100644 --- a/tagsync/scripts/updatetagadminpassword.py +++ b/tagsync/scripts/updatetagadminpassword.py @@ -109,7 +109,7 @@ def main(): log("[I] Using Java:" + str(JAVA_BIN),"info") globalDict=import_properties_from_xml(CFG_FILE,globalDict) - TAGSYNC_KEYSTORE_FILENAME=globalDict['ranger.tagsync.tagadmin.keystore'] + TAGSYNC_KEYSTORE_FILENAME=globalDict['ranger.tagsync.keystore.filename'] log("[I] TAGSYNC_KEYSTORE_FILENAME:" + str(TAGSYNC_KEYSTORE_FILENAME),"info") TAGSYNC_TAGADMIN_ALIAS="tagadmin.user.password" TAGSYNC_TAGADMIN_PASSWORD = '' @@ -120,7 +120,7 @@ def main(): while TAGSYNC_TAGADMIN_PASSWORD == "": TAGSYNC_TAGADMIN_PASSWORD=getpass.getpass("Enter tagadmin user password:") - if TAGSYNC_KEYSTORE_FILENAME != "" or TAGSYNC_TAGADMIN_ALIAS != "" or TAGSYNC_TAGADMIN_USERNAME != "" or TAGSYNC_TAGADMIN_PASSWORD != "": + if TAGSYNC_KEYSTORE_FILENAME != "" or TAGSYNC_TAGADMIN_USERNAME != "" or TAGSYNC_TAGADMIN_PASSWORD != "": log("[I] Storing tagadmin tagsync password in credential store:","info") cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s -provider jceks://file%s" %(JAVA_BIN,TAGSYNC_TAGADMIN_ALIAS,TAGSYNC_TAGADMIN_PASSWORD,TAGSYNC_KEYSTORE_FILENAME) ret=subprocess.call(shlex.split(cmd)) @@ -129,8 +129,7 @@ def main(): ret=subprocess.call(shlex.split(cmd)) if ret == 0: if os.path.isfile(CFG_FILE): - write_properties_to_xml(CFG_FILE,"ranger.tagsync.tagadmin.keystore",TAGSYNC_KEYSTORE_FILENAME) - write_properties_to_xml(CFG_FILE,"ranger.tagsync.tagadmin.alias", TAGSYNC_TAGADMIN_ALIAS) + write_properties_to_xml(CFG_FILE,"ranger.tagsync.keystore.filename",TAGSYNC_KEYSTORE_FILENAME) else: log("[E] Required file not found: ["+CFG_FILE+"]","error") else: http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index b547b8f..d14a9ce 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -30,63 +30,58 @@ import java.net.UnknownHostException; import java.util.Enumeration; import java.util.Properties; -//import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.credentialapi.CredentialReader; public class TagSyncConfig extends Configuration { private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ; - public static final String CONFIG_FILE = "ranger-tagsync-site.xml"; + private static final String CONFIG_FILE = "ranger-tagsync-site.xml"; - public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml"; + private static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml"; + + private static final String CORE_SITE_FILE = "core-site.xml"; public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled" ; public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir" ; - private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.tagadmin.rest.url"; + private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.dest.ranger.endpoint"; - private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file"; + private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.dest.ranger.ssl.config.filename"; - public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.filesource.filename"; + private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.dest.ranger.impl.class"; - private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.filesource.modtime.check.interval"; + private static final String TAGSYNC_DEST_RANGER_PASSWORD_ALIAS = "tagadmin.user.password"; - private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.atlasrestsource.download.interval"; + private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.dest.ranger.password"; - private static final String TAGSYNC_SOURCE_CLASS_PROP = "ranger.tagsync.source.impl.class"; + private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.dest.ranger.connection.check.interval"; - private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.sink.impl.class"; + private static final String TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS_PROP = "ranger.tagsync.atlas.custom.resource.mappers"; - private static final String TAGSYNC_ATLASSOURCE_ENDPOINT_PROP = "ranger.tagsync.atlasrestsource.endpoint"; + private static final String TAGSYNC_ATLASSOURCE_ENDPOINT_PROP = "ranger.tagsync.source.atlasrest.endpoint"; - public static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas."; + private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.source.atlasrest.download.interval.millis"; - public static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service"; + public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.source.file.filename"; - public static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_"; + private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.source.file.check.interval.millis"; - private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.tagadmin.keystore"; - private static final String TAGSYNC_TAGADMIN_ALIAS_PROP = "ranger.tagsync.tagadmin.alias"; - private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.tagadmin.password"; - private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync"; - private static final String TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS_PROP = "ranger.tagsync.source.atlas.custom.resource.mappers"; - - private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000; + private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.keystore.filename"; - private static final long DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL = 900000; - - private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.tagadmin.connection.check.interval"; + private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync"; private static final int DEFAULT_TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL = 2000; + private static final long DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL = 900000; + private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000; private static final String AUTH_TYPE = "hadoop.security.authentication"; private static final String NAME_RULES = "hadoop.security.auth_to_local"; private static final String TAGSYNC_KERBEROS_PRICIPAL = "ranger.tagsync.kerberos.principal"; private static final String TAGSYNC_KERBEROS_KEYTAB = "ranger.tagsync.kerberos.keytab"; - private static final String CORE_SITE_FILE = "core-site.xml"; private static String LOCAL_HOSTNAME = "unknown" ; + private Properties props; static { @@ -98,8 +93,7 @@ public class TagSyncConfig extends Configuration { } public static TagSyncConfig getInstance() { - TagSyncConfig newConfig = new TagSyncConfig(); - return newConfig; + return new TagSyncConfig(); } public Properties getProperties() { @@ -195,8 +189,7 @@ public class TagSyncConfig extends Configuration { } static public String getTagSyncLogdir(Properties prop) { - String val = prop.getProperty(TAGSYNC_LOGDIR_PROP); - return val; + return prop.getProperty(TAGSYNC_LOGDIR_PROP); } static public long getTagSourceFileModTimeCheckIntervalInMillis(Properties prop) { @@ -225,47 +218,28 @@ public class TagSyncConfig extends Configuration { return ret; } - static public String getTagSourceClassName(String sourceName) { - if (StringUtils.equalsIgnoreCase(sourceName, "atlas")) { - return "org.apache.ranger.tagsync.source.atlas.AtlasTagSource"; - } else if (StringUtils.equalsIgnoreCase(sourceName, "file")) { - return "org.apache.ranger.tagsync.source.file.FileTagSource"; - } else if (StringUtils.equalsIgnoreCase(sourceName, "atlasrest")) { - return "org.apache.ranger.tagsync.source.atlasrest.AtlasRESTTagSource"; - } else - return sourceName; - } - - static public String getTagSource(Properties prop) { - return prop.getProperty(TAGSYNC_SOURCE_CLASS_PROP); - } - static public String getTagSinkClassName(Properties prop) { String val = prop.getProperty(TAGSYNC_SINK_CLASS_PROP); - if (StringUtils.equalsIgnoreCase(val, "tagadmin")) { + if (StringUtils.equalsIgnoreCase(val, "ranger")) { return "org.apache.ranger.tagsync.sink.tagadmin.TagAdminRESTSink"; } else return val; } static public String getTagAdminRESTUrl(Properties prop) { - String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP); - return val; + return prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP); } static public String getTagAdminRESTSslConfigFile(Properties prop) { - String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP); - return val; + return prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP); } static public String getTagSourceFileName(Properties prop) { - String val = prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP); - return val; + return prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP); } static public String getAtlasEndpoint(Properties prop) { - String val = prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP); - return val; + return prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP); } static public String getTagAdminPassword(Properties prop) { @@ -277,18 +251,16 @@ public class TagSyncConfig extends Configuration { return password; } } - if (prop != null && prop.containsKey(TAGSYNC_TAGADMIN_KEYSTORE_PROP) && prop.containsKey(TAGSYNC_TAGADMIN_ALIAS_PROP)) { + if (prop != null && prop.containsKey(TAGSYNC_TAGADMIN_KEYSTORE_PROP)) { String path = prop.getProperty(TAGSYNC_TAGADMIN_KEYSTORE_PROP); - String alias = prop.getProperty(TAGSYNC_TAGADMIN_ALIAS_PROP, "tagadmin.user.password"); - if (path != null && alias != null) { - if (!path.trim().isEmpty() && !alias.trim().isEmpty()) { + if (path != null) { + if (!path.trim().isEmpty()) { try { - password = CredentialReader.getDecryptedString(path.trim(), alias.trim()); + password = CredentialReader.getDecryptedString(path.trim(), TAGSYNC_DEST_RANGER_PASSWORD_ALIAS); } catch (Exception ex) { password = null; } if (password != null && !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")) { - prop.setProperty(TAGSYNC_TAGADMIN_PASSWORD_PROP, password); return password; } } @@ -321,7 +293,7 @@ public class TagSyncConfig extends Configuration { // return prop.getProperty(TAGSYNC_KERBEROS_PRICIPAL); String principal = null; try { - return SecureClientLogin.getPrincipal(prop.getProperty(TAGSYNC_KERBEROS_PRICIPAL, ""), LOCAL_HOSTNAME); + principal = SecureClientLogin.getPrincipal(prop.getProperty(TAGSYNC_KERBEROS_PRICIPAL, ""), LOCAL_HOSTNAME); } catch (IOException ignored) { // do nothing } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index 02fbc2c..4436a77 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -35,6 +35,9 @@ public class TagSynchronizer { private List<TagSource> tagSources; private Properties properties = null; + private static final String TAGSYNC_SOURCE_BASE = "ranger.tagsync.source."; + private static final String PROP_CLASS_NAME = "class"; + private final Object shutdownNotifier = new Object(); private volatile boolean isShutdownInProgress = false; @@ -90,29 +93,22 @@ public class TagSynchronizer { boolean ret = false; - String tagSourceNames = TagSyncConfig.getTagSource(properties); - - if (StringUtils.isNotBlank(tagSourceNames)) { - - LOG.info("Initializing TAG source and sink"); + LOG.info("Initializing TAG source and sink"); - tagSink = initializeTagSink(properties); + tagSink = initializeTagSink(properties); - if (tagSink != null) { + if (tagSink != null) { - tagSources = initializeTagSources(tagSourceNames, properties); + tagSources = initializeTagSources(properties); - for (TagSource tagSource : tagSources) { - tagSource.setTagSink(tagSink); - } - ret = true; + for (TagSource tagSource : tagSources) { + tagSource.setTagSink(tagSink); } - } else { - LOG.error("'ranger.tagsync.source.impl.class' value is not specified or is empty!"); + ret = true; } if (LOG.isDebugEnabled()) { - LOG.debug("<== TagSynchronizer.initialize(" + tagSourceNames + ") : " + ret); + LOG.debug("<== TagSynchronizer.initialize() : " + ret); } return ret; @@ -184,7 +180,6 @@ public class TagSynchronizer { if (LOG.isDebugEnabled()) { LOG.debug("==> TagSynchronizer.initializeTagSink()"); } - TagSink ret = null; try { @@ -200,7 +195,6 @@ public class TagSynchronizer { if (!ret.initialize(properties)) { LOG.error("Failed to initialize TAG sink " + tagSinkClassName); - ret = null; } } catch (Throwable t) { @@ -209,58 +203,127 @@ public class TagSynchronizer { } if (LOG.isDebugEnabled()) { - LOG.debug("<== TagSynchronizer.initializeTagSink()"); + LOG.debug("<== TagSynchronizer.initializeTagSink(), result:" + (ret == null ? "false" : "true")); } return ret; } - static public List<TagSource> initializeTagSources(String tagSourceNames, Properties properties) { + static public List<TagSource> initializeTagSources(Properties properties) { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagSynchronizer.initializeTagSources(" + tagSourceNames + ")"); + LOG.debug("==> TagSynchronizer.initializeTagSources()"); } List<TagSource> ret = new ArrayList<TagSource>(); - String[] tagSourcesArray = tagSourceNames.split(","); - - List<String> tagSourceList = Arrays.asList(tagSourcesArray); - - try { - for (String tagSourceName : tagSourceList) { + List<String> tagSourceNameList = new ArrayList<String>(); - String tagSourceClassName = TagSyncConfig.getTagSourceClassName(tagSourceName.trim()); - if (LOG.isDebugEnabled()) { - LOG.debug("tagSourceClassName=" + tagSourceClassName); - } + for (Object propNameObj : properties.keySet()) { + String propName = propNameObj.toString(); + if (!propName.startsWith(TAGSYNC_SOURCE_BASE)) { + continue; + } + String tagSourceName = propName.substring(TAGSYNC_SOURCE_BASE.length()); + List<String> splits = toArray(tagSourceName, "."); + if (splits.size() > 1) { + continue; + } + String value = properties.getProperty(propName); + if (value.equalsIgnoreCase("enable") + || value.equalsIgnoreCase("enabled") + || value.equalsIgnoreCase("true")) { + tagSourceNameList.add(tagSourceName); + LOG.info("Tag source " + propName + " is set to " + + value); + } + } - if (!StringUtils.equalsIgnoreCase(tagSourceClassName, TagSource.TAG_SOURCE_NONE)) { - @SuppressWarnings("unchecked") - Class<TagSource> tagSourceClass = (Class<TagSource>) Class.forName(tagSourceClassName); - TagSource tagSource = tagSourceClass.newInstance(); + List<String> initializedTagSourceNameList = new ArrayList<String>(); - if (LOG.isDebugEnabled()) { - LOG.debug("Created instance of " + tagSourceClassName); - } + for (String tagSourceName : tagSourceNameList) { + String tagSourcePropPrefix = TAGSYNC_SOURCE_BASE + tagSourceName; + TagSource tagSource = getTagSourceFromConfig(properties, + tagSourcePropPrefix, tagSourceName); + if (tagSource != null) { + try { if (!tagSource.initialize(properties)) { - LOG.error("Failed to initialize TAG source " + tagSourceClassName); - + LOG.error("Failed to initialize TAG source " + tagSourceName); ret.clear(); break; + } else { + ret.add(tagSource); + initializedTagSourceNameList.add(tagSourceName); } - ret.add(tagSource); + } catch(Exception exception) { + LOG.error("tag-source:" + tagSourceName + " initialization failed with ", exception); + ret.clear(); + break; } } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagSynchronizer.initializeTagSources(" + initializedTagSourceNameList + ")"); + } + return ret; + } + + static private TagSource getTagSourceFromConfig(Properties props, + String propPrefix, String tagSourceName) { + TagSource tagSource = null; + String className = getStringProperty(props, propPrefix + "." + + PROP_CLASS_NAME); + if (StringUtils.isBlank(className)) { + if (tagSourceName.equals("file")) { + className = "org.apache.ranger.tagsync.source.file.FileTagSource"; + } else if (tagSourceName.equalsIgnoreCase("atlas")) { + className = "org.apache.ranger.tagsync.source.atlas.AtlasTagSource"; + } else if (tagSourceName.equals("atlasrest")) { + className = "org.apache.ranger.tagsync.source.atlasrest.AtlasRESTTagSource"; + } else if (!tagSourceName.equalsIgnoreCase(TagSource.TAG_SOURCE_NONE)) { + LOG.error("tagSource name doesn't have any class associated with it. tagSourceName=" + + tagSourceName + ", propertyPrefix=" + propPrefix); + } + } + if (StringUtils.isNotBlank(className)) { + try { + @SuppressWarnings("unchecked") + Class<TagSource> tagSourceClass = (Class<TagSource>) Class.forName(className); - } catch (Throwable t) { - LOG.error("Failed to initialize TAG sources. Error details: ", t); - ret.clear(); + tagSource = tagSourceClass.newInstance(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created instance of " + className); + } + } catch (Exception e) { + LOG.fatal("Can't instantiate tagSource class for tagSourceName=" + + tagSourceName + ", className=" + className + + ", propertyPrefix=" + propPrefix, e); + } } + return tagSource; + } - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagSynchronizer.initializeTagSources(" + tagSourceNames + ")"); + private static String getStringProperty(Properties props, String propName) { + String ret = null; + + if (props != null && propName != null) { + String val = props.getProperty(propName); + if (val != null) { + ret = val; + } } return ret; } + + private static List<String> toArray(String destListStr, String delim) { + List<String> list = new ArrayList<String>(); + if (destListStr != null && !destListStr.isEmpty()) { + StringTokenizer tokenizer = new StringTokenizer(destListStr, + delim.trim()); + while (tokenizer.hasMoreTokens()) { + list.add(tokenizer.nextToken()); + } + } + return list; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java index 9a6fc13..a318f3b 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java @@ -25,7 +25,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceResource; -import org.apache.ranger.tagsync.process.TagSyncConfig; import java.util.Map; import java.util.HashMap; @@ -48,11 +47,13 @@ public class AtlasHiveResourceMapper extends AtlasResourceMapper { public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME_FOR_HIVE_TABLE = "name"; - static protected final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; + public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; - private static String clusterDelimiter = "@"; + public static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_"; - private static String qualifiedNameDelimiter = "\\."; + public static final String clusterDelimiter = "@"; + + public static final String qualifiedNameDelimiter = "\\."; public static final String[] supportedEntityTypes = { ENTITY_TYPE_HIVE_DB, ENTITY_TYPE_HIVE_TABLE, ENTITY_TYPE_HIVE_COLUMN }; @@ -139,7 +140,7 @@ public class AtlasHiveResourceMapper extends AtlasResourceMapper { String ret = getRangerServiceName(COMPONENT_NAME, clusterName); if (StringUtils.isBlank(ret)) { - ret = clusterName + TagSyncConfig.TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + COMPONENT_NAME; + ret = clusterName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + COMPONENT_NAME; } return ret; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java index fd94928..d47d052 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java @@ -24,7 +24,6 @@ import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerServiceResource; -import org.apache.ranger.tagsync.process.TagSyncConfig; import java.util.Properties; import java.util.List; @@ -33,6 +32,11 @@ import java.util.Map; public abstract class AtlasResourceMapper { private static final Log LOG = LogFactory.getLog(AtlasResourceMapper.class); + protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas."; + + protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service"; + + protected static final String TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = ".instance."; protected Properties properties; public AtlasResourceMapper() { @@ -48,9 +52,9 @@ public abstract class AtlasResourceMapper { protected String getRangerServiceName(String componentName, String atlasInstanceName) { - String propName = TagSyncConfig.TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName - + ".instance." + atlasInstanceName - + TagSyncConfig.TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX; + String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName + + TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + atlasInstanceName + + TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX; return properties.getProperty(propName); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java index e22681e..5f444b8 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java @@ -46,7 +46,6 @@ public class FileTagSource extends AbstractTagSource implements Runnable { private long lastModifiedTimeInMillis = -1L; private Gson gsonBuilder; - private Properties properties; private long fileModTimeCheckIntervalInMs; private Thread myThread = null; @@ -99,11 +98,13 @@ public class FileTagSource extends AbstractTagSource implements Runnable { LOG.debug("==> FileTagSource.initialize()"); } + Properties properties; + if (props == null || MapUtils.isEmpty(props)) { LOG.error("No properties specified for FileTagSource initialization"); - this.properties = new Properties(); + properties = new Properties(); } else { - this.properties = props; + properties = props; } gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/resources/ranger-tagsync-default.xml ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/ranger-tagsync-default.xml b/tagsync/src/main/resources/ranger-tagsync-default.xml index 3784df8..131e319 100644 --- a/tagsync/src/main/resources/ranger-tagsync-default.xml +++ b/tagsync/src/main/resources/ranger-tagsync-default.xml @@ -14,8 +14,6 @@ limitations under the License. See accompanying LICENSE file. --> -<!-- Put site-specific property overrides in this file. --> - <configuration> <property> <name>ranger.tagsync.enabled</name> @@ -26,38 +24,17 @@ <value>log</value> </property> <property> - <name>ranger.tagsync.tagadmin.rest.url</name> - <value></value> + <name>ranger.tagsync.dest.ranger.impl.class</name> + <value>ranger</value> <description></description> </property> <property> - <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name> - <value></value> + <name>ranger.tagsync.dest.ranger.endpoint</name> + <value>http://localhost:6080</value> <description></description> </property> <property> - <name>ranger.tagsync.filesource.modtime.check.interval</name> - <value>60000</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.filesource.filename</name> + <name>ranger.tagsync.dest.ranger.password</name> <value></value> - <description></description> - </property> - <property> - <name>ranger.tagsync.source.impl.class</name> - <value></value> - <description></description> - </property> - <property> - <name>ranger.tagsync.sink.impl.class</name> - <value>tagadmin</value> - <description></description> - </property> - <property> - <name>ranger.tagsync.atlasrestsource.download.interval</name> - <value>600000</value> - <description></description> </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/resources/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml index 44a9641..fe300b4 100644 --- a/tagsync/src/main/resources/ranger-tagsync-site.xml +++ b/tagsync/src/main/resources/ranger-tagsync-site.xml @@ -19,35 +19,77 @@ <configuration> <property> <name>ranger.tagsync.enabled</name> - <value>true</value> + <value /> </property> <property> <name>ranger.tagsync.logdir</name> <value>log</value> </property> <property> - <name>ranger.tagsync.tagadmin.rest.url</name> - <value>localhost:6080</value> - <description></description> + <name>ranger.tagsync.dest.ranger.endpoint</name> + <value>http://localhost:6080</value> </property> <property> - <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name> - <value></value> - <description></description> + <name>ranger.tagsync.dest.ranger.ssl.config.filename</name> + <value /> </property> <property> - <name>ranger.tagsync.filesource.modtime.check.interval</name> - <value>60000</value> - <description></description> + <name>ranger.tagsync.source.atlas</name> + <value>false</value> + </property> + <property> + <name>ranger.tagsync.source.atlasrest</name> + <value>false</value> + </property> + <property> + <name>ranger.tagsync.source.atlasrest.endpoint</name> + <value>http://localhost:21000</value> + </property> + <property> + <name>ranger.tagsync.source.atlasrest.download.interval.millis</name> + <value>900000</value> </property> <property> - <name>ranger.tagsync.filesource.filename</name> + <name>ranger.tagsync.source.file</name> + <value>false</value> + </property> + <property> + <name>ranger.tagsync.source.file.filename</name> <value>/etc/ranger/data/tags.json</value> - <description></description> </property> <property> - <name>ranger.tagsync.source.impl.class</name> - <value>File</value> - <description></description> + <name>ranger.tagsync.source.file.check.interval.millis</name> + <value>60000</value> + </property> + <property> + <name>ranger.tagsync.source.none</name> + <value>false</value> + </property> + <property> + <name>ranger.tagsync.atlas.to.ranger.service.mapping</name> + <value /> + </property> + <property> + <name>ranger.tagsync.atlas.custom.resource.mappers</name> + <value /> + </property> + <property> + <name>ranger.tagsync.keystore.filename</name> + <value>/etc/ranger/tagsync/conf/rangertagsync.jceks</value> + </property> + <property> + <name>ranger.tagsync.kerberos.keytab</name> + <value /> </property> + <property> + <name>ranger.tagsync.kerberos.principal</name> + <value /> + </property> + <property> + <name>ranger.tagsync.dest.ranger.password</name> + <value>rangertagsync</value> + </property> + + + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java index 43c22d0..c644e0d 100644 --- a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java +++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java @@ -60,43 +60,18 @@ public class TestTagSynchronizer { System.out.println("testTagSynchronizer() called"); - //tagSynchronizer.run(); - - tagSynchronizer.shutdown("From testTagSynchronizer: time=up"); - - System.out.println("Exiting test"); - - - } - - @Test - public void testTagDownload() { - - boolean initDone = true; - - /* For tagSynchronizer.initialize() to succeed, edit ranger-tagsync-site.xml file to contain correct - values of the following properties: - ranger.tagsync.tagadmin.rest.url, ranger.tagsync.tagadmin.password - - For example: - <property> - <name>ranger.tagsync.tagadmin.rest.url</name> - <value>http://tagsync-test:6080</value> - </property> - <property> - <name>ranger.tagsync.tagadmin.password</name> - <value>rangertagsync</value> - </property> - */ + boolean initDone = tagSynchronizer.initialize(); + System.out.println("TagSynchronizer initialization result=" + initDone); -// initDone = tagSynchronizer.initialize(null); + if (initDone) { + tagSynchronizer.shutdown("From testTagSynchronizer: time=up"); + } - System.out.println("TagSynchronizer initialization result=" + initDone); + System.out.println("Exiting test"); assert(initDone); - System.out.println("Exiting testTagDownload()"); } @Test
