Repository: ranger Updated Branches: refs/heads/master a6c9065c6 -> d9ace0393
RANGER-2084: Support for service-specific audit configuration in Ranger plugin Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/d9ace039 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/d9ace039 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/d9ace039 Branch: refs/heads/master Commit: d9ace0393d1226c13848f8d17a40bac13a7d6390 Parents: a6c9065 Author: Abhay Kulkarni <akulka...@hortonworks.com> Authored: Wed Apr 25 17:22:08 2018 -0700 Committer: Abhay Kulkarni <akulka...@hortonworks.com> Committed: Wed Apr 25 17:22:08 2018 -0700 ---------------------------------------------------------------------- .../audit/provider/AuditProviderFactory.java | 10 +--- .../StandAloneAuditProviderFactory.java | 46 +++++++++++++++ .../apache/ranger/audit/test/TestEvents.java | 5 +- .../hadoop/config/RangerConfiguration.java | 32 ++-------- .../plugin/audit/RangerDefaultAuditHandler.java | 7 ++- .../ranger/plugin/service/RangerBasePlugin.java | 61 +++++++++++++++++++- .../plugin/policyengine/TestPolicyEngine.java | 5 +- .../solr/authorizer/RangerSolrAuthorizer.java | 5 +- .../org/apache/ranger/audit/TestAuditQueue.java | 8 +-- 9 files changed, 131 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index 43107ba..f858efe 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -76,7 +76,7 @@ public class AuditProviderFactory { private boolean mInitDone = false; private JVMShutdownHook jvmShutdownHook = null; - private AuditProviderFactory() { + public AuditProviderFactory() { LOG.info("AuditProviderFactory: creating.."); mProvider = getDefaultProvider(); @@ -96,11 +96,7 @@ public class AuditProviderFactory { return ret; } - public static AuditHandler getAuditProvider() { - return AuditProviderFactory.getInstance().getProvider(); - } - - public AuditHandler getProvider() { + public AuditHandler getAuditProvider() { return mProvider; } http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/agents-audit/src/main/java/org/apache/ranger/audit/provider/StandAloneAuditProviderFactory.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/StandAloneAuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/StandAloneAuditProviderFactory.java new file mode 100644 index 0000000..4306b24 --- /dev/null +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/StandAloneAuditProviderFactory.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class StandAloneAuditProviderFactory extends AuditProviderFactory { + private static final Log LOG = LogFactory.getLog(StandAloneAuditProviderFactory.class); + + private volatile static StandAloneAuditProviderFactory sFactory = null; + + public static StandAloneAuditProviderFactory getInstance() { + StandAloneAuditProviderFactory ret = sFactory; + if(ret == null) { + synchronized(StandAloneAuditProviderFactory.class) { + ret = sFactory; + if(ret == null) { + ret = sFactory = new StandAloneAuditProviderFactory(); + } + } + } + return ret; + } + + private StandAloneAuditProviderFactory() { + super(); + LOG.info("StandAloneAuditProviderFactory: created.."); + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/agents-audit/src/main/java/org/apache/ranger/audit/test/TestEvents.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/test/TestEvents.java b/agents-audit/src/main/java/org/apache/ranger/audit/test/TestEvents.java index 7bfb3bf..57f76d8 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/test/TestEvents.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/test/TestEvents.java @@ -72,9 +72,10 @@ public class TestEvents { auditProperties.setProperty("xasecure.audit.db.batch.size", "100"); } - AuditProviderFactory.getInstance().init(auditProperties, "hdfs"); + AuditProviderFactory factory = new AuditProviderFactory(); + factory.init(auditProperties, "hdfs"); - AuditHandler provider = AuditProviderFactory.getAuditProvider(); + AuditHandler provider = factory.getAuditProvider(); LOG.info("provider=" + provider.toString()); http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerConfiguration.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerConfiguration.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerConfiguration.java index 2498fb0..0c2e473 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerConfiguration.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerConfiguration.java @@ -25,7 +25,7 @@ import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; -import org.apache.ranger.audit.provider.AuditProviderFactory; + public class RangerConfiguration extends Configuration { private static final Logger LOG = Logger.getLogger(RangerConfiguration.class); @@ -77,7 +77,7 @@ public class RangerConfiguration extends Configuration { return ret; } - private boolean addResourceIfReadable(String aResourceName) { + public boolean addResourceIfReadable(String aResourceName) { boolean ret = false; if(LOG.isDebugEnabled()) { @@ -123,31 +123,10 @@ public class RangerConfiguration extends Configuration { return result; } - public void initAudit(String appType) { - AuditProviderFactory auditFactory = AuditProviderFactory.getInstance(); - - if(auditFactory == null) { - LOG.error("Unable to find the AuditProviderFactory. (null) found"); - return; - } - - Properties props = getProps(); - - if(props == null) { - return; - } - - if(! auditFactory.isInitDone()) { - auditFactory.init(props, appType); - } + public Properties getProperties() { + return getProps(); } - public boolean isAuditInitDone() { - AuditProviderFactory auditFactory = AuditProviderFactory.getInstance(); - - return auditFactory != null && auditFactory.isInitDone(); - } - private URL getFileLocation(String fileName) { URL lurl = RangerConfiguration.class.getClassLoader().getResource(fileName); @@ -184,9 +163,8 @@ public class RangerConfiguration extends Configuration { LOG.debug("==> addAuditResource(Service Type: " + serviceType ); } - URL url = null; try { - url = RangerLegacyConfigBuilder.getAuditConfig(serviceType); + URL url = RangerLegacyConfigBuilder.getAuditConfig(serviceType); if( url != null) { addResource(url); http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java index 2aca2e9..b8da192 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java @@ -28,12 +28,13 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.model.AuthzAuditEvent; -import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.AuditHandler; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants; import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.service.RangerBasePlugin; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.codehaus.jackson.map.ObjectMapper; @@ -170,7 +171,9 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { if(auditEvent != null) { populateDefaults(auditEvent); - if(!AuditProviderFactory.getAuditProvider().log(auditEvent)) { + + AuditHandler auditProvider = RangerBasePlugin.getAuditProvider(auditEvent.getRepositoryName()); + if (auditProvider == null || !auditProvider.log(auditEvent)) { MiscUtil.logErrorMessageByInterval(LOG, "fail to log audit event " + auditEvent); } } http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index e40d223..7221f6b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -32,6 +32,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.admin.client.RangerAdminClient; import org.apache.ranger.admin.client.RangerAdminRESTClient; +import org.apache.ranger.audit.provider.AuditHandler; +import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.audit.provider.StandAloneAuditProviderFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.plugin.model.RangerPolicy; @@ -72,6 +75,7 @@ public class RangerBasePlugin { private String[] trustedProxyAddresses; private Timer policyEngineRefreshTimer; private RangerAuthContextListener authContextListener; + private AuditProviderFactory auditProviderFactory; Map<String, LogHistory> logHistoryList = new Hashtable<String, RangerBasePlugin.LogHistory>(); @@ -81,8 +85,46 @@ public class RangerBasePlugin { return servicePluginMap; } - public RangerBasePlugin(String serviceType, String appId) { + public static AuditHandler getAuditProvider(String serviceName) { + AuditHandler ret = null; + + boolean useStandaloneAuditProvider = false; + + if (StringUtils.isNotEmpty(serviceName)) { + RangerBasePlugin plugin = RangerBasePlugin.getServicePluginMap().get(serviceName); + if (plugin != null) { + if (plugin.getAuditProviderFactory() != null) { + ret = plugin.getAuditProviderFactory().getAuditProvider(); + } else { + LOG.error("NULL AuditProviderFactory for serviceName:[" + serviceName + "]"); + } + } else { + useStandaloneAuditProvider = true; + } + } else { + useStandaloneAuditProvider = true; + } + if (useStandaloneAuditProvider) { + StandAloneAuditProviderFactory factory = StandAloneAuditProviderFactory.getInstance(); + if (factory.isInitDone()) { + ret = factory.getAuditProvider(); + } else { + RangerConfiguration conf = RangerConfiguration.getInstance(); + String auditCfg = "ranger-standalone-audit.xml"; + if (conf.addResourceIfReadable(auditCfg)) { + factory.init(conf.getProperties(), "StandAlone"); + ret = factory.getAuditProvider(); + } else { + LOG.error("StandAlone audit handler configuration not readable:[" + auditCfg + "]"); + } + } + } + + return ret; + } + + public RangerBasePlugin(String serviceType, String appId) { this.serviceType = serviceType; this.appId = appId; } @@ -125,12 +167,13 @@ public class RangerBasePlugin { return serviceName; } + public AuditProviderFactory getAuditProviderFactory() { return auditProviderFactory; } + public void init() { cleanup(); RangerConfiguration configuration = RangerConfiguration.getInstance(); configuration.addResourcesForServiceType(serviceType); - configuration.initAudit(appId); String propertyPrefix = "ranger.plugin." + serviceType; long pollingIntervalMs = configuration.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); @@ -157,6 +200,15 @@ public class RangerBasePlugin { LOG.warn("Ranger plugin will trust RemoteIPAddress and treat first X-Forwarded-Address in the access-request as the clientIPAddress"); } + if (configuration.getProperties() != null) { + auditProviderFactory = new AuditProviderFactory(); + auditProviderFactory.init(configuration.getProperties(), appId); + } else { + LOG.error("Audit subsystem is not initialized correctly. Please check audit configuration. "); + LOG.error("No authorization audits will be generated. "); + auditProviderFactory = null; + } + policyEngineOptions.configureForPlugin(configuration, propertyPrefix); LOG.info(policyEngineOptions); @@ -242,6 +294,8 @@ public class RangerBasePlugin { Timer policyEngineRefreshTimer = this.policyEngineRefreshTimer; + String serviceName = this.serviceName; + this.serviceName = null; this.policyEngine = null; this.refresher = null; @@ -259,6 +313,9 @@ public class RangerBasePlugin { policyEngine.cleanup(); } + if (serviceName != null) { + servicePluginMap.remove(serviceName); + } } public void setResultProcessor(RangerAccessResultProcessor resultProcessor) { http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index 4ed9a6f..1d9b865 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -107,9 +107,10 @@ public class TestPolicyEngine { auditProperties.setProperty("xasecure.audit.db.batch.size", "100"); } - AuditProviderFactory.getInstance().init(auditProperties, "hdfs"); // second parameter does not matter for v2 + AuditProviderFactory factory = new AuditProviderFactory(); + factory.init(auditProperties, "hdfs"); // second parameter does not matter for v2 - AuditHandler provider = AuditProviderFactory.getAuditProvider(); + AuditHandler provider = factory.getAuditProvider(); System.out.println("provider=" + provider.toString()); http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java ---------------------------------------------------------------------- diff --git a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java index 97aa204..d89b467 100644 --- a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java +++ b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java @@ -31,7 +31,6 @@ import java.util.Set; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.audit.provider.AuditProviderFactory; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.audit.RangerMultiResourceAuditHandler; @@ -146,7 +145,9 @@ public class RangerSolrAuthorizer implements AuthorizationPlugin { * are not always invoked and the audit store are not flushed. So * we are forcing a cleanup here. */ - AuditProviderFactory.getInstance().shutdown(); + if (solrPlugin.getAuditProviderFactory() != null) { + solrPlugin.getAuditProviderFactory().shutdown(); + } } catch (Throwable t) { logger.error("Error cleaning up Ranger plugin. Ignoring error", t); } http://git-wip-us.apache.org/repos/asf/ranger/blob/d9ace039/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java ---------------------------------------------------------------------- diff --git a/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java b/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java index dee3156..cb0dcc4 100644 --- a/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java +++ b/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java @@ -192,9 +192,9 @@ public class TestAuditQueue { props.put(propPrefix + "." + BaseAuditHandler.PROP_CLASS_NAME, TestConsumer.class.getName()); - AuditProviderFactory factory = AuditProviderFactory.getInstance(); + AuditProviderFactory factory = new AuditProviderFactory(); factory.init(props, "test"); - AuditQueue queue = (AuditQueue) factory.getProvider(); + AuditQueue queue = (AuditQueue) factory.getAuditProvider(); BaseAuditHandler consumer = (BaseAuditHandler) queue.getConsumer(); while (consumer != null && consumer instanceof AuditQueue) { AuditQueue cQueue = (AuditQueue) consumer; @@ -626,7 +626,7 @@ public class TestAuditQueue { + AuditFileSpool.PROP_FILE_SPOOL_DEST_RETRY_MS, "" + destRetryMS); - AuditProviderFactory factory = AuditProviderFactory.getInstance(); + AuditProviderFactory factory = new AuditProviderFactory(); factory.init(props, "test"); // FileAuditDestination fileDest = new FileAuditDestination(); @@ -636,7 +636,7 @@ public class TestAuditQueue { // queue.init(props, batchPropPrefix); // queue.start(); - AuditHandler queue = factory.getProvider(); + AuditHandler queue = factory.getAuditProvider(); for (int i = 0; i < messageToSend; i++) { queue.log(createEvent());
