RANGER-648: implemented cleanup of policy-engine/context-enrichers to reclaim resouces like threads when no longer needed
Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/a7453154 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/a7453154 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/a7453154 Branch: refs/heads/master Commit: a7453154663bc2532894dda78ac43b24b9d27ef1 Parents: 0e47fbf Author: Abhay Kulkarni <[email protected]> Authored: Mon Sep 7 16:00:16 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Sep 9 17:38:12 2015 -0700 ---------------------------------------------------------------------- .../RangerAbstractContextEnricher.java | 42 +++- .../RangerAdminTagRetriever.java | 32 +-- .../contextenricher/RangerContextEnricher.java | 21 +- .../contextenricher/RangerTagEnricher.java | 195 ++++++++++++++++--- .../RangerTagFileStoreRetriever.java | 35 +--- .../contextenricher/RangerTagReceiver.java | 28 --- .../contextenricher/RangerTagRetriever.java | 29 +-- .../plugin/policyengine/RangerPolicyEngine.java | 5 + .../policyengine/RangerPolicyEngineImpl.java | 61 +++++- .../policyengine/RangerPolicyRepository.java | 6 +- .../ranger/plugin/service/RangerBasePlugin.java | 16 +- 11 files changed, 342 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java index f869d58..c70fdb4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java @@ -40,7 +40,7 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri protected RangerServiceDef serviceDef; @Override - public void setContextEnricherDef(RangerContextEnricherDef enricherDef) { + public void setEnricherDef(RangerContextEnricherDef enricherDef) { this.enricherDef = enricherDef; } @@ -56,12 +56,12 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri } @Override - public void setContextServiceName(String serviceName) { + public void setServiceName(String serviceName) { this.serviceName = serviceName; } @Override - public void setContextServiceDef(RangerServiceDef serviceDef) { + public void setServiceDef(RangerServiceDef serviceDef) { this.serviceDef = serviceDef; } @@ -70,6 +70,42 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri this.appId = appId; } + + @Override + public RangerContextEnricherDef getEnricherDef() { + return enricherDef; + } + + @Override + public String getServiceName() { + return serviceName; + } + + @Override + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + @Override + public String getAppId() { + return appId; + } + + + @Override + public String getName() { + return enricherDef == null ? null : enricherDef.getName(); + } + + @Override + public boolean preCleanup() { + return true; + } + + @Override + public void cleanup() { + } + public String getOption(String name) { String ret = null; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java index 9db8f0a..7c5b378 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java @@ -36,7 +36,8 @@ public class RangerAdminTagRetriever extends RangerTagRetriever { @Override public void init(Map<String, String> options) { - if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId) && tagReceiver != null) { + + if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) { String propertyPrefix = "ranger.plugin." + serviceDef.getName(); adminClient = RangerBasePlugin.createAdminClient(serviceName, appId, propertyPrefix); @@ -47,37 +48,24 @@ public class RangerAdminTagRetriever extends RangerTagRetriever { } @Override - public void retrieveTags() throws InterruptedException { + public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException { + + ServiceTags serviceTags = null; - if (adminClient != null && tagReceiver != null) { - ServiceTags serviceTags = null; + if (adminClient != null) { try { serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion); - } - catch (InterruptedException interruptedException) { + } catch (InterruptedException interruptedException) { LOG.error("Tag-retriever thread was interrupted"); throw interruptedException; - } - catch (ClosedByInterruptException closedByInterruptException) { + } catch (ClosedByInterruptException closedByInterruptException) { LOG.error("Tag-retriever thread was interrupted while blocked on I/O"); throw new InterruptedException(); - } - catch (Exception exception) { + } catch (Exception exception) { LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources, exception=", exception); } - - if (serviceTags != null) { - tagReceiver.setServiceTags(serviceTags); - LOG.info("RangerAdminTagRetriever.retrieveTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion()); - setLastKnownVersion(serviceTags.getTagVersion()); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("RangerAdminTagRetriever.retrieveTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion); - } - } - } else { - LOG.error("RangerAdminTagRetriever.retrieveTags() - No admin client to get tags from or no tag receiver to update tag-cache"); } + return serviceTags; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java index 10fed69..c88cbf1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java @@ -25,14 +25,24 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerContextEnricherDef; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; public interface RangerContextEnricher { - void setContextEnricherDef(RangerContextEnricherDef enricherDef); + void setEnricherDef(RangerContextEnricherDef enricherDef); - void setContextServiceName(String serviceName); + void setServiceName(String serviceName); - void setContextServiceDef(RangerServiceDef serviceDef); + void setServiceDef(RangerServiceDef serviceDef); void setAppId(String appId); + RangerContextEnricherDef getEnricherDef(); + + String getServiceName(); + + RangerServiceDef getServiceDef(); + + String getAppId(); + + String getName(); + //void setContextComponentServiceName(String componentServiceName); //void setContextComponentServiceDef(RangerServiceDef componentServiceDef); @@ -40,4 +50,9 @@ public interface RangerContextEnricher { void init(); void enrich(RangerAccessRequest request); + + boolean preCleanup(); + + void cleanup(); + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 37732a6..6388c78 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -19,11 +19,14 @@ package org.apache.ranger.plugin.contextenricher; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; @@ -32,11 +35,12 @@ import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourc import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.ServiceTags; +import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.Map; -public class RangerTagEnricher extends RangerAbstractContextEnricher implements RangerTagReceiver { +public class RangerTagEnricher extends RangerAbstractContextEnricher { private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class); public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; @@ -67,8 +71,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements if (StringUtils.isNotBlank(tagRetrieverClassName)) { - cleanup(); - try { @SuppressWarnings("unchecked") Class<RangerTagRetriever> tagRetriverClass = (Class<RangerTagRetriever>) Class.forName(tagRetrieverClassName); @@ -86,21 +88,26 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements } if (tagRetriever != null) { + String propertyPrefix = "ranger.plugin." + serviceDef.getName(); + String cacheDir = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.cache.dir"); + String cacheFilename = String.format("%s_%s_tag.json", appId, serviceName); + cacheFilename = cacheFilename.replace(File.separatorChar, '_'); + cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); + + String cacheFile = cacheDir == null ? null : (cacheDir + File.separator + cacheFilename); tagRetriever.setServiceName(serviceName); tagRetriever.setServiceDef(serviceDef); tagRetriever.setAppId(appId); - tagRetriever.setLastKnownVersion(lastKnownVersion); - tagRetriever.setTagReceiver(this); tagRetriever.init(enricherDef.getEnricherOptions()); + tagRefresher = new RangerTagRefresher(tagRetriever, this, lastKnownVersion, cacheFile, pollingIntervalMs); + try { - tagRetriever.retrieveTags(); - } catch (Exception exception) { - // Ignore + tagRefresher.populateTags(); + } catch (Throwable exception) { + LOG.error("Exception when retrieving tag for the first time for this enricher", exception); } - tagRefresher = new RangerTagRefresher(tagRetriever, pollingIntervalMs); - tagRefresher.startRefresher(); } } else { @@ -112,14 +119,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements } } - public void cleanup() { - - if (tagRefresher != null) { - tagRefresher.cleanup(); - tagRefresher = null; - } - } - @Override public void enrich(RangerAccessRequest request) { if (LOG.isDebugEnabled()) { @@ -147,7 +146,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements } } - @Override public void setServiceTags(final ServiceTags serviceTags) { this.serviceTags = serviceTags; this.lastKnownVersion = serviceTags.getTagVersion(); @@ -181,6 +179,25 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements } + @Override + public boolean preCleanup() { + boolean ret = true; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.preCleanup()"); + } + + if (tagRefresher != null) { + tagRefresher.cleanup(); + tagRefresher = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.preCleanup() : result=" + ret); + } + return ret; + } + private List<RangerTag> findMatchingTags(final RangerAccessResource resource, final List<RangerServiceResourceMatcher> resourceMatchers) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerTagEnricher.findMatchingTags(" + resource + ")"); @@ -252,16 +269,30 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements private static final Log LOG = LogFactory.getLog(RangerTagRefresher.class); private final RangerTagRetriever tagRetriever; + private final RangerTagEnricher tagEnricher; + private long lastKnownVersion = -1L; private final long pollingIntervalMs; + private final String cacheFile; + private boolean hasProvidedTagsToReceiver = false; + private Gson gson; + final long getPollingIntervalMs() { return pollingIntervalMs; } - RangerTagRefresher(RangerTagRetriever tagRetriever, long pollingIntervalMs) { + RangerTagRefresher(RangerTagRetriever tagRetriever, RangerTagEnricher tagEnricher, long lastKnownVersion, String cacheFile, long pollingIntervalMs) { this.tagRetriever = tagRetriever; + this.tagEnricher = tagEnricher; + this.lastKnownVersion = lastKnownVersion; + this.cacheFile = cacheFile; this.pollingIntervalMs = pollingIntervalMs; + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); + } catch(Throwable excp) { + LOG.fatal("failed to create GsonBuilder object", excp); + } } @Override @@ -275,7 +306,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements try { - tagRetriever.retrieveTags(); + populateTags(); if (pollingIntervalMs > 0) { Thread.sleep(pollingIntervalMs); @@ -293,8 +324,46 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements } } + private void populateTags() throws InterruptedException { + + if (tagEnricher != null) { + ServiceTags serviceTags = null; + + serviceTags = tagRetriever.retrieveTags(lastKnownVersion); + + if (serviceTags == null) { + if (!hasProvidedTagsToReceiver) { + serviceTags = loadFromCache(); + } + } else { + saveToCache(serviceTags); + } + + if (serviceTags != null) { + tagEnricher.setServiceTags(serviceTags); + LOG.info("RangerTagRefresher.populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion()); + lastKnownVersion = serviceTags.getTagVersion(); + hasProvidedTagsToReceiver = true; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagRefresher.populateTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion); + } + } + } else { + LOG.error("RangerTagRefresher.populateTags() - no tag receiver to update tag-cache"); + } + } + void cleanup() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRefresher.cleanup()"); + } + stopRefresher(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRefresher.cleanup()"); + } } final void startRefresher() { @@ -317,6 +386,88 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements } } } - } + + final ServiceTags loadFromCache() { + ServiceTags serviceTags = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()"); + } + + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + Reader reader = null; + + try { + reader = new FileReader(cacheFile); + + serviceTags = gson.fromJson(reader, ServiceTags.class); + + if (serviceTags != null) { + if (!StringUtils.equals(tagEnricher.getServiceName(), serviceTags.getServiceName())) { + LOG.warn("ignoring unexpected serviceName '" + serviceTags.getServiceName() + "' in cache file '" + cacheFile.getAbsolutePath() + "'"); + + serviceTags.setServiceName(tagEnricher.getServiceName()); + } + } + } catch (Exception excp) { + LOG.error("failed to load service-tags from cache file " + cacheFile.getAbsolutePath(), excp); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(), excp); + } + } + } + } else { + LOG.warn("cache file does not exist or not readble '" + (cacheFile == null ? null : cacheFile.getAbsolutePath()) + "'"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()"); + } + + return serviceTags; + } + + final void saveToCache(ServiceTags serviceTags) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()"); + } + + if (serviceTags != null) { + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + + if (cacheFile != null) { + Writer writer = null; + + try { + writer = new FileWriter(cacheFile); + + gson.toJson(serviceTags, writer); + } catch (Exception excp) { + LOG.error("failed to save service-tags to cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } finally { + if (writer != null) { + try { + writer.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } + } + } + } + } else { + LOG.info("service-tags is null. Nothing to save in cache"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()"); + } + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java index 0259bdf..1ee00d3 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java @@ -34,12 +34,10 @@ public class RangerTagFileStoreRetriever extends RangerTagRetriever { private TagStore tagStore; - public RangerTagFileStoreRetriever() { - } - @Override public void init(Map<String, String> options) { - if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId) && tagReceiver != null) { + + if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) { tagStore = TagFileStore.getInstance(); @@ -49,37 +47,24 @@ public class RangerTagFileStoreRetriever extends RangerTagRetriever { } @Override - public void retrieveTags() throws InterruptedException { + public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException { - if (tagStore != null && tagReceiver != null) { - ServiceTags serviceTags = null; + ServiceTags serviceTags = null; + + if (tagStore != null) { try { serviceTags = tagStore.getServiceTagsIfUpdated(serviceName, lastKnownVersion); - } - catch (InterruptedException interruptedException) { + } catch (InterruptedException interruptedException) { LOG.error("Tag-retriever thread was interrupted"); throw interruptedException; - } - catch (ClosedByInterruptException closedByInterruptException) { + } catch (ClosedByInterruptException closedByInterruptException) { LOG.error("Tag-retriever thread was interrupted while blocked on I/O"); throw new InterruptedException(); - } - catch (Exception exception) { + } catch (Exception exception) { LOG.error("RangerTagFileStoreRetriever.retrieveTags() - Error retrieving resources, exception=", exception); } - - if (serviceTags != null) { - tagReceiver.setServiceTags(serviceTags); - LOG.info("RangerTagFileStoreRetriever.retrieveTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion()); - setLastKnownVersion(serviceTags.getTagVersion()); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("RangerTagFileStoreRetriever.retrieveTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion); - } - } - } else { - LOG.error("RangerTagFileStoreRetriever.retrieveTags() - No tag-store to get tags from or no tag receiver to update tag-cache..."); } + return serviceTags; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java deleted file mode 100644 index 47db707..0000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.contextenricher; - -import org.apache.ranger.plugin.util.ServiceTags; - -import java.util.List; - -public interface RangerTagReceiver { - void setServiceTags(final ServiceTags serviceTags); -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java index 4967cbe..374e5c1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java @@ -19,21 +19,27 @@ package org.apache.ranger.plugin.contextenricher; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.util.ServiceTags; +import java.io.*; import java.util.Map; public abstract class RangerTagRetriever { + private static final Log LOG = LogFactory.getLog(RangerTagRetriever.class); protected String serviceName; protected RangerServiceDef serviceDef; protected String appId; - protected long lastKnownVersion; - protected RangerTagReceiver tagReceiver; public abstract void init(Map<String, String> options); - public abstract void retrieveTags() throws InterruptedException; + public abstract ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException; public String getServiceName() { return serviceName; @@ -58,21 +64,4 @@ public abstract class RangerTagRetriever { public void setAppId(String appId) { this.appId = appId; } - - public long getLastKnownVersion() { - return lastKnownVersion; - } - - public void setLastKnownVersion(long lastKnownVersion) { - this.lastKnownVersion = lastKnownVersion; - } - - public RangerTagReceiver getTagReceiver() { - return tagReceiver; - } - - public void setTagReceiver(RangerTagReceiver tagReceiver) { - this.tagReceiver = tagReceiver; - } - } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java index 055a2db..29080b7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java @@ -57,4 +57,9 @@ public interface RangerPolicyEngine { RangerPolicy getExactMatchPolicy(RangerAccessResource resource); List<RangerPolicy> getAllowedPolicies(String user, Set<String> userGroups, String accessType); + + boolean preCleanup(); + + void cleanup(); + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 3cccde0..5d1140b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -40,7 +40,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private final RangerPolicyRepository policyRepository; private final RangerPolicyRepository tagPolicyRepository; - private final List<RangerContextEnricher> allContextEnrichers; + private List<RangerContextEnricher> allContextEnrichers; public RangerPolicyEngineImpl(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions options) { if (LOG.isDebugEnabled()) { @@ -484,6 +484,65 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { } @Override + public boolean preCleanup() { + + boolean ret = true; + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.preCleanup()"); + } + + if (CollectionUtils.isNotEmpty(allContextEnrichers)) { + for (RangerContextEnricher contextEnricher : allContextEnrichers) { + boolean notReadyForCleanup = contextEnricher.preCleanup(); + if (!notReadyForCleanup) { + if (LOG.isDebugEnabled()) { + LOG.debug("contextEnricher.preCleanup() failed for contextEnricher=" + contextEnricher.getName()); + } + ret = false; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.preCleanup() : result=" + ret); + } + + return ret; + } + + @Override + public void cleanup() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.cleanup()"); + } + + preCleanup(); + + if (CollectionUtils.isNotEmpty(allContextEnrichers)) { + for (RangerContextEnricher contextEnricher : allContextEnrichers) { + contextEnricher.cleanup(); + } + } + + this.allContextEnrichers = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.cleanup()"); + } + } + + @Override + protected void finalize() throws Throwable { + try { + cleanup(); + } + finally { + super.finalize(); + } + } + + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 8519860..37b1ea6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -336,9 +336,9 @@ public class RangerPolicyRepository { } if(ret != null) { - ret.setContextEnricherDef(enricherDef); - ret.setContextServiceName(componentServiceName); - ret.setContextServiceDef(componentServiceDef); + ret.setEnricherDef(enricherDef); + ret.setServiceName(componentServiceName); + ret.setServiceDef(componentServiceDef); ret.setAppId(appId); ret.init(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index c857484..9d9987b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.admin.client.RangerAdminClient; import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.*; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; @@ -107,21 +108,34 @@ public class RangerBasePlugin { } public void setPolicies(ServicePolicies policies) { + RangerPolicyEngine oldPolicyEngine = this.policyEngine; + RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(appId, policies, policyEngineOptions); this.policyEngine = policyEngine; + + if (oldPolicyEngine != null && !oldPolicyEngine.preCleanup()) { + LOG.error("preCleanup() failed on the previous policy engine instance !!"); + } } public void cleanup() { + PolicyRefresher refresher = this.refresher; + RangerPolicyEngine policyEngine = this.policyEngine; + this.serviceName = null; this.policyEngine = null; this.refresher = null; - if(refresher != null) { + if (refresher != null) { refresher.stopRefresher(); } + + if (policyEngine != null) { + policyEngine.cleanup(); + } } public void setResultProcessor(RangerAccessResultProcessor resultProcessor) {
