Merge branch 'master' into tag-policy
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/4915b823 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/4915b823 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/4915b823 Branch: refs/heads/tag-policy Commit: 4915b823fa247b6fe6c1035a9ec15c7cfec5571c Parents: 5d437fa 4c45671 Author: Madhan Neethiraj <[email protected]> Authored: Tue Jun 2 20:11:52 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Jun 2 20:11:52 2015 -0700 ---------------------------------------------------------------------- .../plugin/store/AbstractServiceStore.java | 6 ++- .../plugin/store/EmbeddedServiceDefsUtil.java | 46 +++++++++++--------- .../service-defs/ranger-servicedef-yarn.json | 12 +---- .../org/apache/ranger/common/AppConstants.java | 18 ++++---- .../org/apache/ranger/common/ServiceUtil.java | 4 +- 5 files changed, 43 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4915b823/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java ---------------------------------------------------------------------- diff --cc agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index d394022,0000000..86dfd1f mode 100644,000000..100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@@ -1,430 -1,0 +1,434 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerBaseModelObject; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.util.SearchFilter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +public abstract class AbstractServiceStore implements ServiceStore { + private static final Log LOG = LogFactory.getLog(AbstractServiceStore.class); + + + private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000; + + + @Override + public void updateTagServiceDefForAccessTypes() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefDBStore.updateTagServiceDefForAccessTypes()"); + } + List<RangerServiceDef> allServiceDefs = getServiceDefs(new SearchFilter()); + for (RangerServiceDef serviceDef : allServiceDefs) { + updateTagServiceDefForUpdatingAccessTypes(serviceDef); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefDBStore.updateTagServiceDefForAccessTypes()"); + } + } + + @Override + public void deleteServiceDef(Long id, Boolean forceDelete) throws Exception { + deleteServiceDef(id); // Ignore forceDelete flag + } + + @Override + public PList<RangerServiceDef> getPaginatedServiceDefs(SearchFilter filter) throws Exception { + List<RangerServiceDef> resultList = getServiceDefs(filter); + + return new PList<RangerServiceDef>(resultList, 0, resultList.size(), + (long)resultList.size(), resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList<RangerService> getPaginatedServices(SearchFilter filter) throws Exception { + List<RangerService> resultList = getServices(filter); + + return new PList<RangerService>(resultList, 0, resultList.size(), (long)resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList<RangerPolicy> getPaginatedPolicies(SearchFilter filter) throws Exception { + List<RangerPolicy> resultList = getPolicies(filter); + + return new PList<RangerPolicy>(resultList, 0, resultList.size(), (long)resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList<RangerPolicy> getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception { + List<RangerPolicy> resultList = getServicePolicies(serviceId, filter); + + return new PList<RangerPolicy>(resultList, 0, resultList.size(), (long)resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList<RangerPolicy> getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception { + List<RangerPolicy> resultList = getServicePolicies(serviceName, filter); + + return new PList<RangerPolicy>(resultList, 0, resultList.size(), (long)resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId) { + RangerPolicy ret = null; + try { + ret = getPolicy(policyId); + } catch (Exception e) { + // Do nothing + } + return ret; + } + + @Override + public RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo) { + RangerPolicy ret = null; + try { + ret = getPolicy(policyId); + } catch (Exception e) { + // Do nothing + } + return ret; + } + + @Override + public String getPolicyForVersionNumber(Long policyId) { + RangerPolicy ret = null; + try { + ret = getPolicy(policyId); + } catch (Exception e) { + // Do nothing + } + return ret == null ? null : ret.getName(); + } + + protected void preCreate(RangerBaseModelObject obj) throws Exception { + obj.setId(0L); + obj.setGuid(UUID.randomUUID().toString()); + obj.setCreateTime(new Date()); + obj.setUpdateTime(obj.getCreateTime()); + obj.setVersion(1L); + } + + protected void preCreate(RangerService service) throws Exception { + preCreate((RangerBaseModelObject)service); + + service.setPolicyVersion(0L); + service.setPolicyUpdateTime(service.getCreateTime()); + } + + protected void postCreate(RangerBaseModelObject obj) throws Exception { + if(obj instanceof RangerServiceDef) { + updateTagServiceDefForUpdatingAccessTypes((RangerServiceDef)obj); + } + } + + protected void preUpdate(RangerBaseModelObject obj) throws Exception { + if(obj.getId() == null) { + obj.setId(0L); + } + + if(obj.getGuid() == null) { + obj.setGuid(UUID.randomUUID().toString()); + } + + if(obj.getCreateTime() == null) { + obj.setCreateTime(new Date()); + } + + Long version = obj.getVersion(); + + if(version == null) { + version = 1L; + } else { + version = version + 1; + } + + obj.setVersion(version); + obj.setUpdateTime(new Date()); + } + + protected void postUpdate(RangerBaseModelObject obj) throws Exception { + if(obj instanceof RangerServiceDef) { + RangerServiceDef serviceDef = (RangerServiceDef)obj; + + updateTagServiceDefForUpdatingAccessTypes(serviceDef); + updateServicesForServiceDefUpdate(serviceDef); + } + } + + protected void preDelete(RangerBaseModelObject obj) throws Exception { + // TODO: + } + + protected void postDelete(RangerBaseModelObject obj) throws Exception { + if(obj instanceof RangerServiceDef) { + updateTagServiceDefForDeletingAccessTypes(((RangerServiceDef) obj).getName()); + } + } + + protected long getMaxId(List<? extends RangerBaseModelObject> objs) { + long ret = -1; + + if (objs != null) { + for (RangerBaseModelObject obj : objs) { + if (obj.getId() > ret) { + ret = obj.getId(); + } + } + } + return ret; + } + + // when a service-def is updated, the updated service-def should be made available to plugins + // this is achieved by incrementing policyVersion of all its services + protected void updateServicesForServiceDefUpdate(RangerServiceDef serviceDef) throws Exception { + boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); + + SearchFilter filter = new SearchFilter(); + + filter.setParam(SearchFilter.SERVICE_TYPE, serviceDef.getName()); + + List<RangerService> services = getServices(filter); + + if(CollectionUtils.isNotEmpty(services)) { + for(RangerService service : services) { + service.setPolicyVersion(getNextVersion(service.getPolicyVersion())); + service.setPolicyUpdateTime(serviceDef.getUpdateTime()); + + updateService(service); + + if(isTagServiceDef) { + filter = new SearchFilter(); + + filter.setParam(SearchFilter.TAG_SERVICE_NAME, service.getName()); + + List<RangerService> referrringServices = getServices(filter); + + if(CollectionUtils.isNotEmpty(referrringServices)) { + for(RangerService referringService : referrringServices) { + referringService.setPolicyVersion(getNextVersion(referringService.getPolicyVersion())); + referringService.setPolicyUpdateTime(serviceDef.getUpdateTime()); + + updateService(referringService); + } + } + } + } + } + } + + private RangerServiceDef.RangerAccessTypeDef findAccessTypeDef(long itemId, List<RangerServiceDef.RangerAccessTypeDef> accessTypeDefs) { + RangerServiceDef.RangerAccessTypeDef ret = null; + + for(RangerServiceDef.RangerAccessTypeDef accessTypeDef : accessTypeDefs) { + if(itemId == accessTypeDef.getItemId()) { + ret = accessTypeDef; + break; + } + } + return ret; + } + + private void updateTagServiceDefForUpdatingAccessTypes(RangerServiceDef serviceDef) throws Exception { + if (StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + return; + } + ++ if(EmbeddedServiceDefsUtil.instance().getTagServiceDefId() == -1) { ++ LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes(" + serviceDef.getName() + "): tag service-def does not exist"); ++ } ++ + String serviceDefName = serviceDef.getName(); + - RangerServiceDef tagServiceDef; ++ RangerServiceDef tagServiceDef = null; + try { + tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId()); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes" + serviceDef.getName() + "): could not find TAG ServiceDef.. ", e); + throw e; + } + + if(tagServiceDef == null) { + LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes(" + serviceDef.getName() + "): could not fund TAG ServiceDef.. "); + + return; + } + + List<RangerServiceDef.RangerAccessTypeDef> toAdd = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + List<RangerServiceDef.RangerAccessTypeDef> toUpdate = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + List<RangerServiceDef.RangerAccessTypeDef> toDelete = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + + List<RangerServiceDef.RangerAccessTypeDef> svcDefAccessTypes = serviceDef.getAccessTypes(); + List<RangerServiceDef.RangerAccessTypeDef> tagDefAccessTypes = tagServiceDef.getAccessTypes(); + + long itemIdOffset = serviceDef.getId() * (MAX_ACCESS_TYPES_IN_SERVICE_DEF + 1); + + for (RangerServiceDef.RangerAccessTypeDef svcAccessType : svcDefAccessTypes) { + long tagAccessTypeItemId = svcAccessType.getItemId() + itemIdOffset; + + RangerServiceDef.RangerAccessTypeDef tagAccessType = findAccessTypeDef(tagAccessTypeItemId, tagDefAccessTypes); + + if(tagAccessType == null) { + tagAccessType = new RangerServiceDef.RangerAccessTypeDef(); + + tagAccessType.setItemId(tagAccessTypeItemId); + tagAccessType.setName(serviceDefName + ":" + svcAccessType.getName()); + tagAccessType.setLabel(svcAccessType.getLabel()); + tagAccessType.setRbKeyLabel(svcAccessType.getRbKeyLabel()); + + tagAccessType.setImpliedGrants(new HashSet<String>()); + if(CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) { + for(String svcImpliedGrant : svcAccessType.getImpliedGrants()) { + tagAccessType.getImpliedGrants().add(serviceDefName + ":" + svcImpliedGrant); + } + } + + toAdd.add(tagAccessType); + } + } + + for (RangerServiceDef.RangerAccessTypeDef tagAccessType : tagDefAccessTypes) { + if (tagAccessType.getName().startsWith(serviceDefName + ":")) { + long svcAccessTypeItemId = tagAccessType.getItemId() - itemIdOffset; + + RangerServiceDef.RangerAccessTypeDef svcAccessType = findAccessTypeDef(svcAccessTypeItemId, svcDefAccessTypes); + + if(svcAccessType == null) { // accessType has been deleted in service + toDelete.add(tagAccessType); + continue; + } + + boolean isUpdated = false; + + if(! Objects.equals(tagAccessType.getName().substring(serviceDefName.length() + 1), svcAccessType.getName())) { + isUpdated = true; + } else if(! Objects.equals(tagAccessType.getLabel(), svcAccessType.getLabel())) { + isUpdated = true; + } else if(! Objects.equals(tagAccessType.getRbKeyLabel(), svcAccessType.getRbKeyLabel())) { + isUpdated = true; + } else { + Collection<String> tagImpliedGrants = tagAccessType.getImpliedGrants(); + Collection<String> svcImpliedGrants = svcAccessType.getImpliedGrants(); + + int tagImpliedGrantsLen = tagImpliedGrants == null ? 0 : tagImpliedGrants.size(); + int svcImpliedGrantsLen = svcImpliedGrants == null ? 0 : svcImpliedGrants.size(); + + if(tagImpliedGrantsLen != svcImpliedGrantsLen) { + isUpdated = true; + } else if(tagImpliedGrantsLen > 0) { + for(String svcImpliedGrant : svcImpliedGrants) { + if(! tagImpliedGrants.contains(serviceDefName + ":" + svcImpliedGrant)) { + isUpdated = true; + break; + } + } + } + } + + if(isUpdated) { + tagAccessType.setName(serviceDefName + ":" + svcAccessType.getName()); + tagAccessType.setLabel(svcAccessType.getLabel()); + tagAccessType.setRbKeyLabel(svcAccessType.getRbKeyLabel()); + + tagAccessType.setImpliedGrants(new HashSet<String>()); + if(CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) { + for(String svcImpliedGrant : svcAccessType.getImpliedGrants()) { + tagAccessType.getImpliedGrants().add(serviceDefName + ":" + svcImpliedGrant); + } + } + + toUpdate.add(tagAccessType); + } + } + } + + if(CollectionUtils.isNotEmpty(toAdd) || CollectionUtils.isNotEmpty(toUpdate) || CollectionUtils.isNotEmpty(toDelete)) { + tagDefAccessTypes.addAll(toAdd); + tagDefAccessTypes.removeAll(toDelete); + + try { + updateServiceDef(tagServiceDef); + LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- Failed to update TAG ServiceDef.. ", e); + throw e; + } + } + } + + private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) throws Exception { + if (serviceDefName.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + return; + } + + RangerServiceDef tagServiceDef; + try { + tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId()); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes(" + serviceDefName + "): could not fund TAG ServiceDef.. ", e); + throw e; + } + + if(tagServiceDef == null) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes(" + serviceDefName + "): could not fund TAG ServiceDef.. "); + + return; + } + + List<RangerServiceDef.RangerAccessTypeDef> accessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) { + if (accessType.getName().startsWith(serviceDefName + ":")) { + accessTypes.add(accessType); + } + } + + tagServiceDef.getAccessTypes().removeAll(accessTypes); + try { + updateServiceDef(tagServiceDef); + LOG.info("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Failed to update TAG ServiceDef.. ", e); + throw e; + } + } + + protected long getNextVersion(Long currentVersion) { + return currentVersion == null ? 1L : currentVersion + 1; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4915b823/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java ---------------------------------------------------------------------- diff --cc agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index 73e880b,77e2c80..959e3f4 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@@ -75,10 -74,9 +75,11 @@@ public class EmbeddedServiceDefsUtil private RangerServiceDef knoxServiceDef = null; private RangerServiceDef stormServiceDef = null; private RangerServiceDef yarnServiceDef = null; - private RangerServiceDef kmsServiceDef = null; - private RangerServiceDef kafkaServiceDef = null; + private RangerServiceDef kafkaServiceDef = null; private RangerServiceDef solrServiceDef = null; ++ private RangerServiceDef tagServiceDef = null; ++ private Gson gsonBuilder = null; @@@ -105,12 -107,8 +110,13 @@@ knoxServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KNOX_NAME); stormServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_STORM_NAME); yarnServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_YARN_NAME); - kmsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KMS_NAME); - kafkaServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KAFKA_NAME); + kafkaServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KAFKA_NAME); solrServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_SOLR_NAME); + ++ tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME); ++ + // Ensure that tag service def is updated with access types of all service defs + store.updateTagServiceDefForAccessTypes(); } catch(Throwable excp) { LOG.fatal("EmbeddedServiceDefsUtil.init(): failed", excp); } @@@ -156,6 -152,6 +160,8 @@@ return getId(solrServiceDef); } ++ public long getTagServiceDefId() { return getId(tagServiceDef); } ++ private long getId(RangerServiceDef serviceDef) { return serviceDef == null || serviceDef.getId() == null ? -1 : serviceDef.getId().longValue(); }
