This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 6030613 RANGER-3371: Update algorithm to build Ranger policy-database
object from Ranger policy-view object
6030613 is described below
commit 6030613254ae628b924b2337a59c6ddb1fba1155
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Tue Aug 17 10:51:26 2021 -0700
RANGER-3371: Update algorithm to build Ranger policy-database object from
Ranger policy-view object
---
.../ranger/plugin/util/RangerPolicyDeltaUtil.java | 2 +-
.../java/org/apache/ranger/biz/ServiceDBStore.java | 61 +++++++++++++++-------
.../ranger/common/RangerServicePoliciesCache.java | 6 +--
.../RangerTransactionSynchronizationAdapter.java | 27 +++++++---
.../org/apache/ranger/db/XXPolicyChangeLogDao.java | 15 +++---
.../ranger/service/RangerPolicyServiceBase.java | 28 +++++++---
6 files changed, 94 insertions(+), 45 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
index f040a66..42143d0 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
@@ -90,7 +90,7 @@ public class RangerPolicyDeltaUtil {
while (iter.hasNext()) {
RangerPolicy policy = iter.next();
- if (policyId.equals(policy.getId())) {
+ if (policyId.equals(policy.getId()) && changeType
== RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
deletedPolicies.add(policy);
iter.remove();
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index c5add3a..b9a926b 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -1401,7 +1401,7 @@ public class ServiceDBStore extends AbstractServiceStore {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("== ServiceDBStore.getServiceDefByName(" +
name + "): " + ret);
+ LOG.debug("== ServiceDBStore.getServiceDefByName(" +
name + "): " );
}
return ret;
@@ -3139,7 +3139,7 @@ public class ServiceDBStore extends AbstractServiceStore {
boolean isValid;
- resourcePolicyDeltas =
daoMgr.getXXPolicyChangeLog().findLaterThan(policyService, lastKnownVersion,
service.getId());
+ resourcePolicyDeltas =
daoMgr.getXXPolicyChangeLog().findLaterThan(lastKnownVersion, service.getId());
if (CollectionUtils.isNotEmpty(resourcePolicyDeltas)) {
isValid =
RangerPolicyDeltaUtil.isValidDeltas(resourcePolicyDeltas, componentServiceType);
@@ -3151,7 +3151,7 @@ public class ServiceDBStore extends AbstractServiceStore {
if (isValid && tagService != null) {
Long id =
resourcePolicyDeltas.get(0).getId();
- tagPolicyDeltas =
daoMgr.getXXPolicyChangeLog().findGreaterThan(policyService, id,
tagService.getId());
+ tagPolicyDeltas =
daoMgr.getXXPolicyChangeLog().findGreaterThan(id, tagService.getId());
if
(CollectionUtils.isNotEmpty(tagPolicyDeltas)) {
@@ -3542,46 +3542,53 @@ public class ServiceDBStore extends
AbstractServiceStore {
XXServiceVersionInfo serviceVersionInfoDbObj =
serviceVersionInfoDao.findByServiceId(id);
XXService service = daoMgr.getXXService().getById(id);
- Long nextPolicyVersion = 1L;
+ Long nextVersion = 1L;
Date now = new Date();
if (serviceVersionInfoDbObj != null) {
if (versionType == VERSION_TYPE.POLICY_VERSION) {
- nextPolicyVersion =
getNextVersion(serviceVersionInfoDbObj.getPolicyVersion());
-
-
serviceVersionInfoDbObj.setPolicyVersion(nextPolicyVersion);
+ nextVersion =
getNextVersion(serviceVersionInfoDbObj.getPolicyVersion());
+
serviceVersionInfoDbObj.setPolicyVersion(nextVersion);
serviceVersionInfoDbObj.setPolicyUpdateTime(now);
- }
- if (versionType == VERSION_TYPE.TAG_VERSION) {
-
serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion()));
+ } else if (versionType == VERSION_TYPE.TAG_VERSION) {
+ nextVersion =
getNextVersion(serviceVersionInfoDbObj.getTagVersion());
+
serviceVersionInfoDbObj.setTagVersion(nextVersion);
serviceVersionInfoDbObj.setTagUpdateTime(now);
- }
-
- if(versionType == VERSION_TYPE.ROLE_VERSION) {
+ } else if(versionType == VERSION_TYPE.ROLE_VERSION) {
// get the LatestRoleVersion from the
GlobalTable and update ServiceInfo for a service
XXGlobalStateDao xxGlobalStateDao =
daoMgr.getXXGlobalState();
if (xxGlobalStateDao != null) {
Long roleVersion =
xxGlobalStateDao.getAppDataVersion("RangerRole");
if (roleVersion != null) {
-
serviceVersionInfoDbObj.setRoleVersion(roleVersion);
-
serviceVersionInfoDbObj.setRoleUpdateTime(now);
+ nextVersion = roleVersion;
+ } else {
+ LOG.error("No Global state for
'RoleVersion'. Cannot execute this object:[" + serviceVersionUpdater + "]");
}
+
serviceVersionInfoDbObj.setRoleVersion(nextVersion);
+
serviceVersionInfoDbObj.setRoleUpdateTime(now);
+ } else {
+ LOG.error("No Global state DAO. Cannot
execute this object:[" + serviceVersionUpdater + "]");
+ return;
}
+ } else {
+ LOG.error("Unknown VERSION_TYPE:" + versionType
+ ". Cannot execute this object:[" + serviceVersionUpdater + "]");
+ return;
}
-
+ serviceVersionUpdater.version = nextVersion;
serviceVersionInfoDao.update(serviceVersionInfoDbObj);
} else {
if (service != null) {
serviceVersionInfoDbObj = new
XXServiceVersionInfo();
serviceVersionInfoDbObj.setServiceId(service.getId());
- serviceVersionInfoDbObj.setPolicyVersion(1L);
+
serviceVersionInfoDbObj.setPolicyVersion(nextVersion);
serviceVersionInfoDbObj.setPolicyUpdateTime(new
Date());
- serviceVersionInfoDbObj.setTagVersion(1L);
+
serviceVersionInfoDbObj.setTagVersion(nextVersion);
serviceVersionInfoDbObj.setTagUpdateTime(new
Date());
- serviceVersionInfoDbObj.setRoleVersion(1L);
+
serviceVersionInfoDbObj.setRoleVersion(nextVersion);
serviceVersionInfoDbObj.setRoleUpdateTime(new
Date());
+ serviceVersionUpdater.version = nextVersion;
serviceVersionInfoDao.create(serviceVersionInfoDbObj);
}
}
@@ -5493,6 +5500,7 @@ public class ServiceDBStore extends AbstractServiceStore {
final String zoneName;
final Integer policyDeltaChange;
final RangerPolicy policy;
+ long version = -1;
final ServiceTags.TagsChangeType tagChangeType;
final Long resourceId;
@@ -5530,6 +5538,21 @@ public class ServiceDBStore extends AbstractServiceStore
{
public void run() {
ServiceDBStore.persistVersionChange(this);
}
+
+ @Override
+ public String toString() {
+ return "ServiceVersionUpdater:[ " +
+ "serviceId=" + serviceId +
+ ", versionType=" + versionType +
+ ", version=" + version +
+ ", zoneName=" + zoneName +
+ ", policyDeltaChange=" +
policyDeltaChange +
+ ", policy=" + policy +
+ ", tagChangeType=" +
tagChangeType +
+ ", resourceId=" + resourceId +
+ ", tagId=" + tagId +
+ " ]";
+ }
}
@Override
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
index 04aa472..d1c6923 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
@@ -228,9 +228,7 @@ public class RangerServicePoliciesCache {
}
}
} else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Could not get lock
in [" + waitTimeInSeconds + "] seconds, returning cached ServicePolicies");
- }
+ LOG.error("Could not get lock in [" +
waitTimeInSeconds + "] seconds, returning cached ServicePolicies and wait Queue
Length:[" +lock.getQueueLength() + "], servicePolicies version:[" +
servicePolicies.getPolicyVersion() + "]");
ret = this.servicePolicies;
}
} catch (InterruptedException exception) {
@@ -363,7 +361,7 @@ public class RangerServicePoliciesCache {
result = Objects.equals(dbPolicyVersion,
cachedPolicyVersion);
- if (!result) {
+ if (!result && cachedPolicyVersion != null &&
dbPolicyVersion != null && cachedPolicyVersion < dbPolicyVersion) {
LOG.info("checkCacheSanity(serviceName=" +
serviceName + "): policy cache has a different version than one in the
database. However, changes from " + cachedPolicyVersion + " to " +
dbPolicyVersion + " will be downloaded in the next download.
policyVersionInDB=" + dbPolicyVersion + ", policyVersionInCache=" +
cachedPolicyVersion);
}
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
index fc44e6d..6c4902b 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
@@ -32,11 +32,14 @@ import org.springframework.stereotype.Component;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.TransactionSystemException;
import org.springframework.transaction.support.TransactionCallback;
import
org.springframework.transaction.support.TransactionSynchronizationAdapter;
import
org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.transaction.support.TransactionTemplate;
+import javax.persistence.OptimisticLockException;
+
@Component
public class RangerTransactionSynchronizationAdapter extends
TransactionSynchronizationAdapter {
@@ -156,7 +159,7 @@ public class RangerTransactionSynchronizationAdapter
extends TransactionSynchron
LOG.debug("Executing {" + runnables.size() + "} runnables");
}
for (Runnable runnable : runnables) {
- boolean isThisTransactionCommitted;
+ boolean isThisTransactionCommitted = false;
do {
try {
//Create new transaction
@@ -165,16 +168,22 @@ public class RangerTransactionSynchronizationAdapter
extends TransactionSynchron
Object result = txTemplate.execute(new
TransactionCallback<Object>() {
public Object doInTransaction(TransactionStatus
status) {
- Object result;
+ Object result = null;
if (LOG.isDebugEnabled()) {
LOG.debug("Executing runnable {" +
runnable + "}");
}
try {
runnable.run();
result = runnable;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("executed runnable " +
runnable);
+ }
+ } catch (OptimisticLockException
optimisticLockException) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to execute runnable
" + runnable + "because of OpmimisticLockException");
+ }
} catch (Throwable e) {
LOG.error("Failed to execute runnable " +
runnable, e);
- result = null;
}
return result;
}
@@ -182,12 +191,16 @@ public class RangerTransactionSynchronizationAdapter
extends TransactionSynchron
isThisTransactionCommitted = result == runnable;
- } catch (Exception e) {
+ } catch (OptimisticLockException optimisticLockException) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to commit TransactionService
transaction for runnable:[" + runnable + "]");
+ }
+ } catch (TransactionSystemException tse) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Failed to commit TransactionService
transaction for runnable:[" + runnable + "]", e);
+ LOG.debug("Failed to commit TransactionService
transaction, exception:[" + tse + "]");
}
- LOG.warn("Failed to commit TransactionService
transaction for runnable:[" + runnable + "]");
- isThisTransactionCommitted = false;
+ } catch (Throwable e){
+ LOG.warn("Failed to commit TransactionService
transaction, throwable:[" + e + "]");
}
} while (isParentTransactionCommitted &&
!isThisTransactionCommitted);
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyChangeLogDao.java
b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyChangeLogDao.java
index 047228b..6136bd3 100644
---
a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyChangeLogDao.java
+++
b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyChangeLogDao.java
@@ -31,7 +31,7 @@ import org.apache.ranger.entity.XXPolicy;
import org.apache.ranger.entity.XXPolicyChangeLog;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicyDelta;
-import org.apache.ranger.service.RangerPolicyService;
+import org.apache.ranger.plugin.util.JsonUtilsV2;
import org.springframework.stereotype.Service;
/**
@@ -56,7 +56,7 @@ public class XXPolicyChangeLogDao extends
BaseDao<XXPolicyChangeLog> {
super(daoManager);
}
- public List<RangerPolicyDelta> findLaterThan(RangerPolicyService
policyService, Long version, Long serviceId) {
+ public List<RangerPolicyDelta> findLaterThan(Long version, Long serviceId)
{
final List<RangerPolicyDelta> ret;
if (version != null) {
List<Object[]> logs = getEntityManager()
@@ -81,7 +81,7 @@ public class XXPolicyChangeLogDao extends
BaseDao<XXPolicyChangeLog> {
}
}
if (foundAndRemoved) {
- ret = convert(policyService, logs);
+ ret = convert(logs);
} else {
ret = null;
}
@@ -94,7 +94,7 @@ public class XXPolicyChangeLogDao extends
BaseDao<XXPolicyChangeLog> {
return ret;
}
- public List<RangerPolicyDelta> findGreaterThan(RangerPolicyService
policyService, Long id, Long serviceId) {
+ public List<RangerPolicyDelta> findGreaterThan(Long id, Long serviceId) {
final List<RangerPolicyDelta> ret;
if (id != null) {
List<Object[]> logs = getEntityManager()
@@ -102,7 +102,7 @@ public class XXPolicyChangeLogDao extends
BaseDao<XXPolicyChangeLog> {
.setParameter("id", id)
.setParameter("serviceId", serviceId)
.getResultList();
- ret = convert(policyService, logs);
+ ret = convert(logs);
} else {
ret = null;
}
@@ -120,7 +120,7 @@ public class XXPolicyChangeLogDao extends
BaseDao<XXPolicyChangeLog> {
getEntityManager().createNamedQuery("XXPolicyChangeLog.deleteOlderThan").setParameter("olderThan",
since).executeUpdate();
}
- private List<RangerPolicyDelta> convert(RangerPolicyService policyService,
List<Object[]> queryResult) {
+ private List<RangerPolicyDelta> convert(List<Object[]> queryResult) {
final List<RangerPolicyDelta> ret;
@@ -142,7 +142,8 @@ public class XXPolicyChangeLogDao extends
BaseDao<XXPolicyChangeLog> {
XXPolicy xxPolicy =
daoManager.getXXPolicy().getById(policyId);
if (xxPolicy != null) {
try {
- policy = policyService.read(policyId);
+ policy =
JsonUtilsV2.jsonToObj(xxPolicy.getPolicyText(), RangerPolicy.class);
+ policy.setId(policyId);
} catch (Exception e) {
LOG.error("Cannot read policy:[" + policyId + "].
Should not have come here!! Offending log-record-id:[" + logRecordId + "] and
returning...", e);
ret.clear();
diff --git
a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
index e8c593d..7ee8292 100644
---
a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
+++
b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
@@ -96,20 +96,34 @@ public abstract class RangerPolicyServiceBase<T extends
XXPolicyBase, V extends
@Override
protected T mapViewToEntityBean(V vObj, T xObj, int OPERATION_CONTEXT) {
- String guid = (StringUtils.isEmpty(vObj.getGuid())) ?
guidUtil.genGUID() : vObj.getGuid();
-
- xObj.setGuid(guid);
- xObj.setVersion(vObj.getVersion());
-
XXService xService =
daoMgr.getXXService().findByName(vObj.getService());
if (xService == null) {
throw restErrorUtil.createRESTException("No
corresponding service found for policyName: " + vObj.getName()
+ "Service Not Found : " +
vObj.getService(), MessageEnums.INVALID_INPUT_DATA);
}
+
+ String guid = vObj.getGuid();
+ if (StringUtils.isEmpty(guid)) {
+ guid = guidUtil.genGUID();
+ vObj.setGuid(guid);
+ }
+ Integer policyPriority = vObj.getPolicyPriority();
+ if (policyPriority == null) {
+ policyPriority = RangerPolicy.POLICY_PRIORITY_NORMAL;
+ vObj.setPolicyPriority(policyPriority);
+ }
+ Integer policyType = vObj.getPolicyType();
+ if (policyType == null) {
+ policyType = RangerPolicy.POLICY_TYPE_ACCESS;
+ vObj.setPolicyType(policyType);
+ }
+
+ xObj.setGuid(guid);
+ xObj.setVersion(vObj.getVersion());
xObj.setService(xService.getId());
xObj.setName(StringUtils.trim(vObj.getName()));
- xObj.setPolicyType(vObj.getPolicyType() == null ?
RangerPolicy.POLICY_TYPE_ACCESS : vObj.getPolicyType());
- xObj.setPolicyPriority(vObj.getPolicyPriority() == null ?
RangerPolicy.POLICY_PRIORITY_NORMAL : vObj.getPolicyPriority());
+ xObj.setPolicyType(policyType);
+ xObj.setPolicyPriority(policyPriority);
xObj.setDescription(vObj.getDescription());
xObj.setResourceSignature(vObj.getResourceSignature());
xObj.setIsAuditEnabled(vObj.getIsAuditEnabled());
