This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 774d159e2 RANGER-4185: Improve debugging messages when policy-deltas
are enabled
774d159e2 is described below
commit 774d159e2a2967132e8a1eda7f5ddeed08b37a55
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Tue Apr 18 17:15:15 2023 -0700
RANGER-4185: Improve debugging messages when policy-deltas are enabled
---
.../ranger/plugin/model/RangerPolicyDelta.java | 2 +-
.../ranger/plugin/policyengine/PolicyEngine.java | 10 +++++++-
.../ranger/plugin/util/RangerPolicyDeltaUtil.java | 2 +-
.../java/org/apache/ranger/biz/ServiceDBStore.java | 29 +++++++++++++++++++---
.../java/org/apache/ranger/biz/TagDBStore.java | 3 +++
.../ranger/common/RangerServicePoliciesCache.java | 2 +-
.../RangerTransactionSynchronizationAdapter.java | 15 +++++++++--
7 files changed, 53 insertions(+), 10 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java
index 33183727c..e4d9b3a40 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java
@@ -87,7 +87,7 @@ public class RangerPolicyDelta implements
java.io.Serializable {
public void setId(Long id) { this.id = id;}
- private void setChangeType(Integer changeType) { this.changeType =
changeType; }
+ public void setChangeType(Integer changeType) { this.changeType =
changeType; }
private void setPoliciesVersion(Long policiesVersion) {
this.policiesVersion = policiesVersion; }
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
index 3864f30d2..86b6cd376 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
@@ -200,7 +200,15 @@ public class PolicyEngine {
this.pluginContext = pluginContext;
this.lock = new RangerReadWriteLock(isUseReadWriteLock);
- LOG.info("Policy engine will" + (isUseReadWriteLock ? " " : " not ") +
"perform in place update while processing policy-deltas.");
+ Boolean hasPolicyDeltas =
RangerPolicyDeltaUtil.hasPolicyDeltas(servicePolicies);
+
+ if (hasPolicyDeltas != null) {
+ if (hasPolicyDeltas.equals(Boolean.TRUE)) {
+ LOG.info("Policy engine will" + (isUseReadWriteLock ? " " : "
not ") + "perform in place update while processing policy-deltas.");
+ } else {
+ LOG.info("Policy engine will" + (isUseReadWriteLock ? " " : "
not ") + "perform in place update while processing policies.");
+ }
+ }
this.pluginContext.setAuthContext(new RangerAuthContext(null, roles));
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
index 86b18aace..b47888e9a 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
@@ -42,7 +42,7 @@ public class RangerPolicyDeltaUtil {
public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies,
List<RangerPolicyDelta> deltas, String serviceType) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")");
+ LOG.debug("==> applyDeltas(serviceType=" + serviceType + ",
deltas=" + deltas + ")");
}
List<RangerPolicy> ret;
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index e52a92e04..60903cc97 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -393,6 +393,12 @@ public class ServiceDBStore extends AbstractServiceStore {
isRolesDownloadedByService =
config.getBoolean("ranger.support.for.service.specific.role.download", false);
SUPPORTS_IN_PLACE_POLICY_UPDATES =
SUPPORTS_POLICY_DELTAS && config.getBoolean("ranger.admin" +
RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES,
RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT);
+ LOG.info("SUPPORTS_POLICY_DELTAS=" +
SUPPORTS_POLICY_DELTAS);
+ LOG.info("RETENTION_PERIOD_IN_DAYS=" +
RETENTION_PERIOD_IN_DAYS);
+
LOG.info("TAG_RETENTION_PERIOD_IN_DAYS=" + TAG_RETENTION_PERIOD_IN_DAYS);
+ LOG.info("isRolesDownloadedByService="
+ isRolesDownloadedByService);
+
LOG.info("SUPPORTS_IN_PLACE_POLICY_UPDATES=" +
SUPPORTS_IN_PLACE_POLICY_UPDATES);
+
TransactionTemplate txTemplate = new
TransactionTemplate(txManager);
final ServiceDBStore dbStore = this;
@@ -2924,11 +2930,16 @@ public class ServiceDBStore extends
AbstractServiceStore {
@Override
public ServicePolicies getServicePolicyDeltas(String serviceName, Long
lastKnownVersion) throws Exception {
- boolean getOnlyDeltas = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Support for incremental policy updates
enabled using \"ranger.admin" +
RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA + "\" configuation
parameter :[" + SUPPORTS_POLICY_DELTAS +"]");
+ ServicePolicies ret = null;
+
+ if (SUPPORTS_POLICY_DELTAS) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Support for incremental policy
updates enabled using \"ranger.admin" +
RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA + "\" configuation
parameter :[" + SUPPORTS_POLICY_DELTAS + "]");
+ }
+ ret = getServicePolicies(serviceName, lastKnownVersion,
true, SUPPORTS_POLICY_DELTAS);
}
- return getServicePolicies(serviceName, lastKnownVersion,
getOnlyDeltas, SUPPORTS_POLICY_DELTAS);
+
+ return ret;
}
@Override
@@ -3104,6 +3115,9 @@ public class ServiceDBStore extends AbstractServiceStore {
break;
}
}
+
policyDeltasForPolicy.clear();
+
policyDeltas.get(index).setChangeType(RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE);
+
policyDeltasForPolicy.add(policyDeltas.get(index));
index++;
break;
case
RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE:
@@ -3174,8 +3188,15 @@ public class ServiceDBStore extends AbstractServiceStore
{
break;
}
if (policyDeltasForPolicy != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processed multiple
deltas for policy:[" + entry.getKey() + "], compressed-deltas:[" +
policyDeltasForPolicy + "]");
+ }
+ if (policyDeltasForPolicy.size() > 1) {
+ LOG.error("More than one
Compressed-deltas for policy:[" + entry.getKey() + "], compressed-deltas:[" +
policyDeltasForPolicy + "].. Should not have come here!!");
+ }
ret.addAll(policyDeltasForPolicy);
} else {
+ LOG.error("Error processing deltas for
policy:[" + entry.getKey() + "], Cannot compress deltas");
ret = null;
break;
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index e434cf1bb..fb912d4f8 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -1367,6 +1367,9 @@ public class TagDBStore extends AbstractTagStore {
SUPPORTS_TAG_DELTAS = config.getBoolean("ranger.admin"
+ RangerCommonConstants.RANGER_ADMIN_SUFFIX_TAG_DELTA,
RangerCommonConstants.RANGER_ADMIN_SUFFIX_TAG_DELTA_DEFAULT);
SUPPORTS_IN_PLACE_TAG_UPDATES = SUPPORTS_TAG_DELTAS
&& config.getBoolean("ranger.admin" +
RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES,
RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT);
IS_SUPPORTS_TAG_DELTAS_INITIALIZED = true;
+
+ LOG.info("SUPPORTS_TAG_DELTAS=" + SUPPORTS_TAG_DELTAS);
+ LOG.info("SUPPORTS_IN_PLACE_TAG_UPDATES=" +
SUPPORTS_IN_PLACE_TAG_UPDATES);
}
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
index a34d7d1d7..21f06834d 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
@@ -351,12 +351,12 @@ public class RangerServicePoliciesCache {
if (LOG.isDebugEnabled()) {
LOG.debug("Retrieved
policy-deltas from database. These will be applied on top of ServicePolicy
version:[" + cachedServicePoliciesVersion +"], policy-deltas:[" +
servicePoliciesFromDb.getPolicyDeltas() + "]");
}
-
servicePolicies.setPolicyVersion(servicePoliciesFromDb.getPolicyVersion());
final List<RangerPolicy>
policies = servicePolicies.getPolicies() == null ? new ArrayList<>() :
servicePolicies.getPolicies();
final List<RangerPolicy>
newPolicies = RangerPolicyDeltaUtil.applyDeltas(policies,
servicePoliciesFromDb.getPolicyDeltas(),
servicePolicies.getServiceDef().getName());
servicePolicies.setPolicies(newPolicies);
+
servicePolicies.setPolicyVersion(servicePoliciesFromDb.getPolicyVersion());
checkCacheSanity(serviceName,
serviceStore, false);
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
index ff1165480..d84d772a9 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
@@ -186,13 +186,22 @@ public class RangerTransactionSynchronizationAdapter
extends TransactionSynchron
LOG.debug("Failed to execute runnable
" + runnable + "because of OpmimisticLockException");
}
} catch (Throwable e) {
- LOG.error("Failed to execute runnable " +
runnable, e);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to execute runnable
" + runnable, e);
+ }
}
return result;
}
});
isThisTransactionCommitted = result == runnable;
+ if (isParentTransactionCommitted) {
+ if (!isThisTransactionCommitted) {
+ LOG.info("Failed to commit runnable:[" +
runnable + "]. Will retry!");
+ } else {
+ LOG.info("Committed runnable:[" + runnable +
"].");
+ }
+ }
} catch (OptimisticLockException optimisticLockException) {
if (LOG.isDebugEnabled()) {
@@ -203,7 +212,9 @@ public class RangerTransactionSynchronizationAdapter
extends TransactionSynchron
LOG.debug("Failed to commit TransactionService
transaction, exception:[" + tse + "]");
}
} catch (Throwable e){
- LOG.warn("Failed to commit TransactionService
transaction, throwable:[" + e + "]");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to commit TransactionService
transaction, throwable:[" + e + "]");
+ }
}
} while (isParentTransactionCommitted &&
!isThisTransactionCommitted);
}
