Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 445eeae67 -> 4a1365de9
SENTRY-1773: Delta change cleaner should leave way more then a single entry intact (Alex Kolbasov, reviewed by Vamsee Yarlagadda) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/4a1365de Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/4a1365de Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/4a1365de Branch: refs/heads/sentry-ha-redesign Commit: 4a1365de92281f87792a75a4829a5a3292446dda Parents: 445eeae Author: Alexander Kolbasov <[email protected]> Authored: Mon Jun 5 13:59:18 2017 -0700 Committer: Alexander Kolbasov <[email protected]> Committed: Mon Jun 5 13:59:18 2017 -0700 ---------------------------------------------------------------------- .../provider/db/service/persistent/SentryStore.java | 12 ++++++++---- .../apache/sentry/service/thrift/ServiceConstants.java | 8 ++++++++ .../provider/db/service/persistent/TestSentryStore.java | 7 +++++-- 3 files changed, 21 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/4a1365de/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 97b3636..37eb0b2 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -517,23 +517,27 @@ public class SentryStore { /** * Purge delta change tables, {@link MSentryPermChange} and {@link MSentryPathChange}. + * The number of deltas to keep is configurable */ public void purgeDeltaChangeTables() { - LOGGER.info("Purging MSentryPathUpdate and MSentyPermUpdate tables."); + final int changesToKeep = conf.getInt(ServerConfig.SENTRY_DELTA_KEEP_COUNT, + ServerConfig.SENTRY_DELTA_KEEP_COUNT_DEFAULT); + LOGGER.info("Purging MSentryPathUpdate and MSentyPermUpdate tables, leaving {} entries", + changesToKeep); try { tm.executeTransaction(new TransactionBlock<Object>() { @Override public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects - purgeDeltaChangeTableCore(MSentryPermChange.class, pm, 1); + purgeDeltaChangeTableCore(MSentryPermChange.class, pm, changesToKeep); LOGGER.info("MSentryPermChange table has been purged."); - purgeDeltaChangeTableCore(MSentryPathChange.class, pm, 1); + purgeDeltaChangeTableCore(MSentryPathChange.class, pm, changesToKeep); LOGGER.info("MSentryPathUpdate table has been purged."); return null; } }); } catch (Exception e) { - LOGGER.error("Delta change cleaning process encounter an error.", e); + LOGGER.error("Delta change cleaning process encounter an error", e); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/4a1365de/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java index 638c079..099ebd0 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java @@ -223,6 +223,14 @@ public class ServiceConstants { public static final String SENTRY_KERBEROS_TGT_AUTORENEW = "sentry.service.kerberos.tgt.autorenew"; @Deprecated public static final Boolean SENTRY_KERBEROS_TGT_AUTORENEW_DEFAULT = false; + + /** + * Number of path/priv deltas to keep around during cleaning + * The value which is too small may cause unnecessary full snapshots sent to the Name Node + * A value which is too large may cause slowdown due to too many deltas lying around in the DB. + */ + public static final String SENTRY_DELTA_KEEP_COUNT = "sentry.server.delta.keep.count"; + public static final int SENTRY_DELTA_KEEP_COUNT_DEFAULT = 200; } public static class ClientConfig { http://git-wip-us.apache.org/repos/asf/sentry/blob/4a1365de/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index c5dddfb..23b685b 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -59,6 +59,7 @@ import org.apache.sentry.provider.db.service.thrift.TSentryGroup; import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; import org.apache.sentry.provider.db.service.thrift.TSentryRole; import org.apache.sentry.provider.file.PolicyFile; +import org.apache.sentry.service.thrift.ServiceConstants; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.junit.After; import org.junit.AfterClass; @@ -2902,8 +2903,10 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(0, sentryStore.getMSentryPathChanges().size()); sentryStore.createSentryRole(role); + int privCleanCount = ServerConfig.SENTRY_DELTA_KEEP_COUNT_DEFAULT; + int extraPrivs = 5; - final int numPermChanges = 5; + final int numPermChanges = extraPrivs + privCleanCount; for (int i = 0; i < numPermChanges; i++) { TSentryPrivilege privilege = new TSentryPrivilege(); privilege.setPrivilegeScope("Column"); @@ -2920,7 +2923,7 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(numPermChanges, sentryStore.getMSentryPermChanges().size()); sentryStore.purgeDeltaChangeTables(); - assertEquals(1, sentryStore.getMSentryPermChanges().size()); + assertEquals(privCleanCount, sentryStore.getMSentryPermChanges().size()); // TODO: verify MSentryPathChange being purged. // assertEquals(1, sentryStore.getMSentryPathChanges().size());
