Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 3817f18be -> b596fe9b8
SENTRY-1611: Purge MSentryPerm/PathChange tables (Lei (Eddy) Xu, Reviewed by: Hao Hao and Alexander Kolbasov) Change-Id: I59ec5b83abd7f86cadc434e46caf9be142dd3ab6 Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/b596fe9b Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/b596fe9b Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/b596fe9b Branch: refs/heads/sentry-ha-redesign Commit: b596fe9b884db583215a58a3309677590969b71f Parents: 3817f18 Author: hahao <[email protected]> Authored: Wed Feb 22 11:28:49 2017 -0800 Committer: hahao <[email protected]> Committed: Wed Feb 22 11:28:49 2017 -0800 ---------------------------------------------------------------------- .../db/service/model/MSentryChange.java | 24 ++++ .../db/service/model/MSentryPathChange.java | 2 +- .../db/service/model/MSentryPermChange.java | 2 +- .../db/service/persistent/SentryStore.java | 117 ++++++++++++++++--- .../db/service/persistent/TestSentryStore.java | 33 ++++++ 5 files changed, 162 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java new file mode 100644 index 0000000..6011ef4 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.provider.db.service.model; + +/** + * The base class for various delta changes stored in Sentry DB. + */ +public interface MSentryChange { +} http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java index 12e9a09..0ca7fe2 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java @@ -57,7 +57,7 @@ import javax.jdo.annotations.PrimaryKey; */ @PersistenceCapable -public class MSentryPathChange { +public class MSentryPathChange implements MSentryChange { @PrimaryKey private long changeID; http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java index 3662bfd..476fbcb 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java @@ -55,7 +55,7 @@ import javax.jdo.annotations.PrimaryKey; * can re-enhance. */ @PersistenceCapable -public class MSentryPermChange { +public class MSentryPermChange implements MSentryChange { @PrimaryKey private long changeID; http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 4cc123f..2ae9f3b 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -425,6 +425,8 @@ public class SentryStore { pm.newQuery(MSentryGroup.class).deletePersistentAll(); pm.newQuery(MSentryUser.class).deletePersistentAll(); pm.newQuery(MSentryPrivilege.class).deletePersistentAll(); + pm.newQuery(MSentryPermChange.class).deletePersistentAll(); + pm.newQuery(MSentryPathChange.class).deletePersistentAll(); return null; } }); @@ -435,6 +437,56 @@ public class SentryStore { } /** + * Purge a given delta change table, with a specified number of changes to be kept. + * + * @param cls the class of a perm/path delta change {@link MSentryPermChange} or + * {@link MSentryPathChange}. + * @param pm a {@link PersistenceManager} instance. + * @param changesToKeep the number of changes the caller want to keep. + * @param <T> the type of delta change class. + */ + @VisibleForTesting + <T extends MSentryChange> void purgeDeltaChangeTableCore( + Class<T> cls, PersistenceManager pm, long changesToKeep) { + Preconditions.checkArgument(changesToKeep >= 0, + "changes to keep must be a non-negative number"); + long lastChangedID = getLastProcessedChangeIDCore(pm, cls); + long maxIDDeleted = lastChangedID - changesToKeep; + + Query query = pm.newQuery(cls); + + // It is an approximation of "SELECT ... LIMIT CHANGE_TO_KEEP" in SQL, because JDO w/ derby + // does not support "LIMIT". + // See: http://www.datanucleus.org/products/datanucleus/jdo/jdoql_declarative.html + query.setFilter("changeID <= maxChangedIdDeleted"); + query.declareParameters("long maxChangedIdDeleted"); + long numDeleted = query.deletePersistentAll(maxIDDeleted); + LOGGER.info(String.format("Purged %d of %s to changeID=%d", + numDeleted, cls.getSimpleName(), maxIDDeleted)); + } + + /** + * Purge delta change tables, {@link MSentryPermChange} and {@link MSentryPathChange}. + */ + public void purgeDeltaChangeTables() { + LOGGER.info("Purging MSentryPathUpdate and MSentyPermUpdate tables."); + try { + tm.executeTransaction(new TransactionBlock<Object>() { + @Override + public Object execute(PersistenceManager pm) throws Exception { + purgeDeltaChangeTableCore(MSentryPermChange.class, pm, 1); + LOGGER.info("MSentryPermChange table has been purged."); + purgeDeltaChangeTableCore(MSentryPathChange.class, pm, 1); + LOGGER.info("MSentryPathUpdate table has been purged."); + return null; + } + }); + } catch (Exception e) { + LOGGER.error("Delta change cleaning process encounter an error.", e); + } + } + + /** * Alter a given sentry role to grant a privilege. * * @param grantorPrincipal User name @@ -3289,34 +3341,34 @@ public class SentryStore { } /** - * Get the last processed perm change ID. + * Get the last processed change ID for perm/path delta changes. * * @param pm the PersistenceManager - * @return the last processed perm changedID + * @param changeCls the class of a delta c + * + * @return the last processed changedID for the delta changes. */ - private long getLastProcessedPermChangeIDCore(PersistenceManager pm) { - Query query = pm.newQuery(MSentryPermChange.class); - query.setResult("max(this.changeID)"); + private <T extends MSentryChange> long getLastProcessedChangeIDCore( + PersistenceManager pm, Class<T> changeCls) { + Query query = pm.newQuery(changeCls); + query.setResult("max(changeID)"); Long changeID = (Long) query.execute(); - if (changeID == null) { - return EMPTY_CHANGE_ID; - } else { - return changeID; - } + return changeID == null ? EMPTY_CHANGE_ID : changeID; } /** - * Get the MSentryPermChange object by ChangeID. Internally invoke - * getLastProcessedPermChangeIDCore(). + * Get the last processed change ID for perm delta changes. * - * @return MSentryPermChange + * Internally invoke {@link #getLastProcessedChangeIDCore(PersistenceManager, Class)} + * + * @return latest perm change ID. */ @VisibleForTesting long getLastProcessedPermChangeID() throws Exception { return tm.executeTransaction( new TransactionBlock<Long>() { public Long execute(PersistenceManager pm) throws Exception { - return getLastProcessedPermChangeIDCore(pm); + return getLastProcessedChangeIDCore(pm, MSentryPermChange.class); } }); } @@ -3348,6 +3400,35 @@ public class SentryStore { } /** + * Fetch all {@link MSentryChange} in the database. + * + * @param cls the class of the Sentry delta change. + * @return a list of Sentry delta changes. + * @throws Exception + */ + @SuppressWarnings("unchecked") + private <T extends MSentryChange> List<T> getMSentryChanges(final Class<T> cls) throws Exception { + return tm.executeTransaction( + new TransactionBlock<List<T>>() { + public List<T> execute(PersistenceManager pm) throws Exception { + Query query = pm.newQuery(cls); + return (List<T>) query.execute(); + } + }); + } + + /** + * Fetch all {@link MSentryPermChange} in the database. It should only be used in the tests. + * + * @return a list of permission changes. + * @throws Exception + */ + @VisibleForTesting + List<MSentryPermChange> getMSentryPermChanges() throws Exception { + return getMSentryChanges(MSentryPermChange.class); + } + + /** * Get the MSentryPathChange object by ChangeID. */ public MSentryPathChange getMSentryPathChangeByID(final long changeID) throws Exception { @@ -3371,6 +3452,14 @@ public class SentryStore { } /** + * Fetch all {@link MSentryPathChange} in the database. It should only be used in the tests. + */ + @VisibleForTesting + List<MSentryPathChange> getMSentryPathChanges() throws Exception { + return getMSentryChanges(MSentryPathChange.class); + } + + /** * Execute Perm/Path UpdateTransaction and corresponding actual * action transaction, e.g dropSentryRole, in a single transaction. * The order of the transaction does not matter because there is no http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index dfaac15..d9fce69 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -2479,4 +2479,37 @@ public class TestSentryStore extends org.junit.Assert { policyFile.write(policyFilePath); } + @Test + public void testPurgeDeltaChanges() throws Exception { + String role = "purgeRole"; + String grantor = "g1"; + String table = "purgeTable"; + + assertEquals(0, sentryStore.getMSentryPermChanges().size()); + assertEquals(0, sentryStore.getMSentryPathChanges().size()); + + sentryStore.createSentryRole(role); + + final int numPermChanges = 5; + for (int i = 0; i < numPermChanges; i++) { + TSentryPrivilege privilege = new TSentryPrivilege(); + privilege.setPrivilegeScope("Column"); + privilege.setServerName("server"); + privilege.setDbName("db"); + privilege.setTableName(table); + privilege.setColumnName("column"); + privilege.setAction(AccessConstants.SELECT); + privilege.setCreateTime(System.currentTimeMillis()); + + PermissionsUpdate update = new PermissionsUpdate(i + 1, false); + sentryStore.alterSentryRoleGrantPrivilege(grantor, role, privilege, update); + } + assertEquals(numPermChanges, sentryStore.getMSentryPermChanges().size()); + + sentryStore.purgeDeltaChangeTables(); + assertEquals(1, sentryStore.getMSentryPermChanges().size()); + + // TODO: verify MSentryPathChange being purged. + // assertEquals(1, sentryStore.getMSentryPathChanges().size()); + } }
