Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 268ee50ef -> 2811311ea
http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java deleted file mode 100644 index d12b134..0000000 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java +++ /dev/null @@ -1,359 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.hdfs; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.locks.ReadWriteLock; - -import org.apache.thrift.TException; -import org.junit.Assert; - -import org.apache.hadoop.conf.Configuration; -import org.apache.sentry.hdfs.Updateable.Update; -import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; -import org.junit.After; -import org.junit.Assume; -import org.junit.Test; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; - -public class TestUpdateForwarder { - - public static class DummyUpdate implements Update { - private long seqNum = 0; - private boolean hasFullUpdate = false; - private String state; - public DummyUpdate() { - this(0, false); - } - public DummyUpdate(long seqNum, boolean hasFullUpdate) { - this.seqNum = seqNum; - this.hasFullUpdate = hasFullUpdate; - } - public String getState() { - return state; - } - public DummyUpdate setState(String stuff) { - this.state = stuff; - return this; - } - @Override - public boolean hasFullImage() { - return hasFullUpdate; - } - @Override - public long getSeqNum() { - return seqNum; - } - @Override - public void setSeqNum(long seqNum) { - this.seqNum = seqNum; - } - @Override - public byte[] serialize() throws IOException { - return state.getBytes(); - } - - @Override - public void deserialize(byte[] data) throws IOException { - state = new String(data); - } - - @Override - public String JSONSerialize() throws TException { - return state; - } - - @Override - public void JSONDeserialize(String update) throws TException { - state = new String(update); - } - } - - static class DummyUpdatable implements Updateable<DummyUpdate> { - - private List<String> state = new LinkedList<String>(); - private long lastUpdatedSeqNum = 0; - - @Override - public void updatePartial(Iterable<DummyUpdate> update, ReadWriteLock lock) { - for (DummyUpdate u : update) { - state.add(u.getState()); - lastUpdatedSeqNum = u.seqNum; - } - } - - @Override - public Updateable<DummyUpdate> updateFull(DummyUpdate update) { - DummyUpdatable retVal = new DummyUpdatable(); - retVal.lastUpdatedSeqNum = update.seqNum; - retVal.state = Lists.newArrayList(update.state.split(",")); - return retVal; - } - - @Override - public long getLastUpdatedSeqNum() { - return lastUpdatedSeqNum; - } - - @Override - public DummyUpdate createFullImageUpdate(long currSeqNum) { - DummyUpdate retVal = new DummyUpdate(currSeqNum, true); - retVal.state = Joiner.on(",").join(state); - return retVal; - } - - public String getState() { - return Joiner.on(",").join(state); - } - - @Override - public String getUpdateableTypeName() { - // TODO Auto-generated method stub - return "DummyUpdator"; - } - } - - static class DummyImageRetreiver implements ImageRetriever<DummyUpdate> { - - private String state; - public void setState(String state) { - this.state = state; - } - @Override - public DummyUpdate retrieveFullImage(long currSeqNum) { - DummyUpdate retVal = new DummyUpdate(currSeqNum, true); - retVal.state = state; - return retVal; - } - } - - protected Configuration testConf = new Configuration(); - protected UpdateForwarder<DummyUpdate> updateForwarder; - - @After - public void cleanup() throws Exception { - if (updateForwarder != null) { - updateForwarder.close(); - updateForwarder = null; - } - } - - @Test - public void testInit() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10, true); - Assert.assertEquals(-2, updateForwarder.getLastUpdatedSeqNum()); - List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertTrue(allUpdates.size() == 1); - Assert.assertEquals("a,b,c", allUpdates.get(0).getState()); - - // If the current process has restarted the input seqNum will be > currSeq - allUpdates = updateForwarder.getAllUpdatesFrom(100); - Assert.assertTrue(allUpdates.size() == 1); - Assert.assertEquals("a,b,c", allUpdates.get(0).getState()); - Assert.assertEquals(-2, allUpdates.get(0).getSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(-1); - Assert.assertEquals(0, allUpdates.size()); - } - - @Test - public void testUpdateReceive() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(5, updateForwarder.getLastUpdatedSeqNum()); - List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(2, allUpdates.size()); - Assert.assertEquals("a,b,c", allUpdates.get(0).getState()); - Assert.assertEquals("d", allUpdates.get(1).getState()); - } - - // This happens when we the first update from HMS is a -1 (If the heartbeat - // thread checks Sentry's current seqNum before any update has come in).. - // This will lead the first and second entries in the updatelog to differ - // by more than +1.. - @Test - public void testUpdateReceiveWithNullImageRetriver() throws Exception { - Assume.assumeTrue(!testConf.getBoolean(ServerConfig.SENTRY_HA_ENABLED, - false)); - updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), null, 5, false); - updateForwarder.handleUpdateNotification(new DummyUpdate(-1, true).setState("a")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(1); - Assert.assertEquals("a", allUpdates.get(0).getState()); - updateForwarder.handleUpdateNotification(new DummyUpdate(6, false).setState("b")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - updateForwarder.handleUpdateNotification(new DummyUpdate(7, false).setState("c")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(7, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(2, allUpdates.size()); - Assert.assertEquals("b", allUpdates.get(0).getState()); - Assert.assertEquals("c", allUpdates.get(1).getState()); - } - - @Test - public void testGetUpdates() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(5, updateForwarder.getLastUpdatedSeqNum()); - List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(2, allUpdates.size()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(6, false).setState("e")); - updateForwarder.handleUpdateNotification(new DummyUpdate(7, false).setState("f")); - - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(7, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(4, allUpdates.size()); - Assert.assertEquals("a,b,c", allUpdates.get(0).getState()); - Assert.assertEquals(4, allUpdates.get(0).getSeqNum()); - Assert.assertEquals("d", allUpdates.get(1).getState()); - Assert.assertEquals(5, allUpdates.get(1).getSeqNum()); - Assert.assertEquals("e", allUpdates.get(2).getState()); - Assert.assertEquals(6, allUpdates.get(2).getSeqNum()); - Assert.assertEquals("f", allUpdates.get(3).getState()); - Assert.assertEquals(7, allUpdates.get(3).getSeqNum()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(8, false).setState("g")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(8, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(8); - Assert.assertEquals(1, allUpdates.size()); - Assert.assertEquals("g", allUpdates.get(0).getState()); - } - - @Test - public void testGetUpdatesAfterExternalEntityReset() throws Exception { - /* - * Disabled for Sentry HA. Since the sequence numbers are trakced in ZK, the - * lower sequence updates are ignored which causes this test to fail in HA - * mode - */ - Assume.assumeTrue(!testConf.getBoolean(ServerConfig.SENTRY_HA_ENABLED, - false)); - - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - - updateForwarder.handleUpdateNotification(new DummyUpdate(6, false).setState("e")); - updateForwarder.handleUpdateNotification(new DummyUpdate(7, false).setState("f")); - - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(7, updateForwarder.getLastUpdatedSeqNum()); - List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(4, allUpdates.size()); - Assert.assertEquals("f", allUpdates.get(3).getState()); - Assert.assertEquals(7, allUpdates.get(3).getSeqNum()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(8, false).setState("g")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(8, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(8); - Assert.assertEquals(1, allUpdates.size()); - Assert.assertEquals("g", allUpdates.get(0).getState()); - - imageRetreiver.setState("a,b,c,d,e,f,g,h"); - - // New update comes with SeqNum = 1 - updateForwarder.handleUpdateNotification(new DummyUpdate(1, false).setState("h")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - // NN plugin asks for next update - allUpdates = updateForwarder.getAllUpdatesFrom(9); - Assert.assertEquals(1, allUpdates.size()); - Assert.assertEquals("a,b,c,d,e,f,g,h", allUpdates.get(0).getState()); - // Assert.assertEquals(1, allUpdates.get(0).getSeqNum()); - } - - @Test - public void testUpdateLogCompression() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - updateForwarder = UpdateForwarder.create( - testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(5, updateForwarder.getLastUpdatedSeqNum()); - List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(2, allUpdates.size()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(6, false).setState("e")); - updateForwarder.handleUpdateNotification(new DummyUpdate(7, false).setState("f")); - updateForwarder.handleUpdateNotification(new DummyUpdate(8, false).setState("g")); - updateForwarder.handleUpdateNotification(new DummyUpdate(9, false).setState("h")); - updateForwarder.handleUpdateNotification(new DummyUpdate(10, false).setState("i")); - updateForwarder.handleUpdateNotification(new DummyUpdate(11, false).setState("j")); - - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(11, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(3, allUpdates.size()); - Assert.assertEquals("a,b,c,d,e,f,g,h", allUpdates.get(0).getState()); - Assert.assertEquals(9, allUpdates.get(0).getSeqNum()); - Assert.assertEquals("i", allUpdates.get(1).getState()); - Assert.assertEquals(10, allUpdates.get(1).getSeqNum()); - Assert.assertEquals("j", allUpdates.get(2).getState()); - Assert.assertEquals(11, allUpdates.get(2).getSeqNum()); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 6ea6d3f..bbfa713 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -3156,7 +3156,7 @@ public class SentryStore { } /** - * Get the last processed change ID for perm/path delta changes. + * Gets the last processed change ID for perm/path delta changes. * * @param pm the PersistenceManager * @param changeCls the class of a delta c @@ -3164,7 +3164,7 @@ public class SentryStore { * @return the last processed changedID for the delta changes. If no * change found then return 0. */ - private <T extends MSentryChange> long getLastProcessedChangeIDCore( + private <T extends MSentryChange> Long getLastProcessedChangeIDCore( PersistenceManager pm, Class<T> changeCls) { Query query = pm.newQuery(changeCls); query.setResult("max(changeID)"); @@ -3173,14 +3173,13 @@ public class SentryStore { } /** - * Get the last processed change ID for perm delta changes. + * Gets the last processed change ID for perm delta changes. * * Internally invoke {@link #getLastProcessedChangeIDCore(PersistenceManager, Class)} * * @return latest perm change ID. */ - @VisibleForTesting - long getLastProcessedPermChangeID() throws Exception { + public Long getLastProcessedPermChangeID() throws Exception { return tm.executeTransaction( new TransactionBlock<Long>() { public Long execute(PersistenceManager pm) throws Exception { @@ -3190,12 +3189,26 @@ public class SentryStore { } /** + * Gets the last processed change ID for path delta changes. + * + * @return latest path change ID. + */ + public Long getLastProcessedPathChangeID() throws Exception { + return tm.executeTransaction( + new TransactionBlock<Long>() { + public Long execute(PersistenceManager pm) throws Exception { + return getLastProcessedChangeIDCore(pm, MSentryPathChange.class); + } + }); + } + + /** * Get the notification ID of last processed path delta change. * * @return the notification ID of latest path change. If no change * found then return 0. */ - public long getLastProcessedNotificationID() throws Exception { + public Long getLastProcessedNotificationID() throws Exception { return tm.executeTransaction( new TransactionBlock<Long>() { public Long execute(PersistenceManager pm) throws Exception { @@ -3221,8 +3234,8 @@ public class SentryStore { new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { Query query = pm.newQuery(MSentryPermChange.class); - query.setFilter("this.changeID == t"); - query.declareParameters("long t"); + query.setFilter("this.changeID == id"); + query.declareParameters("long id"); List<MSentryPermChange> permChanges = (List<MSentryPermChange>)query.execute(changeID); if (permChanges == null) { noSuchUpdate(changeID); @@ -3244,7 +3257,8 @@ public class SentryStore { * @throws Exception */ @SuppressWarnings("unchecked") - private <T extends MSentryChange> List<T> getMSentryChanges(final Class<T> cls) throws Exception { + private <T extends MSentryChange> List<T> getMSentryChanges(final Class<T> cls) + throws Exception { return tm.executeTransaction( new TransactionBlock<List<T>>() { public List<T> execute(PersistenceManager pm) throws Exception { @@ -3266,15 +3280,71 @@ public class SentryStore { } /** - * Get the MSentryPathChange object by ChangeID. + * Checks if any MSentryChange object exists with the given changeID. + * + * @param pm PersistenceManager + * @param changeCls class instance of type {@link MSentryChange} + * @param changeID changeID + * @return true if found the MSentryChange object, otherwise false. + * @throws Exception + */ + @SuppressWarnings("unchecked") + private <T extends MSentryChange> Boolean changeExistsCore( + PersistenceManager pm, Class<T> changeCls, final long changeID) + throws Exception { + Query query = pm.newQuery(changeCls); + query.setFilter("this.changeID == id"); + query.declareParameters("long id"); + List<T> changes = (List<T>)query.execute(changeID); + return !changes.isEmpty(); + } + + /** + * Checks if any MSentryPermChange object exists with the given changeID. + * + * @param changeID + * @return true if found the MSentryPermChange object, otherwise false. + * @throws Exception + */ + public Boolean permChangeExists(final long changeID) throws Exception { + return tm.executeTransaction( + new TransactionBlock<Boolean>() { + public Boolean execute(PersistenceManager pm) throws Exception { + return changeExistsCore(pm, MSentryPermChange.class, changeID); + } + }); + } + + /** + * Checks if any MSentryPathChange object exists with the given changeID. + * + * @param changeID + * @return true if found the MSentryPathChange object, otherwise false. + * @throws Exception + */ + public Boolean pathChangeExists(final long changeID) throws Exception { + return tm.executeTransaction( + new TransactionBlock<Boolean>() { + public Boolean execute(PersistenceManager pm) throws Exception { + return changeExistsCore(pm, MSentryPathChange.class, changeID); + } + }); + } + + /** + * Gets the MSentryPathChange object by ChangeID. + * + * @param changeID the given changeID + * @return the MSentryPathChange object with corresponding changeID. + * @throws Exception */ public MSentryPathChange getMSentryPathChangeByID(final long changeID) throws Exception { return (MSentryPathChange) tm.executeTransaction( new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { Query query = pm.newQuery(MSentryPathChange.class); - query.setFilter("this.changeID == t"); - query.declareParameters("long t"); + query.setFilter("this.changeID == id"); + query.declareParameters("long id"); List<MSentryPathChange> pathChanges = (List<MSentryPathChange>)query.execute(changeID); if (pathChanges == null) { noSuchUpdate(changeID); @@ -3297,6 +3367,88 @@ public class SentryStore { } /** + * Gets a list of MSentryChange objects greater than or equal to the given changeID. + * + * @param changeID + * @return a list of MSentryChange objects. It can returns an empty list. + * @throws Exception + */ + @SuppressWarnings("unchecked") + private <T extends MSentryChange> List<T> getMSentryChangesCore(PersistenceManager pm, + Class<T> changeCls, final long changeID) throws Exception { + Query query = pm.newQuery(changeCls); + query.setFilter("this.changeID >= t"); + query.declareParameters("long t"); + List<T> changes = (List<T>) query.execute(changeID); + return changes; + } + + /** + * Gets a list of MSentryPathChange objects greater than or equal to the given changeID. + * If there is any path deltas missing in {@link MSentryPathChange} table, which means + * the size of retrieved paths deltas is less than the requested one, an empty list will + * be returned to caller. + * + * @param changeID + * @return a list of MSentryPathChange objects. It can returns an empty list. + * @throws Exception + */ + public List<MSentryPathChange> getMSentryPathChanges(final long changeID) + throws Exception { + return tm.executeTransaction(new TransactionBlock<List<MSentryPathChange>>() { + public List<MSentryPathChange> execute(PersistenceManager pm) throws Exception { + List<MSentryPathChange> pathChanges = + getMSentryChangesCore(pm, MSentryPathChange.class, changeID); + long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPathChange.class); + long expectedSize = curChangeID - changeID + 1; + long actualSize = pathChanges.size(); + if (actualSize < expectedSize) { + LOGGER.error(String.format("Certain path delta is missing in " + + "SENTRY_PATH_CHANEG table! Current size of elements = %s and expected size = %s, " + + "from changeID: %s. The table may get corrupted.", + actualSize, expectedSize, changeID)); + return Collections.emptyList(); + } else { + return pathChanges; + } + } + }); + } + + /** + * Gets a list of MSentryPermChange objects greater than or equal to the given ChangeID. + * If there is any perm deltas missing in {@link MSentryPermChange} table, which means + * the size of retrieved perm deltas is less than the requested one, an empty list will + * be returned to caller. + * + * @param changeID + * @return a list of MSentryPermChange objects + * @throws Exception + */ + public List<MSentryPermChange> getMSentryPermChanges(final long changeID) + throws Exception { + return tm.executeTransaction( + new TransactionBlock<List<MSentryPermChange>>() { + public List<MSentryPermChange> execute(PersistenceManager pm) throws Exception { + List<MSentryPermChange> permChanges = + getMSentryChangesCore(pm, MSentryPermChange.class, changeID); + long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPermChange.class); + long expectedSize = curChangeID - changeID + 1; + long actualSize = permChanges.size(); + if (actualSize < expectedSize) { + LOGGER.error(String.format("Certain perm delta is missing in " + + "SENTRY_PERM_CHANEG table! Current size of elements = %s and expected size = %s, " + + "from changeID: %s. The table may get corrupted.", + actualSize, expectedSize, changeID)); + return Collections.emptyList(); + } else { + return permChanges; + } + } + }); + } + + /** * Execute Perm/Path UpdateTransaction and corresponding actual * action transaction, e.g dropSentryRole, in a single transaction. * The order of the transaction does not matter because there is no http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index f3f51da..6c14f5e 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -32,9 +32,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.core.common.exception.*; -import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.hdfs.PermissionsUpdate; -import org.apache.sentry.hdfs.UpdateableAuthzPaths; import org.apache.sentry.hdfs.FullUpdateInitializer; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index 75f855c..aaa0b9f 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -2371,8 +2371,17 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(0, privileges.size()); // Query the persisted perm change and ensure it equals to the original one - MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID + 1); + lastChangeID = sentryStore.getLastProcessedPermChangeID(); + MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID); assertEquals(delUpdate.JSONSerialize(), delPermChange.getPermChange()); + + // Verify getMSentryPermChanges will return all MSentryPermChanges up + // to the given changeID. + List<MSentryPermChange> mSentryPermChanges = sentryStore.getMSentryPermChanges(1); + assertEquals(lastChangeID, mSentryPermChanges.size()); + + // Verify ifPermChangeExists will return true for persisted MSentryPermChange. + assertEquals(true, sentryStore.permChangeExists(1)); } @Test @@ -2480,7 +2489,7 @@ public class TestSentryStore extends org.junit.Assert { @Test public void testRenameObjWithPermUpdate() throws Exception { - String roleName1 = "role1", roleName2 = "role2", roleName3 = "role3"; + String roleName1 = "role1"; String grantor = "g1"; String table1 = "tbl1", table2 = "tbl2"; http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java index 5e8b2fa..1530eb2 100644 --- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java +++ b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java @@ -216,7 +216,6 @@ public class TestHDFSIntegration { @Override public Void run() throws Exception { HiveConf hiveConf = new HiveConf(); - hiveConf.set("sentry.metastore.plugins", "org.apache.sentry.hdfs.MetastorePlugin"); hiveConf.set("sentry.service.client.server.rpc-address", "localhost"); hiveConf.set("sentry.hdfs.service.client.server.rpc-address", "localhost"); hiveConf.set("sentry.hdfs.service.client.server.rpc-port", String.valueOf(sentryPort)); @@ -444,6 +443,7 @@ public class TestHDFSIntegration { properties.put(ServerConfig.RPC_ADDRESS, "localhost"); properties.put(ServerConfig.RPC_PORT, String.valueOf(sentryPort > 0 ? sentryPort : 0)); properties.put(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false"); + properties.put(ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED,"true"); properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING); properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath()); http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java index 1b5eb53..8de4f29 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.metastore.api.Table; /** * Advanced tests for HDFS Sync integration */ +@Ignore public class TestHDFSIntegrationAdvanced extends TestHDFSIntegrationBase { private static final Logger LOGGER = LoggerFactory http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java index 859c8f8..7769f24 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java @@ -497,7 +497,7 @@ public abstract class TestHDFSIntegrationBase { hiveConf.set("hive.metastore.authorization.storage.checks", "true"); hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort); hiveConf.set("hive.metastore.pre.event.listeners", "org.apache.sentry.binding.metastore.MetastoreAuthzBinding"); - hiveConf.set("hive.metastore.event.listeners", "org.apache.sentry.binding.metastore.SentryMetastorePostEventListener"); + hiveConf.set("hive.metastore.transactional.event.listeners", "org.apache.hive.hcatalog.listener.DbNotificationListener"); hiveConf.set("hive.security.authorization.task.factory", "org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl"); hiveConf.set("hive.server2.session.hook", "org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook"); hiveConf.set("sentry.metastore.service.users", "hive");// queries made by hive user (beeline) skip meta store check @@ -696,6 +696,7 @@ public abstract class TestHDFSIntegrationBase { properties.put(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy"); properties.put("sentry.service.processor.factories", "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + properties.put(ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED,"true"); properties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin"); properties.put(ServerConfig.RPC_MIN_THREADS, "3"); for (Map.Entry<String, String> entry : properties.entrySet()) { http://git-wip-us.apache.org/repos/asf/sentry/blob/2811311e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java index c791272..0e97466 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java @@ -40,6 +40,7 @@ import java.util.ArrayList; /** * This test class includes all HDFS Sync smoke tests */ +@Ignore public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase { private static final Logger LOGGER = LoggerFactory .getLogger(TestHDFSIntegrationEnd2End.class);
