Repository: sentry Updated Branches: refs/heads/master 398a8a937 -> a7a2d69c7
SENTRY-1918: NN snapshot should not be served while HMS snapshot is collected (Brian Towles, reviewed by Alexander Kolbasov, Sergio Pena, Na Li) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/a7a2d69c Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/a7a2d69c Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/a7a2d69c Branch: refs/heads/master Commit: a7a2d69c798d562bb1dbc94f73761d0a3d6ee214 Parents: 398a8a9 Author: Sergio Pena <[email protected]> Authored: Fri Sep 8 17:26:12 2017 -0500 Committer: Sergio Pena <[email protected]> Committed: Fri Sep 8 17:26:12 2017 -0500 ---------------------------------------------------------------------- .../apache/sentry/hdfs/ServiceConstants.java | 2 + .../apache/sentry/hdfs/DBUpdateForwarder.java | 32 ++-- .../sentry/hdfs/TestDBUpdateForwarder.java | 45 ++++++ .../sentry/service/thrift/HMSFollower.java | 112 +++++++------ .../sentry/service/thrift/HMSFollowerState.java | 43 +++++ .../sentry/service/thrift/SentryService.java | 2 + .../service/thrift/SentryServiceState.java | 44 +++++ .../sentry/service/thrift/SentryState.java | 27 ++++ .../sentry/service/thrift/SentryStateBank.java | 159 +++++++++++++++++++ .../thrift/SentryStateBankTestHelper.java | 29 ++++ .../service/thrift/TestSentryStateBank.java | 84 ++++++++++ 11 files changed, 523 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java index 0ff717d..d65207f 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ServiceConstants.java @@ -26,6 +26,8 @@ public class ServiceConstants { // number used in authz paths and permissions to request initial syncs static final long IMAGE_NUMBER_UPDATE_UNINITIALIZED = 0L; + static final long SEQUENCE_NUMBER_FULL_UPDATE_REQUEST = SEQUENCE_NUMBER_UPDATE_UNINITIALIZED + 1; + public static class ServerConfig { /** * This configuration parameter is only meant to be used for testing purposes. http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java index bb85c13..8a34d56 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java @@ -17,17 +17,17 @@ */ package org.apache.sentry.hdfs; +import static org.apache.sentry.hdfs.ServiceConstants.IMAGE_NUMBER_UPDATE_UNINITIALIZED; +import static org.apache.sentry.hdfs.ServiceConstants.SEQUENCE_NUMBER_FULL_UPDATE_REQUEST; + import java.util.Collections; import java.util.List; - +import javax.annotation.concurrent.ThreadSafe; +import org.apache.sentry.service.thrift.SentryServiceState; +import org.apache.sentry.service.thrift.SentryStateBank; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.concurrent.ThreadSafe; - -import static org.apache.sentry.hdfs.ServiceConstants.IMAGE_NUMBER_UPDATE_UNINITIALIZED; -import static org.apache.sentry.hdfs.ServiceConstants.SEQUENCE_NUMBER_UPDATE_UNINITIALIZED; - /** * DBUpdateForwarder propagates a complete snapshot or delta update of either * Sentry Permissions ({@code PermissionsUpdate}) or Sentry representation of @@ -81,10 +81,12 @@ class DBUpdateForwarder<K extends Updateable.Update> { if (curImgNum == IMAGE_NUMBER_UPDATE_UNINITIALIZED) { // Sentry has not fetched a full HMS snapshot yet. return Collections.emptyList(); - } else if (curImgNum > imgNum) { + } + + if (curImgNum > imgNum) { // In case a new HMS snapshot has been processed, then return a full paths image. LOGGER.info("A newer full update is found with image number: {}", curImgNum); - return Collections.singletonList(imageRetriever.retrieveFullImage()); + return retrieveFullImage(); } } @@ -102,7 +104,7 @@ class DBUpdateForwarder<K extends Updateable.Update> { // Checks if newer deltas exist in the persistent storage. // If there are, return the list of delta updates. - if (seqNum > SEQUENCE_NUMBER_UPDATE_UNINITIALIZED && deltaRetriever.isDeltaAvailable(seqNum)) { + if (seqNum > SEQUENCE_NUMBER_FULL_UPDATE_REQUEST && deltaRetriever.isDeltaAvailable(seqNum)) { List<K> deltas = deltaRetriever.retrieveDelta(seqNum, imgNum); if (!deltas.isEmpty()) { LOGGER.info("Newer delta updates are found up to sequence number: {}", curSeqNum); @@ -113,6 +115,16 @@ class DBUpdateForwarder<K extends Updateable.Update> { // If the sequence number is < 0 or the requested delta is not available, then we // return a full update. LOGGER.info("A full update is returned due to an unavailable sequence number: {}", seqNum); - return Collections.singletonList(imageRetriever.retrieveFullImage()); + return retrieveFullImage(); + } + + private List<K> retrieveFullImage() throws Exception { + if (SentryStateBank.isEnabled(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING)){ + LOGGER.debug("A full update is being loaded. Delaying updating client with full image until its finished."); + return Collections.emptyList(); + } + else { + return Collections.singletonList(imageRetriever.retrieveFullImage()); + } } } http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java index 830d00e..cd1ad03 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java @@ -18,6 +18,9 @@ package org.apache.sentry.hdfs; import org.apache.sentry.provider.db.service.persistent.SentryStore; +import org.apache.sentry.service.thrift.SentryServiceState; +import org.apache.sentry.service.thrift.SentryStateBank; +import org.apache.sentry.service.thrift.SentryStateBankTestHelper; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -25,6 +28,7 @@ import org.mockito.Mockito; import java.util.Arrays; import java.util.List; +import static org.apache.sentry.hdfs.ServiceConstants.SEQUENCE_NUMBER_UPDATE_UNINITIALIZED; import static org.apache.sentry.hdfs.service.thrift.sentry_hdfs_serviceConstants.UNUSED_PATH_UPDATE_IMG_NUM; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -40,6 +44,7 @@ public class TestDBUpdateForwarder { imageRetriever = Mockito.mock(ImageRetriever.class); deltaRetriever = Mockito.mock(DeltaRetriever.class); updater = new DBUpdateForwarder<>(imageRetriever, deltaRetriever); + SentryStateBankTestHelper.clearAllStates(); } @Test @@ -50,6 +55,34 @@ public class TestDBUpdateForwarder { assertTrue(updates.isEmpty()); } + /** + * Test if a empty list is returned when the Sentry Server is doing a Full update from + * HMS. + * @throws Exception + */ + @Test + public void testEmptyListIsReturnedWhenFullUpdateRunningFromStart() throws Exception { + SentryStateBank.enableState(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING); + Mockito.when(imageRetriever.getLatestImageID()).thenReturn(1L); + + List updates = updater.getAllUpdatesFrom(SEQUENCE_NUMBER_UPDATE_UNINITIALIZED,1); + assertTrue(updates.isEmpty()); + } + + /** + * Test if a empty list is returned when the Sentry Server is doing a Full update from + * HMS. + * @throws Exception + */ + @Test + public void testEmptyListIsReturnedWhenFullUpdateRunning() throws Exception { + SentryStateBank.enableState(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING); + Mockito.when(imageRetriever.getLatestImageID()).thenReturn(2L); + + List updates = updater.getAllUpdatesFrom(SEQUENCE_NUMBER_UPDATE_UNINITIALIZED,1); + assertTrue(updates.isEmpty()); + } + @Test public void testEmptyListIsReturnedWhenImageIsUnusedAndNoDeltaChangesArePersisted() throws Exception { Mockito.when(deltaRetriever.getLatestDeltaID()).thenReturn(SentryStore.EMPTY_NOTIFICATION_ID); @@ -72,6 +105,18 @@ public class TestDBUpdateForwarder { } @Test + public void testFirstImageSyncGetsEmptySetWhenImageNumIsZeroAndFullUpdateRunning() throws Exception { + Mockito.when(imageRetriever.getLatestImageID()).thenReturn(1L); + Mockito.when(imageRetriever.retrieveFullImage()) + .thenReturn(new PathsUpdate(1, 1, true)); + SentryStateBank.enableState(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING); + + List<PathsUpdate> updates = updater.getAllUpdatesFrom(0, SentryStore.EMPTY_PATHS_SNAPSHOT_ID); + assertTrue(updates.isEmpty()); + } + + + @Test public void testFirstImageSyncIsReturnedWhenImageNumIsUnusedButDeltasAreAvailable() throws Exception { Mockito.when(deltaRetriever.getLatestDeltaID()).thenReturn(1L); Mockito.when(imageRetriever.retrieveFullImage()) http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index 4d83ad5..b600487 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -19,14 +19,16 @@ package org.apache.sentry.service.thrift; +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME; +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED; + import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import java.util.Collection; import java.util.List; import javax.jdo.JDODataStoreException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME; -import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED; import org.apache.sentry.provider.db.service.persistent.PathsImage; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.thrift.TException; @@ -107,6 +109,7 @@ public class HMSFollower implements Runnable, AutoCloseable { // Close any outstanding connections to HMS try { client.disconnect(); + SentryStateBank.disableState(HMSFollowerState.COMPONENT,HMSFollowerState.CONNECTED); } catch (Exception failure) { LOGGER.error("Failed to close the Sentry Hms Client", failure); } @@ -117,24 +120,29 @@ public class HMSFollower implements Runnable, AutoCloseable { @Override public void run() { + SentryStateBank.enableState(HMSFollowerState.COMPONENT,HMSFollowerState.STARTED); long lastProcessedNotificationId; try { - // Initializing lastProcessedNotificationId based on the latest persisted notification ID. - lastProcessedNotificationId = sentryStore.getLastProcessedNotificationID(); - } catch (Exception e) { - LOGGER.error("Failed to get the last processed notification id from sentry store, " - + "Skipping the processing", e); - return; - } - // Wake any clients connected to this service waiting for HMS already processed notifications. - wakeUpWaitingClientsForSync(lastProcessedNotificationId); - // Only the leader should listen to HMS updates - if (!isLeader()) { - // Close any outstanding connections to HMS - close(); - return; + try { + // Initializing lastProcessedNotificationId based on the latest persisted notification ID. + lastProcessedNotificationId = sentryStore.getLastProcessedNotificationID(); + } catch (Exception e) { + LOGGER.error("Failed to get the last processed notification id from sentry store, " + + "Skipping the processing", e); + return; + } + // Wake any clients connected to this service waiting for HMS already processed notifications. + wakeUpWaitingClientsForSync(lastProcessedNotificationId); + // Only the leader should listen to HMS updates + if (!isLeader()) { + // Close any outstanding connections to HMS + close(); + return; + } + syncupWithHms(lastProcessedNotificationId); + } finally { + SentryStateBank.disableState(HMSFollowerState.COMPONENT,HMSFollowerState.STARTED); } - syncupWithHms(lastProcessedNotificationId); } private boolean isLeader() { @@ -160,6 +168,7 @@ public class HMSFollower implements Runnable, AutoCloseable { try { client.connect(); connectedToHms = true; + SentryStateBank.enableState(HMSFollowerState.COMPONENT,HMSFollowerState.CONNECTED); } catch (Throwable e) { LOGGER.error("HMSFollower cannot connect to HMS!!", e); return; @@ -269,46 +278,57 @@ public class HMSFollower implements Runnable, AutoCloseable { } /** - * Request for full snapshot and persists it if there is no snapshot available in the - * sentry store. Also, wakes-up any waiting clients. + * Request for full snapshot and persists it if there is no snapshot available in the sentry + * store. Also, wakes-up any waiting clients. * * @return ID of last notification processed. * @throws Exception if there are failures */ private long createFullSnapshot() throws Exception { LOGGER.debug("Attempting to take full HMS snapshot"); - PathsImage snapshotInfo = client.getFullSnapshot(); - if (snapshotInfo.getPathImage().isEmpty()) { - return snapshotInfo.getId(); - } + Preconditions.checkState(!SentryStateBank.isEnabled(SentryServiceState.COMPONENT, + SentryServiceState.FULL_UPDATE_RUNNING), + "HMSFollower shown loading full snapshot when it should not be."); + try { + // Set that the full update is running + SentryStateBank + .enableState(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING); - // Check we're still the leader before persisting the new snapshot - if (!isLeader()) { - return SentryStore.EMPTY_NOTIFICATION_ID; - } + PathsImage snapshotInfo = client.getFullSnapshot(); + if (snapshotInfo.getPathImage().isEmpty()) { + return snapshotInfo.getId(); + } - try { - LOGGER.debug("Persisting HMS path full snapshot"); + // Check we're still the leader before persisting the new snapshot + if (!isLeader()) { + return SentryStore.EMPTY_NOTIFICATION_ID; + } + try { + LOGGER.debug("Persisting HMS path full snapshot"); - if (hdfsSyncEnabled) { - sentryStore.persistFullPathsImage(snapshotInfo.getPathImage(), snapshotInfo.getId()); - } else { - // We need to persist latest notificationID for next poll - sentryStore.setLastProcessedNotificationID(snapshotInfo.getId()); + if (hdfsSyncEnabled) { + sentryStore.persistFullPathsImage(snapshotInfo.getPathImage(), snapshotInfo.getId()); + } else { + // We need to persist latest notificationID for next poll + sentryStore.setLastProcessedNotificationID(snapshotInfo.getId()); + } + // Only reset the counter if the above operations succeeded + resetCounterWait(snapshotInfo.getId()); + } catch (Exception failure) { + LOGGER.error("Received exception while persisting HMS path full snapshot "); + throw failure; } - // Only reset the counter if the above operations succeeded - resetCounterWait(snapshotInfo.getId()); - } catch (Exception failure) { - LOGGER.error("Received exception while persisting HMS path full snapshot "); - throw failure; + // Wake up any HMS waiters that could have been put on hold before getting the + // eventIDBefore value. + wakeUpWaitingClientsForSync(snapshotInfo.getId()); + // HMSFollower connected to HMS and it finished full snapshot if that was required + // Log this message only once + LOGGER.info("Sentry HMS support is ready"); + return snapshotInfo.getId(); + } finally { + SentryStateBank + .disableState(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING); } - // Wake up any HMS waiters that could have been put on hold before getting the - // eventIDBefore value. - wakeUpWaitingClientsForSync(snapshotInfo.getId()); - // HMSFollower connected to HMS and it finished full snapshot if that was required - // Log this message only once - LOGGER.info("Sentry HMS support is ready"); - return snapshotInfo.getId(); } /** http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollowerState.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollowerState.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollowerState.java new file mode 100644 index 0000000..74268f7 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollowerState.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sentry.service.thrift; + +/** + * States for the HMSFollower + */ +public enum HMSFollowerState implements SentryState { + /** + * If the HMSFollower has been started or not. + */ + STARTED, + + /** + * If the HMSFollower is connected to the HMS + */ + CONNECTED; + + /** + * The component name this state is for. + */ + public static final String COMPONENT = "HMSFollower"; + + /** + * {@inheritDoc} + */ + @Override + public long getValue() { + return 1 << this.ordinal(); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java index d44abff..d2a4c2d 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java @@ -270,6 +270,7 @@ public class SentryService implements Callable, SigUtils.SigListener { // thriftServer.serve() does not return until thriftServer is stopped. Need to log before // calling thriftServer.serve() LOGGER.info("Sentry service is ready to serve client requests"); + SentryStateBank.enableState(SentryServiceState.COMPONENT, SentryServiceState.SERVICE_RUNNING); thriftServer.serve(); } @@ -481,6 +482,7 @@ public class SentryService implements Callable, SigUtils.SigListener { if (exception != null) { exception.ifExceptionThrow(); } + SentryStateBank.disableState(SentryServiceState.COMPONENT,SentryServiceState.SERVICE_RUNNING); LOGGER.info("Stopped..."); } http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java new file mode 100644 index 0000000..4219adc --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sentry.service.thrift; + +/** + * States for the SentryService + */ +public enum SentryServiceState implements SentryState { + /** + * The SentryService is running all of its threads and services. This include the store cleaner, + * the web interface, the HMS poller, and the Thrift Server + */ + SERVICE_RUNNING, + + /** + * A full update of data from the HMS is running by the thread handling the update. + */ + FULL_UPDATE_RUNNING; + + /** + * The component name this state is for. + */ + public static final String COMPONENT = "SentryService"; + + /** + * {@inheritDoc} + */ + @Override + public long getValue() { + return 1 << this.ordinal(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java new file mode 100644 index 0000000..040d82a --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.sentry.service.thrift; + +/** + * Interface for SentryState enums. + */ +public interface SentryState { + + /** + * This gets the Bitmask value associated with the state. + */ + long getValue(); +} http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java new file mode 100644 index 0000000..2c05d49 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sentry.service.thrift; + + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.AtomicLongMap; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.annotation.concurrent.ThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * <p>SentryStateBank is a state visibility manager to allow components to communicate state to other + * parts of the application.</p> + * + * <p>It allows you to provide multiple boolean states for a component and expose those states to + * other parts of the application without having references to the actual instances of the classes + * setting those states.</p> + * + * <p>SentryStateBank uses a bitmasked long in order to store the states, so its very compact and + * efficient.</p> + * + * <p>States are defined using an enum that implements the {@link SentryState} interface. The + * {@link SentryState} implementation can provide up to 64 states per components. The {@link SentryState#getValue()} + * implementation should return a bitshift of the oridinal of the enum value. This gives the bitmask + * location to be checking for the state.</p> + * + * <p>The following is an example of a simple {@link SentryState} enum implementation</p> + * + * <pre> + * {@code + * + * public enum ExampleState implements SentryState { + * FIRST_STATE, + * SECOND_STATE; + * + * public static final String COMPONENT = "ExampleState"; + * + * @Override + * public long getValue() { + * return 1 << this.ordinal(); + * } + * } + * } + * </pre> + * + * <p>This class is thread safe. It uses a {@link ReentrantReadWriteLock} to wrap accesses and changes + * to the state.</p> + */ +@ThreadSafe +public final class SentryStateBank { + + private static final Logger LOGGER = LoggerFactory.getLogger(SentryStateBank.class); + private static final AtomicLongMap<String> states = AtomicLongMap.create(); + private static final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + protected SentryStateBank() { + } + + @VisibleForTesting + static void clearAllStates() { + states.clear(); + LOGGER.debug("All states have been cleared."); + } + + @VisibleForTesting + static void resetComponentState(String component) { + states.remove(component); + LOGGER.debug("All states have been cleared for component {}", component); + } + + /** + * Enables a state + * + * @param component the component for the state + * @param state the state to disable + */ + public static void enableState(String component, SentryState state) { + lock.writeLock().lock(); + try { + states.put(component, states.get(component) | state.getValue()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("{} entered state {}", component, state.toString()); + } + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Disables a state for a component + * + * @param component the component for the state + * @param state the state to disable + */ + public static void disableState(String component, SentryState state) { + lock.writeLock().lock(); + try { + states.put(component, states.get(component) & (~state.getValue())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("{} exited state {}", component, state.toString()); + } + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Returns if a state is enabled or not + * + * @param component The component for the state + * @param state the SentryState to check + * @return true if the state for the component is enabled + */ + public static boolean isEnabled(String component, SentryState state) { + lock.readLock().lock(); + try { + return (states.get(component) & state.getValue()) == state.getValue(); + } finally { + lock.readLock().unlock(); + } + + } + + /** + * Checks if all of the states passed in are enabled + * + * @param component The component for the states + * @param passedStates the SentryStates to check + */ + public static boolean hasStatesEnabled(String component, Set<SentryState> passedStates) { + lock.readLock().lock(); + try { + long value = 0L; + + for (SentryState state : passedStates) { + value += state.getValue(); + } + return (states.get(component) & value) == value; + } finally { + lock.readLock().unlock(); + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/SentryStateBankTestHelper.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/SentryStateBankTestHelper.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/SentryStateBankTestHelper.java new file mode 100644 index 0000000..48212a0 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/SentryStateBankTestHelper.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sentry.service.thrift; + +/** + * + */ +public class SentryStateBankTestHelper { + + public static void clearAllStates() { + SentryStateBank.clearAllStates(); + } + + public static void resetComponentState(String component) { + SentryStateBank.resetComponentState(component); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/a7a2d69c/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryStateBank.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryStateBank.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryStateBank.java new file mode 100644 index 0000000..4f71e1c --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryStateBank.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sentry.service.thrift; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.HashSet; +import org.junit.Before; +import org.junit.Test; + +/** + * + */ +public class TestSentryStateBank { + + @Before + public void setUp() { + SentryStateBank.clearAllStates(); + } + + @Test + public void testEnableState() { + SentryStateBank.enableState(TestState.COMPONENT, TestState.FIRST_STATE); + assertTrue("Expected FIRST_STATE to be enabled", + SentryStateBank.isEnabled(TestState.COMPONENT, TestState.FIRST_STATE)); + assertFalse("Expected SECOND_STATE to be disabled", + SentryStateBank.isEnabled(TestState.COMPONENT, TestState.SECOND_STATE)); + } + + @Test + public void testStatesGetDisabled() { + SentryStateBank.enableState(TestState.COMPONENT, TestState.FIRST_STATE); + assertTrue("Expected FIRST_STATE to be enabled", + SentryStateBank.isEnabled(TestState.COMPONENT, TestState.FIRST_STATE)); + SentryStateBank.disableState(TestState.COMPONENT, TestState.FIRST_STATE); + assertFalse("Expected FIRST_STATE to be disabled", + SentryStateBank.isEnabled(TestState.COMPONENT, TestState.FIRST_STATE)); + } + + @Test + public void testCheckMultipleStateCheckSuccess() { + SentryStateBank.enableState(TestState.COMPONENT, TestState.FIRST_STATE); + SentryStateBank.enableState(TestState.COMPONENT, TestState.SECOND_STATE); + + assertTrue("Expected both FIRST_STATE and SECOND_STATE to be enabled", + SentryStateBank.hasStatesEnabled(TestState.COMPONENT, new HashSet<SentryState>( + Arrays.asList(TestState.FIRST_STATE, TestState.SECOND_STATE)))); + } + + @Test + public void testCheckMultipleStateCheckFailure() { + SentryStateBank.enableState(TestState.COMPONENT, TestState.FIRST_STATE); + assertFalse("Expected only FIRST_STATE to be enabled", + SentryStateBank.hasStatesEnabled(TestState.COMPONENT, new HashSet<SentryState>( + Arrays.asList(TestState.FIRST_STATE, TestState.SECOND_STATE)))); + } + + + public enum TestState implements SentryState { + FIRST_STATE, + SECOND_STATE; + + public static final String COMPONENT = "TestState"; + + @Override + public long getValue() { + return 1 << this.ordinal(); + } + } +}
