Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 15613a1cc -> 9739dd80a
SENTRY-1677: Add metrics to measure how much time to get Delta Path/Perm Updates (Alex Kolabsov, reviewed by Na Li) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/9739dd80 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/9739dd80 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/9739dd80 Branch: refs/heads/sentry-ha-redesign Commit: 9739dd80a1635dc07f71f05857b5d3278872d1ca Parents: 15613a1 Author: Alexander Kolbasov <[email protected]> Authored: Tue Apr 4 15:40:46 2017 -0700 Committer: Alexander Kolbasov <[email protected]> Committed: Tue Apr 4 15:40:46 2017 -0700 ---------------------------------------------------------------------- .../apache/sentry/hdfs/PathDeltaRetriever.java | 45 +++++++++++--------- .../apache/sentry/hdfs/PermDeltaRetriever.java | 41 ++++++++++-------- .../sentry/hdfs/SentryHdfsMetricsUtil.java | 37 ++++++++++++---- 3 files changed, 78 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java index cea5b9d..5425114 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java @@ -17,6 +17,7 @@ */ package org.apache.sentry.hdfs; +import com.codahale.metrics.Timer; import org.apache.sentry.provider.db.service.model.MSentryPathChange; import org.apache.sentry.provider.db.service.persistent.SentryStore; @@ -43,34 +44,40 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate> { @Override public Collection<PathsUpdate> retrieveDelta(long seqNum) throws Exception { - Collection<MSentryPathChange> mSentryPathChanges = - sentryStore.getMSentryPathChanges(seqNum); - if (mSentryPathChanges.isEmpty()) { - return Collections.emptyList(); - } + try (final Timer.Context timerContext = + SentryHdfsMetricsUtil.getDeltaPathChangesTimer.time()) { + Collection<MSentryPathChange> mSentryPathChanges = + sentryStore.getMSentryPathChanges(seqNum); + + SentryHdfsMetricsUtil.getDeltaPathChangesHistogram.update(mSentryPathChanges.size()); + + if (mSentryPathChanges.isEmpty()) { + return Collections.emptyList(); + } - Collection<PathsUpdate> updates = new ArrayList<>(mSentryPathChanges.size()); - for (MSentryPathChange mSentryPathChange : mSentryPathChanges) { - // Gets the changeID from the persisted MSentryPathChange. - long changeID = mSentryPathChange.getChangeID(); - // Creates a corresponding PathsUpdate and deserialize the - // persisted delta update in JSON format to TPathsUpdate with - // associated changeID. - PathsUpdate pathsUpdate = new PathsUpdate(); - pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange()); - pathsUpdate.setSeqNum(changeID); - updates.add(pathsUpdate); + Collection<PathsUpdate> updates = new ArrayList<>(mSentryPathChanges.size()); + for (MSentryPathChange mSentryPathChange : mSentryPathChanges) { + // Gets the changeID from the persisted MSentryPathChange. + long changeID = mSentryPathChange.getChangeID(); + // Creates a corresponding PathsUpdate and deserialize the + // persisted delta update in JSON format to TPathsUpdate with + // associated changeID. + PathsUpdate pathsUpdate = new PathsUpdate(); + pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange()); + pathsUpdate.setSeqNum(changeID); + updates.add(pathsUpdate); + } + return updates; } - return updates; } @Override - public boolean isDeltaAvailable(long seqNum) throws Exception { + public boolean isDeltaAvailable ( long seqNum) throws Exception { return sentryStore.pathChangeExists(seqNum); } @Override - public long getLatestDeltaID() throws Exception { + public long getLatestDeltaID () throws Exception { return sentryStore.getLastProcessedPathChangeID(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java index 9649b02..a29fc74 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java @@ -17,6 +17,7 @@ */ package org.apache.sentry.hdfs; +import com.codahale.metrics.Timer; import org.apache.sentry.provider.db.service.model.MSentryPermChange; import org.apache.sentry.provider.db.service.persistent.SentryStore; @@ -43,25 +44,31 @@ public class PermDeltaRetriever implements DeltaRetriever<PermissionsUpdate> { @Override public Collection<PermissionsUpdate> retrieveDelta(long seqNum) throws Exception { - Collection<MSentryPermChange> mSentryPermChanges = - sentryStore.getMSentryPermChanges(seqNum); - if (mSentryPermChanges.isEmpty()) { - return Collections.emptyList(); - } + try (final Timer.Context timerContext = + SentryHdfsMetricsUtil.getDeltaPermChangesTimer.time()) { + Collection<MSentryPermChange> mSentryPermChanges = + sentryStore.getMSentryPermChanges(seqNum); + + SentryHdfsMetricsUtil.getDeltaPermChangesHistogram.update(mSentryPermChanges.size()); + + if (mSentryPermChanges.isEmpty()) { + return Collections.emptyList(); + } - Collection<PermissionsUpdate> updates = new ArrayList<>(mSentryPermChanges.size()); - for (MSentryPermChange mSentryPermChange : mSentryPermChanges) { - // Get the changeID from the persisted MSentryPermChange - long changeID = mSentryPermChange.getChangeID(); - // Create a corresponding PermissionsUpdate and deserialize the - // persisted delta update in JSON format to TPermissionsUpdate with - // associated changeID. - PermissionsUpdate permsUpdate = new PermissionsUpdate(); - permsUpdate.JSONDeserialize(mSentryPermChange.getPermChange()); - permsUpdate.setSeqNum(changeID); - updates.add(permsUpdate); + Collection<PermissionsUpdate> updates = new ArrayList<>(mSentryPermChanges.size()); + for (MSentryPermChange mSentryPermChange : mSentryPermChanges) { + // Get the changeID from the persisted MSentryPermChange + long changeID = mSentryPermChange.getChangeID(); + // Create a corresponding PermissionsUpdate and deserialize the + // persisted delta update in JSON format to TPermissionsUpdate with + // associated changeID. + PermissionsUpdate permsUpdate = new PermissionsUpdate(); + permsUpdate.JSONDeserialize(mSentryPermChange.getPermChange()); + permsUpdate.setSeqNum(changeID); + updates.add(permsUpdate); + } + return updates; } - return updates; } @Override http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java index 28bf20e..932a5c0 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java @@ -27,20 +27,20 @@ import org.apache.sentry.provider.db.service.thrift.SentryMetrics; /** * Util class to support metrics. */ -public class SentryHdfsMetricsUtil { +final class SentryHdfsMetricsUtil { // SentryMetrics private static final SentryMetrics sentryMetrics = SentryMetrics.getInstance(); // Metrics for get_all_authz_updates_from in SentryHDFSServiceProcessor // The time used for each get_all_authz_updates_from - public static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer( + static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer( MetricRegistry.name(SentryHDFSServiceProcessor.class, "get-all-authz-updates-from")); // The size of perm updates for each get_all_authz_updates_from - public static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram( + static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram( MetricRegistry.name(SentryHDFSServiceProcessor.class, "perm-updates-size")); // The size of path updates for each get_all_authz_updates_from - public static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram( + static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram( MetricRegistry.name(SentryHDFSServiceProcessor.class, "paths-updates-size")); // Metrics for handle_hms_notification in SentryHDFSServiceProcessor @@ -62,27 +62,46 @@ public class SentryHdfsMetricsUtil { // Metrics for retrievePermFullImage in PermImageRetriever // The time used for each retrievePermFullImage - public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer( + static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer( MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image")); // The size of privilege changes for each retrievePermFullImage - public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( + static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "privilege-changes-size")); // The size of role changes for each retrievePermFullImage call - public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( + static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "role-changes-size")); // Metrics for retrievePathFullImage in PathImageRetriever // The time used for each retrievePathFullImage - public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer( + static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer( MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image")); // The size of path changes for each retrievePathFullImage - public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram( + static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram( MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image", "path-changes-size")); + // Timer for getting path changes deltas + static final Timer getDeltaPathChangesTimer = sentryMetrics.getTimer( + MetricRegistry.name(PathDeltaRetriever.class, "path", "delta", "time") + ); + + // Histogram for the number of path changes processed for deltas + static final Histogram getDeltaPathChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(PathDeltaRetriever.class, "path", "delta", "size")); + + + // Timer for getting permission changes deltas + static final Timer getDeltaPermChangesTimer = sentryMetrics.getTimer( + MetricRegistry.name(PathDeltaRetriever.class, "perm", "delta", "time") + ); + + // Histogram for the number of permissions changes processed for deltas + static final Histogram getDeltaPermChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(PathDeltaRetriever.class, "perm", "delta", "size")); + private SentryHdfsMetricsUtil() { // Make constructor private to avoid instantiation }
