Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 037059774 -> 2fcd69aa2
SENTRY-1696: Expose time spent creating the initial snapshot as a metric (Alex Kolbasov, reviewed by Vamsee Yarlagadda) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/2fcd69aa Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/2fcd69aa Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/2fcd69aa Branch: refs/heads/sentry-ha-redesign Commit: 2fcd69aa2ed6b0f80c5c0e91a6a84ce24459a336 Parents: 0370597 Author: Alexander Kolbasov <[email protected]> Authored: Wed Jul 12 19:45:52 2017 +0200 Committer: Alexander Kolbasov <[email protected]> Committed: Wed Jul 12 19:45:52 2017 +0200 ---------------------------------------------------------------------- .../service/thrift/FullUpdateInitializer.java | 24 +++++++++++++++++++- .../sentry/service/thrift/HMSFollower.java | 17 +++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/2fcd69aa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java index 1490581..b74fa50 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java @@ -17,6 +17,7 @@ */ package org.apache.sentry.service.thrift; +import com.codahale.metrics.Counter; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.Database; @@ -25,6 +26,7 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.hdfs.SentryMalformedPathException; import org.apache.sentry.hdfs.ServiceConstants.ServerConfig; +import org.apache.sentry.provider.db.service.thrift.SentryMetrics; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,6 +48,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import static com.codahale.metrics.MetricRegistry.name; + /** * Manage fetching full snapshot from HMS. * Snapshot is represented as a map from the hive object name to @@ -103,6 +107,18 @@ public final class FullUpdateInitializer implements AutoCloseable { new ObjectMapping(Collections.<String, Set<String>>emptyMap()); private final HiveConnectionFactory clientFactory; + /** Total number of database objects */ + private final Counter databaseCount = SentryMetrics.getInstance() + .getCounter(name(FullUpdateInitializer.class, "total", "db")); + + /** Total number of table objects */ + private final Counter tableCount = SentryMetrics.getInstance() + .getCounter(name(FullUpdateInitializer.class, "total", "tables")); + + /** Total number of partition objects */ + private final Counter partitionCount = SentryMetrics.getInstance() + .getCounter(name(FullUpdateInitializer.class, "total", "partitions")); + /** * Extract path (not starting with "/") from the full URI * @param uri - resource URI (usually with scheme) @@ -323,7 +339,10 @@ public final class FullUpdateInitializer implements AutoCloseable { String tableName = safeIntern(tbl.getTableName().toLowerCase()); String authzObject = (dbName + "." + tableName).intern(); - List<String> tblPartNames = client.getClient().listPartitionNames(dbName, tableName, (short) -1); + List<String> tblPartNames = + client.getClient().listPartitionNames(dbName, tableName, (short) -1); + // Count total number of partitions + partitionCount.inc(tblPartNames.size()); for (int i = 0; i < tblPartNames.size(); i += maxPartitionsPerCall) { List<String> partsToFetch = tblPartNames.subList(i, Math.min(i + maxPartitionsPerCall, tblPartNames.size())); @@ -359,6 +378,7 @@ public final class FullUpdateInitializer implements AutoCloseable { DbTask(String dbName) { //Database names are case insensitive this.dbName = safeIntern(dbName.toLowerCase()); + databaseCount.inc(); } @Override @@ -372,6 +392,8 @@ public final class FullUpdateInitializer implements AutoCloseable { return emptyObjectMapping; } List<String> allTblStr = client.getClient().getAllTables(dbName); + // Count total number of tables + tableCount.inc(allTblStr.size()); for (int i = 0; i < allTblStr.size(); i += maxTablesPerCall) { List<String> tablesToFetch = allTblStr.subList(i, Math.min(i + maxTablesPerCall, allTblStr.size())); http://git-wip-us.apache.org/repos/asf/sentry/blob/2fcd69aa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index 2d581f7..1b6ae18 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -17,6 +17,9 @@ */ package org.apache.sentry.service.thrift; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Timer; +import com.codahale.metrics.Timer.Context; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -35,6 +38,7 @@ import org.apache.sentry.hdfs.PermissionsUpdate; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; import org.apache.sentry.provider.db.service.persistent.SentryStore; +import org.apache.sentry.provider.db.service.thrift.SentryMetrics; import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -52,6 +56,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; +import static com.codahale.metrics.MetricRegistry.name; import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE; import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE; import static org.apache.sentry.hdfs.Updateable.Update; @@ -77,6 +82,14 @@ public class HMSFollower implements Runnable, AutoCloseable { private boolean needLogHMSSupportReady = true; private final LeaderStatusMonitor leaderMonitor; + private static final String SNAPSHOT = "snapshot"; + /** Measures time to get full snapshot */ + private final Timer updateTimer = SentryMetrics.getInstance() + .getTimer(name(FullUpdateInitializer.class, SNAPSHOT)); + /** Number of times update failed */ + private final Counter failedSnapshotsCount = SentryMetrics.getInstance() + .getCounter(name(FullUpdateInitializer.class, "failed")); + HMSFollower(Configuration conf, SentryStore store, LeaderStatusMonitor leaderMonitor, HiveSimpleConnectionFactory hiveConnectionFactory) { authzConf = conf; @@ -299,11 +312,13 @@ public class HMSFollower implements Runnable, AutoCloseable { throws TException, ExecutionException { LOGGER.info("Request full HMS snapshot"); try (FullUpdateInitializer updateInitializer = - new FullUpdateInitializer(hiveConnectionFactory, authzConf)) { + new FullUpdateInitializer(hiveConnectionFactory, authzConf); + Context context = updateTimer.time()) { Map<String, Set<String>> pathsUpdate = updateInitializer.getFullHMSSnapshot(); LOGGER.info("Obtained full HMS snapshot"); return pathsUpdate; } catch (Exception ignored) { + failedSnapshotsCount.inc(); // Caller will retry later return Collections.emptyMap(); }
