Repository: sentry Updated Branches: refs/heads/SENTRY-1820 [created] 010446fc7
SENTRY-1820 Add JSON file reporter for Sentry metrics Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/010446fc Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/010446fc Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/010446fc Branch: refs/heads/SENTRY-1820 Commit: 010446fc7d71a7e0f5ee65379a2565a9a355d2e2 Parents: 8d53da5 Author: Alexander Kolbasov <[email protected]> Authored: Tue Jun 27 17:53:00 2017 -0700 Committer: Alexander Kolbasov <[email protected]> Committed: Tue Jun 27 17:53:00 2017 -0700 ---------------------------------------------------------------------- .../sentry/service/thrift/ServiceConstants.java | 5 +- .../db/service/thrift/SentryMetrics.java | 106 +++++++++++++++++-- 2 files changed, 101 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/010446fc/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java index de97a31..affb8a4 100644 --- a/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java +++ b/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java @@ -166,11 +166,14 @@ public class ServiceConstants { public static final Boolean SENTRY_WEB_ENABLE_DEFAULT = false; public static final String SENTRY_WEB_PORT = "sentry.service.web.port"; public static final int SENTRY_WEB_PORT_DEFAULT = 29000; - // Reporter is either "console" or "jmx" + // Reporter is either "console", "log" or "jmx" public static final String SENTRY_REPORTER = "sentry.service.reporter"; // for console reporter, reporting interval in seconds public static final String SENTRY_REPORTER_INTERVAL_SEC = "sentry.service.reporter.interval.sec"; + public static final String SENTRY_JSON_REPORTER_FILE = "sentry.service.reporter.file"; + public static final String SENTRY_JSON_REPORTER_FILE_DEFAULT = "/tmp/sentry-metrics.json"; + // Report every 5 minutes by default public static final int SENTRY_REPORTER_INTERVAL_DEFAULT = 300; http://git-wip-us.apache.org/repos/asf/sentry/blob/010446fc/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java index af70c99..1a5fd24 100644 --- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java +++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java @@ -21,20 +21,35 @@ import com.codahale.metrics.*; import static com.codahale.metrics.MetricRegistry.name; +import com.codahale.metrics.json.MetricsModule; import com.codahale.metrics.jvm.BufferPoolMetricSet; import com.codahale.metrics.jvm.GarbageCollectorMetricSet; import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.codahale.metrics.jvm.ThreadStatesGaugeSet; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.io.File.createTempFile; import static org.apache.sentry.provider.db.service.thrift.SentryMetricsServletContextListener.METRIC_REGISTRY; import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -42,6 +57,13 @@ import java.util.concurrent.atomic.AtomicBoolean; * A singleton class which holds metrics related utility functions as well as the list of metrics */ public final class SentryMetrics { + public enum Reporting { + JMX, + CONSOLE, + LOG, + JSON, + } + private static final Logger LOGGER = LoggerFactory .getLogger(SentryMetrics.class); @@ -152,8 +174,8 @@ public final class SentryMetrics { switch(SentryMetrics.Reporting.valueOf(reporter.toUpperCase())) { case CONSOLE: - LOGGER.info(String.format("Enabled console metrics reporter with %d seconds interval", - reportInterval)); + LOGGER.info("Enabled console metrics reporter with {} seconds interval", + reportInterval); final ConsoleReporter consoleReporter = ConsoleReporter.forRegistry(METRIC_REGISTRY) .convertRatesTo(TimeUnit.SECONDS) @@ -170,8 +192,8 @@ public final class SentryMetrics { jmxReporter.start(); break; case LOG: - LOGGER.info(String.format("Enabled Log4J metrics reporter with %d seconds interval", - reportInterval)); + LOGGER.info("Enabled Log4J metrics reporter with {} seconds interval", + reportInterval); final Slf4jReporter logReporter = Slf4jReporter.forRegistry(METRIC_REGISTRY) .outputTo(LOGGER) .convertRatesTo(TimeUnit.SECONDS) @@ -179,8 +201,13 @@ public final class SentryMetrics { .build(); logReporter.start(reportInterval, TimeUnit.SECONDS); break; + case JSON: + LOGGER.info("Enabled JSON metrics reporter with {} seconds interval", reportInterval); + final JsonFileReporter jsonReporter = new JsonFileReporter(conf, reportInterval, TimeUnit.SECONDS); + jsonReporter.start(); + break; default: - LOGGER.warn("Invalid metrics reporter " + reporter); + LOGGER.warn("Invalid metrics reporter {}", reporter); break; } } @@ -200,9 +227,70 @@ public final class SentryMetrics { } } - public enum Reporting { - JMX, - CONSOLE, - LOG, + static class JsonFileReporter implements AutoCloseable, Runnable { + private static final short PERMISSIONS = 0644; + + private ScheduledExecutorService executor = null; + private final ObjectMapper jsonMapper = + new ObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS, + TimeUnit.MILLISECONDS, + false)); + private final Configuration conf; + private final String pathString; + private final long interval; + private final TimeUnit unit; + + JsonFileReporter(Configuration conf, long interval, TimeUnit unit) { + this.conf = conf; + pathString = conf.get(ServerConfig.SENTRY_JSON_REPORTER_FILE, + ServerConfig.SENTRY_JSON_REPORTER_FILE_DEFAULT); + this.interval = interval; + this.unit = unit; + } + + private void start() { + executor = Executors.newScheduledThreadPool(1, + new ThreadFactoryBuilder().setNameFormat("json-reporter").build()); + executor.scheduleAtFixedRate(this, 0, interval, unit); + } + + @Override + public void run() { + String json = null; + try { + json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(METRIC_REGISTRY); + } catch (JsonProcessingException e) { + LOGGER.error("Error converting metrics to JSON", e); + return; + } + File tmpFile = null; + try { + tmpFile = createTempFile("sentry-json", null); + } catch (IOException e) { + LOGGER.error("failed to create temp file for JSON metrics", e); + } + + assert tmpFile != null; + try (LocalFileSystem fs = FileSystem.getLocal(conf); + BufferedWriter bw = new BufferedWriter(new FileWriter(tmpFile))) { + bw.write(json); + Path tmpPath = new Path(tmpFile.getAbsolutePath()); + fs.setPermission(tmpPath, FsPermission.createImmutable(PERMISSIONS)); + Path path = new Path(pathString); + fs.rename(tmpPath, path); + fs.setPermission(path, FsPermission.createImmutable(PERMISSIONS)); + } catch (IOException e) { + LOGGER.warn("Error writing JSON metrics", e); + } + } + + @Override + public void close() { + if (executor != null) { + executor.shutdownNow(); + executor = null; + } + } + } }
