This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 450f27c [STATS] introduce `StatsDoc` annotation for better
documenting metrics exposed by bookkeeper
450f27c is described below
commit 450f27ce3c80f78151bdd962c9deb31e92df6620
Author: Sijie Guo <guosi...@gmail.com>
AuthorDate: Tue Nov 13 00:54:56 2018 -0800
[STATS] introduce `StatsDoc` annotation for better documenting metrics
exposed by bookkeeper
Descriptions of the changes in this PR:
### Motivation
A common ask from people using bookkeeper is how they can monitor bookies
and bookkeeper clients, what kind of metrics that bookkeeper exposes
and what are the important metrics. Currently bookkeeper doesn't provide
any metrics page for guiding people on monitoring bookkeeper services.
In order to help people on this, we need to provide a few documentation
pages about metrics. However if we just write such pages, those pages
can quickly get out-of-dated when code is changed.
### Changes
- Introduce an annotation `StatsDoc` for annotating the
counters/gauges/opstats in the source code.
- Provide a tool to generate the stats and their documentation into a yaml
file.
The yaml file will be used by the website for rendering a metrics reference
page.
### Results
```
"server":
"bookie_BOOKIE_READ_ENTRY_BYTES":
"description": |-
bytes stats of ReadEntry on a bookie
"type": |-
OPSTATS
"bookie_WRITE_BYTES":
"description": |-
total bytes written to a bookie
"type": |-
COUNTER
"bookie_BOOKIE_ADD_ENTRY":
"description": |-
operations stats of AddEntry on a bookie
"type": |-
OPSTATS
"bookie_BOOKIE_RECOVERY_ADD_ENTRY":
"description": |-
operation stats of RecoveryAddEntry on a bookie
"type": |-
OPSTATS
"bookie_BOOKIE_ADD_ENTRY_BYTES":
"description": |-
bytes stats of AddEntry on a bookie
"type": |-
OPSTATS
"bookie_BOOKIE_FORCE_LEDGER":
"description": |-
total force operations occurred on a bookie
"type": |-
COUNTER
"bookie_READ_BYTES":
"description": |-
total bytes read from a bookie
"type": |-
COUNTER
"bookie_BOOKIE_READ_ENTRY":
"description": |-
operation stats of ReadEntry on a bookie
"type": |-
OPSTATS
```
Master Issue: #1786
Reviewers: Ivan Kelly <iv...@apache.org>, Jia Zhai <None>
This closes #1787 from sijie/stats_generator
---
.../apache/bookkeeper/test/TestStatsProvider.java | 12 +
.../bookkeeper/bookie/BookKeeperServerStats.java | 2 +
.../java/org/apache/bookkeeper/bookie/Bookie.java | 62 ++---
.../bookkeeper/bookie/stats/BookieStats.java | 84 ++++++
.../bookkeeper/bookie/stats/package-info.java | 23 ++
.../bookkeeper/stats/CodahaleMetricsProvider.java | 11 +
.../stats/codahale/CodahaleMetricsProvider.java | 11 +
.../prometheus/PrometheusMetricsProvider.java | 21 +-
.../twitter/finagle/FinagleStatsProvider.java | 5 +
.../stats/twitter/ostrich/OstrichProvider.java | 5 +
.../twitter/science/TwitterStatsProvider.java | 11 +
.../bookkeeper/stats/CachingStatsProvider.java | 5 +
.../java/org/apache/bookkeeper/stats/Stats.java | 4 +
.../org/apache/bookkeeper/stats/StatsProvider.java | 11 +
.../bookkeeper/stats/annotations/StatsDoc.java | 62 +++++
.../bookkeeper/stats/annotations/package-info.java | 23 ++
dev/stats-doc-gen | 66 +++++
pom.xml | 18 ++
stats/pom.xml | 39 +++
stats/utils/pom.xml | 55 ++++
.../bookkeeper/stats/utils/StatsDocGenerator.java | 298 +++++++++++++++++++++
.../bookkeeper/stats/utils/package-info.java | 23 ++
22 files changed, 806 insertions(+), 45 deletions(-)
diff --git
a/bookkeeper-common/src/test/java/org/apache/bookkeeper/test/TestStatsProvider.java
b/bookkeeper-common/src/test/java/org/apache/bookkeeper/test/TestStatsProvider.java
index aca154e..bef0dfe 100644
---
a/bookkeeper-common/src/test/java/org/apache/bookkeeper/test/TestStatsProvider.java
+++
b/bookkeeper-common/src/test/java/org/apache/bookkeeper/test/TestStatsProvider.java
@@ -31,6 +31,7 @@ import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.StatsProvider;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
/**
* Simple in-memory stat provider for use in unit tests.
@@ -261,4 +262,15 @@ public class TestStatsProvider implements StatsProvider {
private <T extends Number> void unregisterGauge(String name, Gauge<T>
gauge) {
gaugeMap.remove(name, gauge);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ if (statsComponents.length == 0) {
+ return "";
+ } else if (statsComponents[0].isEmpty()) {
+ return StringUtils.join(statsComponents, '.', 1,
statsComponents.length);
+ } else {
+ return StringUtils.join(statsComponents, '.');
+ }
+ }
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
index e048bd7..736d341 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
@@ -25,6 +25,8 @@ package org.apache.bookkeeper.bookie;
*/
public interface BookKeeperServerStats {
+ String CATEGORY_SERVER = "server";
+
String SERVER_SCOPE = "bookkeeper_server";
String BOOKIE_SCOPE = "bookie";
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
index 3795b37..d0db80e 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
@@ -21,18 +21,9 @@
package org.apache.bookkeeper.bookie;
-import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_ADD_ENTRY;
-import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_ADD_ENTRY_BYTES;
-import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_FORCE_LEDGER;
-import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_READ_ENTRY;
-import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_READ_ENTRY_BYTES;
-import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_RECOVERY_ADD_ENTRY;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.JOURNAL_SCOPE;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_BYTES;
-import static org.apache.bookkeeper.bookie.BookKeeperServerStats.WRITE_BYTES;
-import static org.apache.bookkeeper.bookie.Bookie.METAENTRY_ID_FENCE_KEY;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@@ -71,6 +62,7 @@ import
org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint;
import org.apache.bookkeeper.bookie.Journal.JournalScanner;
import org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener;
import
org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException;
+import org.apache.bookkeeper.bookie.stats.BookieStats;
import org.apache.bookkeeper.common.util.Watcher;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.discover.RegistrationManager;
@@ -82,9 +74,7 @@ import
org.apache.bookkeeper.meta.exceptions.MetadataException;
import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.net.DNS;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback;
-import org.apache.bookkeeper.stats.Counter;
import org.apache.bookkeeper.stats.NullStatsLogger;
-import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.util.BookKeeperConstants;
import org.apache.bookkeeper.util.DiskChecker;
@@ -142,16 +132,7 @@ public class Bookie extends BookieCriticalThread {
// Expose Stats
final StatsLogger statsLogger;
- private final Counter writeBytes;
- private final Counter readBytes;
- private final Counter forceLedgerOps;
- // Bookie Operation Latency Stats
- private final OpStatsLogger addEntryStats;
- private final OpStatsLogger recoveryAddEntryStats;
- private final OpStatsLogger readEntryStats;
- // Bookie Operation Bytes Stats
- private final OpStatsLogger addBytesStats;
- private final OpStatsLogger readBytesStats;
+ private final BookieStats bookieStats;
/**
* Exception is thrown when no such a ledger is found in this bookie.
@@ -744,14 +725,7 @@ public class Bookie extends BookieCriticalThread {
handles = new HandleFactoryImpl(ledgerStorage);
// Expose Stats
- writeBytes = statsLogger.getCounter(WRITE_BYTES);
- readBytes = statsLogger.getCounter(READ_BYTES);
- forceLedgerOps = statsLogger.getCounter(BOOKIE_FORCE_LEDGER);
- addEntryStats = statsLogger.getOpStatsLogger(BOOKIE_ADD_ENTRY);
- recoveryAddEntryStats =
statsLogger.getOpStatsLogger(BOOKIE_RECOVERY_ADD_ENTRY);
- readEntryStats = statsLogger.getOpStatsLogger(BOOKIE_READ_ENTRY);
- addBytesStats = statsLogger.getOpStatsLogger(BOOKIE_ADD_ENTRY_BYTES);
- readBytesStats = statsLogger.getOpStatsLogger(BOOKIE_READ_ENTRY_BYTES);
+ this.bookieStats = new BookieStats(statsLogger);
}
StateManager initializeStateManager() throws IOException {
@@ -1163,7 +1137,7 @@ public class Bookie extends BookieCriticalThread {
long ledgerId = handle.getLedgerId();
long entryId = handle.addEntry(entry);
- writeBytes.add(entry.readableBytes());
+ bookieStats.getWriteBytes().add(entry.readableBytes());
// journal `addEntry` should happen after the entry is added to ledger
storage.
// otherwise the journal entry can potentially be rolled before the
ledger is created in ledger storage.
@@ -1213,11 +1187,11 @@ public class Bookie extends BookieCriticalThread {
} finally {
long elapsedNanos = MathUtils.elapsedNanos(requestNanos);
if (success) {
- recoveryAddEntryStats.registerSuccessfulEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
- addBytesStats.registerSuccessfulValue(entrySize);
+
bookieStats.getRecoveryAddEntryStats().registerSuccessfulEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
+
bookieStats.getAddBytesStats().registerSuccessfulValue(entrySize);
} else {
- recoveryAddEntryStats.registerFailedEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
- addBytesStats.registerFailedValue(entrySize);
+
bookieStats.getRecoveryAddEntryStats().registerFailedEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
+ bookieStats.getAddBytesStats().registerFailedValue(entrySize);
}
entry.release();
@@ -1272,7 +1246,7 @@ public class Bookie extends BookieCriticalThread {
}
Journal journal = getJournal(ledgerId);
journal.forceLedger(ledgerId, cb, ctx);
- forceLedgerOps.inc();
+ bookieStats.getForceLedgerOps().inc();
}
/**
@@ -1301,11 +1275,11 @@ public class Bookie extends BookieCriticalThread {
} finally {
long elapsedNanos = MathUtils.elapsedNanos(requestNanos);
if (success) {
- addEntryStats.registerSuccessfulEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
- addBytesStats.registerSuccessfulValue(entrySize);
+
bookieStats.getAddEntryStats().registerSuccessfulEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
+
bookieStats.getAddBytesStats().registerSuccessfulValue(entrySize);
} else {
- addEntryStats.registerFailedEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
- addBytesStats.registerFailedValue(entrySize);
+
bookieStats.getAddEntryStats().registerFailedEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
+ bookieStats.getAddBytesStats().registerFailedValue(entrySize);
}
entry.release();
@@ -1355,17 +1329,17 @@ public class Bookie extends BookieCriticalThread {
LOG.trace("Reading {}@{}", entryId, ledgerId);
}
ByteBuf entry = handle.readEntry(entryId);
- readBytes.add(entry.readableBytes());
+ bookieStats.getReadBytes().add(entry.readableBytes());
success = true;
return entry;
} finally {
long elapsedNanos = MathUtils.elapsedNanos(requestNanos);
if (success) {
- readEntryStats.registerSuccessfulEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
- readBytesStats.registerSuccessfulValue(entrySize);
+
bookieStats.getReadEntryStats().registerSuccessfulEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
+
bookieStats.getReadBytesStats().registerSuccessfulValue(entrySize);
} else {
- readEntryStats.registerFailedEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
- readBytesStats.registerFailedValue(entrySize);
+
bookieStats.getReadEntryStats().registerFailedEvent(elapsedNanos,
TimeUnit.NANOSECONDS);
+ bookieStats.getReadEntryStats().registerFailedValue(entrySize);
}
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/BookieStats.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/BookieStats.java
new file mode 100644
index 0000000..72921d7
--- /dev/null
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/BookieStats.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.bookie.stats;
+
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_ADD_ENTRY;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_ADD_ENTRY_BYTES;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_FORCE_LEDGER;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_READ_ENTRY;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_READ_ENTRY_BYTES;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_RECOVERY_ADD_ENTRY;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_SCOPE;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_BYTES;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.WRITE_BYTES;
+
+import lombok.Getter;
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.stats.annotations.StatsDoc;
+
+/**
+ * A umbrella class for bookie related stats.
+ */
+@StatsDoc(
+ name = BOOKIE_SCOPE,
+ category = CATEGORY_SERVER,
+ help = "Bookie related stats"
+)
+@Getter
+public class BookieStats {
+
+ // Expose Stats
+ final StatsLogger statsLogger;
+ @StatsDoc(name = WRITE_BYTES, help = "total bytes written to a bookie")
+ private final Counter writeBytes;
+ @StatsDoc(name = READ_BYTES, help = "total bytes read from a bookie")
+ private final Counter readBytes;
+ @StatsDoc(name = BOOKIE_FORCE_LEDGER, help = "total force operations
occurred on a bookie")
+ private final Counter forceLedgerOps;
+ // Bookie Operation Latency Stats
+ @StatsDoc(name = BOOKIE_ADD_ENTRY, help = "operations stats of AddEntry on
a bookie")
+ private final OpStatsLogger addEntryStats;
+ @StatsDoc(name = BOOKIE_RECOVERY_ADD_ENTRY, help = "operation stats of
RecoveryAddEntry on a bookie")
+ private final OpStatsLogger recoveryAddEntryStats;
+ @StatsDoc(name = BOOKIE_READ_ENTRY, help = "operation stats of ReadEntry
on a bookie")
+ private final OpStatsLogger readEntryStats;
+ // Bookie Operation Bytes Stats
+ @StatsDoc(name = BOOKIE_ADD_ENTRY_BYTES, help = "bytes stats of AddEntry
on a bookie")
+ private final OpStatsLogger addBytesStats;
+ @StatsDoc(name = BOOKIE_READ_ENTRY_BYTES, help = "bytes stats of ReadEntry
on a bookie")
+ private final OpStatsLogger readBytesStats;
+
+ public BookieStats(StatsLogger statsLogger) {
+ this.statsLogger = statsLogger;
+ writeBytes = statsLogger.getCounter(WRITE_BYTES);
+ readBytes = statsLogger.getCounter(READ_BYTES);
+ forceLedgerOps = statsLogger.getCounter(BOOKIE_FORCE_LEDGER);
+ addEntryStats = statsLogger.getOpStatsLogger(BOOKIE_ADD_ENTRY);
+ recoveryAddEntryStats =
statsLogger.getOpStatsLogger(BOOKIE_RECOVERY_ADD_ENTRY);
+ readEntryStats = statsLogger.getOpStatsLogger(BOOKIE_READ_ENTRY);
+ addBytesStats = statsLogger.getOpStatsLogger(BOOKIE_ADD_ENTRY_BYTES);
+ readBytesStats = statsLogger.getOpStatsLogger(BOOKIE_READ_ENTRY_BYTES);
+ }
+
+
+}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/package-info.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/package-info.java
new file mode 100644
index 0000000..9926176
--- /dev/null
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Package of the classes for defining bookie stats.
+ */
+package org.apache.bookkeeper.bookie.stats;
\ No newline at end of file
diff --git
a/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
b/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
index 01658c7..bbf7bcd 100644
---
a/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
+++
b/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
@@ -140,4 +140,15 @@ public class CodahaleMetricsProvider implements
StatsProvider {
initIfNecessary();
return new CodahaleStatsLogger(getMetrics(), name);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ if (statsComponents.length == 0) {
+ return "";
+ }
+ String baseName = statsComponents[0];
+ String[] names = new String[statsComponents.length - 1];
+ System.arraycopy(statsComponents, 1, names, 0, names.length);
+ return MetricRegistry.name(baseName, names);
+ }
}
diff --git
a/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/codahale/CodahaleMetricsProvider.java
b/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/codahale/CodahaleMetricsProvider.java
index f4ca952..1bd5b18 100644
---
a/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/codahale/CodahaleMetricsProvider.java
+++
b/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/codahale/CodahaleMetricsProvider.java
@@ -141,4 +141,15 @@ public class CodahaleMetricsProvider implements
StatsProvider {
initIfNecessary();
return new CodahaleStatsLogger(getMetrics(), name);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ if (statsComponents.length == 0) {
+ return "";
+ }
+ String baseName = statsComponents[0];
+ String[] names = new String[statsComponents.length - 1];
+ System.arraycopy(statsComponents, 1, names, 0, names.length);
+ return MetricRegistry.name(baseName, names);
+ }
}
diff --git
a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java
b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java
index df8279e..0b4c0c2 100644
---
a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java
+++
b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java
@@ -46,6 +46,7 @@ import org.apache.bookkeeper.stats.CachingStatsProvider;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.StatsProvider;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
@@ -101,6 +102,19 @@ public class PrometheusMetricsProvider implements
StatsProvider {
public StatsLogger getStatsLogger(String scope) {
return new
PrometheusStatsLogger(PrometheusMetricsProvider.this, scope);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ String completeName;
+ if (statsComponents.length == 0) {
+ return "";
+ } else if (statsComponents[0].isEmpty()) {
+ completeName = StringUtils.join(statsComponents, '_', 1,
statsComponents.length);
+ } else {
+ completeName = StringUtils.join(statsComponents, '_');
+ }
+ return Collector.sanitizeMetricName(completeName);
+ }
});
}
@@ -184,6 +198,11 @@ public class PrometheusMetricsProvider implements
StatsProvider {
opStats.forEach((name, opStatLogger) ->
PrometheusTextFormatUtil.writeOpStat(writer, name, opStatLogger));
}
+ @Override
+ public String getStatsName(String... statsComponents) {
+ return cachingStatsProvider.getStatsName(statsComponents);
+ }
+
@VisibleForTesting
void rotateLatencyCollection() {
opStats.forEach((name, metric) -> {
@@ -222,4 +241,4 @@ public class PrometheusMetricsProvider implements
StatsProvider {
directMemoryUsage = tmpDirectMemoryUsage;
}
-}
\ No newline at end of file
+}
diff --git
a/bookkeeper-stats-providers/twitter-finagle-provider/src/main/java/org/apache/bookkeeper/stats/twitter/finagle/FinagleStatsProvider.java
b/bookkeeper-stats-providers/twitter-finagle-provider/src/main/java/org/apache/bookkeeper/stats/twitter/finagle/FinagleStatsProvider.java
index aff129d..a3b569c 100644
---
a/bookkeeper-stats-providers/twitter-finagle-provider/src/main/java/org/apache/bookkeeper/stats/twitter/finagle/FinagleStatsProvider.java
+++
b/bookkeeper-stats-providers/twitter-finagle-provider/src/main/java/org/apache/bookkeeper/stats/twitter/finagle/FinagleStatsProvider.java
@@ -64,4 +64,9 @@ public class FinagleStatsProvider implements StatsProvider {
public StatsLogger getStatsLogger(final String scope) {
return this.cachingStatsProvider.getStatsLogger(scope);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ return cachingStatsProvider.getStatsName(statsComponents);
+ }
}
diff --git
a/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
b/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
index 1cc7bf6..173b200 100644
---
a/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
+++
b/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
@@ -118,4 +118,9 @@ public class OstrichProvider implements StatsProvider {
public StatsLogger getStatsLogger(String scope) {
return cachingStatsProvider.getStatsLogger(scope);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ return cachingStatsProvider.getStatsName(statsComponents);
+ }
}
diff --git
a/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
b/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
index 75c2842..2a410c0 100644
---
a/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
+++
b/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
@@ -20,6 +20,7 @@ import org.apache.bookkeeper.stats.CachingStatsProvider;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.StatsProvider;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,6 +54,11 @@ public class TwitterStatsProvider implements StatsProvider {
public StatsLogger getStatsLogger(String scope) {
return new TwitterStatsLoggerImpl(scope);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ return StringUtils.join(statsComponents, '_').toLowerCase();
+ }
});
}
@@ -85,4 +91,9 @@ public class TwitterStatsProvider implements StatsProvider {
public StatsLogger getStatsLogger(String name) {
return this.cachingStatsProvider.getStatsLogger(name);
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ return this.cachingStatsProvider.getStatsName(statsComponents);
+ }
}
diff --git
a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/CachingStatsProvider.java
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/CachingStatsProvider.java
index 4d38e9e..e3fa3aa 100644
---
a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/CachingStatsProvider.java
+++
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/CachingStatsProvider.java
@@ -57,4 +57,9 @@ public class CachingStatsProvider implements StatsProvider {
}
return statsLogger;
}
+
+ @Override
+ public String getStatsName(String... statsComponents) {
+ return underlying.getStatsName(statsComponents);
+ }
}
diff --git
a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/Stats.java
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/Stats.java
index a8115b3..a3799b0 100644
--- a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/Stats.java
+++ b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/Stats.java
@@ -37,6 +37,10 @@ public class Stats {
public static void loadStatsProvider(Configuration conf) {
String className = conf.getString(STATS_PROVIDER_CLASS);
+ loadStatsProvider(className);
+ }
+
+ public static void loadStatsProvider(String className) {
if (className != null) {
try {
Class cls = Class.forName(className);
diff --git
a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/StatsProvider.java
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/StatsProvider.java
index b6e3460..0bb236a 100644
---
a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/StatsProvider.java
+++
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/StatsProvider.java
@@ -19,6 +19,7 @@ package org.apache.bookkeeper.stats;
import java.io.IOException;
import java.io.Writer;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
/**
* Provider to provide stats logger for different scopes.
@@ -53,4 +54,14 @@ public interface StatsProvider {
* @return stats logger for the given <i>scope</i>
*/
StatsLogger getStatsLogger(String scope);
+
+ /**
+ * Return the fully qualified stats name comprised of given
<tt>statsComponents</tt>.
+ *
+ * @param statsComponents stats components to comprise the fully qualified
stats name
+ * @return the fully qualified stats name
+ */
+ default String getStatsName(String...statsComponents) {
+ return StringUtils.join(statsComponents, '/');
+ }
}
diff --git
a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/annotations/StatsDoc.java
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/annotations/StatsDoc.java
new file mode 100644
index 0000000..97f487a
--- /dev/null
+++
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/annotations/StatsDoc.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.stats.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Documenting the stats.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Documented
+public @interface StatsDoc {
+
+ /**
+ * The name of the category to group stats together.
+ *
+ * @return name of the stats category.
+ */
+ String category() default "";
+
+ /**
+ * The scope of this stats.
+ *
+ * @return scope of this stats
+ */
+ String scope() default "";
+
+ /**
+ * The name of this stats.
+ *
+ * @return name of this stats
+ */
+ String name();
+
+ /**
+ * The help message of this stats.
+ *
+ * @return help message of this stats
+ */
+ String help();
+
+
+}
diff --git
a/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/annotations/package-info.java
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/annotations/package-info.java
new file mode 100644
index 0000000..b8daf75
--- /dev/null
+++
b/bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/annotations/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Annotations for bookkeeper stats api.
+ */
+package org.apache.bookkeeper.stats.annotations;
\ No newline at end of file
diff --git a/dev/stats-doc-gen b/dev/stats-doc-gen
new file mode 100755
index 0000000..3a20aef
--- /dev/null
+++ b/dev/stats-doc-gen
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Stats Documentation Generator
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+
+source ${BK_HOME}/bin/common.sh
+source ${BK_HOME}/conf/bk_cli_env.sh
+
+CLI_MODULE_PATH=stats/utils
+CLI_MODULE_NAME="(org.apache.bookkeeper.stats-)?bookkeeper-stats-utils"
+CLI_MODULE_HOME=${BK_HOME}/${CLI_MODULE_PATH}
+
+# find the module jar
+CLI_JAR=$(find_module_jar ${CLI_MODULE_PATH} ${CLI_MODULE_NAME})
+
+# set up the classpath
+CLI_CLASSPATH=$(set_module_classpath ${CLI_MODULE_PATH})
+
+DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.cli.properties
+if [ -z "${CLI_LOG_CONF}" ]; then
+ CLI_LOG_CONF=${DEFAULT_LOG_CONF}
+fi
+CLI_LOG_DIR=${CLI_LOG_DIR:-"$BK_HOME/logs"}
+CLI_LOG_FILE=${CLI_LOG_FILE:-"stats-doc-gen.log"}
+CLI_ROOT_LOGGER=${CLI_ROOT_LOGGER:-"INFO,ROLLINGFILE"}
+
+# add all dependencies in the classpath
+ALL_MODULE_PATH=bookkeeper-dist/all
+ALL_MODULE_CLASSPATH=$(set_module_classpath ${ALL_MODULE_PATH})
+
+# Configure the classpath
+CLI_CLASSPATH="$CLI_JAR:$CLI_CLASSPATH:$CLI_EXTRA_CLASSPATH:$ALL_MODULE_CLASSPATH"
+CLI_CLASSPATH="`dirname $CLI_LOG_CONF`:$CLI_CLASSPATH"
+
+# Build the OPTs
+BOOKIE_OPTS=$(build_bookie_opts)
+GC_OPTS=$(build_cli_jvm_opts ${CLI_LOG_DIR} "stats-doc-gen-gc.log")
+NETTY_OPTS=$(build_netty_opts)
+LOGGING_OPTS=$(build_cli_logging_opts ${CLI_LOG_CONF} ${CLI_LOG_DIR}
${CLI_LOG_FILE} ${CLI_ROOT_LOGGER})
+
+OPTS="${OPTS} -cp ${CLI_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS}
${LOGGING_OPTS} ${CLI_EXTRA_OPTS}"
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+echo "running stats-doc-gen, logging to ${CLI_LOG_DIR}/${CLI_LOG_FILE}"
+exec ${JAVA} ${OPTS} org.apache.bookkeeper.stats.utils.StatsDocGenerator $@
diff --git a/pom.xml b/pom.xml
index 25624b2..5c6d9eb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,6 +55,8 @@
<module>circe-checksum</module>
<module>bookkeeper-common</module>
<module>bookkeeper-common-allocator</module>
+ <module>stats</module>
+ <!-- TODO: move `bookkeeper-stats` and `bookkeeper-stats-providers` as
submodules of `stats` -->
<module>bookkeeper-stats</module>
<module>bookkeeper-proto</module>
<module>bookkeeper-server</module>
@@ -156,9 +158,11 @@
<protobuf.version>3.5.1</protobuf.version>
<protoc3.version>3.5.1-1</protoc3.version>
<protoc-gen-grpc-java.version>1.12.0</protoc-gen-grpc-java.version>
+ <reflections.version>0.9.11</reflections.version>
<rocksdb.version>5.13.1</rocksdb.version>
<shrinkwrap.version>3.0.1</shrinkwrap.version>
<slf4j.version>1.7.25</slf4j.version>
+ <snakeyaml.version>1.23</snakeyaml.version>
<spotbugs-annotations.version>3.1.1</spotbugs-annotations.version>
<javax-annotations-api.version>1.3.2</javax-annotations-api.version>
<testcontainers.version>1.8.3</testcontainers.version>
@@ -297,6 +301,13 @@
<version>${commons-lang3.version}</version>
</dependency>
+ <!-- reflection libs -->
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>${reflections.version}</version>
+ </dependency>
+
<!-- compression libs -->
<dependency>
<groupId>net.jpountz.lz4</groupId>
@@ -311,6 +322,13 @@
<version>${jna.version}</version>
</dependency>
+ <!-- yaml dependencies -->
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
+
<!-- jackson dependencies -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
diff --git a/stats/pom.xml b/stats/pom.xml
new file mode 100644
index 0000000..c699ed4
--- /dev/null
+++ b/stats/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";
xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper</artifactId>
+ <version>4.9.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <packaging>pom</packaging>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>bookkeeper-stats-parent</artifactId>
+ <name>Apache BookKeeper :: Stats :: Parent</name>
+
+ <modules>
+ <module>utils</module>
+ </modules>
+
+</project>
+
diff --git a/stats/utils/pom.xml b/stats/utils/pom.xml
new file mode 100644
index 0000000..9be19cf
--- /dev/null
+++ b/stats/utils/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";
xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>bookkeeper-stats-parent</artifactId>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <version>4.9.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>bookkeeper-stats-utils</artifactId>
+ <name>Apache BookKeeper :: Stats :: Utils</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>bookkeeper-stats-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git
a/stats/utils/src/main/java/org/apache/bookkeeper/stats/utils/StatsDocGenerator.java
b/stats/utils/src/main/java/org/apache/bookkeeper/stats/utils/StatsDocGenerator.java
new file mode 100644
index 0000000..db2e13f
--- /dev/null
+++
b/stats/utils/src/main/java/org/apache/bookkeeper/stats/utils/StatsDocGenerator.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.stats.utils;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.google.common.base.Predicate;
+import com.google.common.base.Strings;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.reflect.Field;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.Stats;
+import org.apache.bookkeeper.stats.StatsProvider;
+import org.apache.bookkeeper.stats.annotations.StatsDoc;
+import org.reflections.Reflections;
+import org.reflections.util.ConfigurationBuilder;
+import org.reflections.util.FilterBuilder;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
+import org.yaml.snakeyaml.DumperOptions.ScalarStyle;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * Generator stats documentation.
+ */
+@Slf4j
+public class StatsDocGenerator {
+
+ enum StatsType {
+ COUNTER,
+ GAUGE,
+ OPSTATS
+ }
+
+ @AllArgsConstructor
+ @Data
+ static class StatsDocEntry {
+ private String name;
+ private StatsType type;
+ private String description;
+
+ public Map<String, String> properties() {
+ Map<String, String> properties = new TreeMap<>();
+ properties.put("type", type.name());
+ properties.put("description", description);
+ return properties;
+ }
+ }
+
+ private static Reflections newReflections(String packagePrefix) {
+ List<URL> urls = new ArrayList<>();
+ ClassLoader[] classLoaders = new ClassLoader[] {
+ StatsDocGenerator.class.getClassLoader(),
+ Thread.currentThread().getContextClassLoader()
+ };
+ for (int i = 0; i < classLoaders.length; i++) {
+ if (classLoaders[i] instanceof URLClassLoader) {
+ urls.addAll(Arrays.asList(((URLClassLoader)
classLoaders[i]).getURLs()));
+ } else {
+ throw new RuntimeException("ClassLoader '" + classLoaders[i] +
" is not an instance of URLClassLoader");
+ }
+ }
+ Predicate<String> filters = new FilterBuilder()
+ .includePackage(packagePrefix);
+ ConfigurationBuilder confBuilder = new ConfigurationBuilder();
+ confBuilder.filterInputsBy(filters);
+ confBuilder.setUrls(urls);
+ return new Reflections(confBuilder);
+ }
+
+ private final String packagePrefix;
+ private final Reflections reflections;
+ private final StatsProvider statsProvider;
+ private final NavigableMap<String, NavigableMap<String, StatsDocEntry>>
docEntries = new TreeMap<>();
+
+ public StatsDocGenerator(String packagePrefix,
+ StatsProvider provider) {
+ this.packagePrefix = packagePrefix;
+ this.reflections = newReflections(packagePrefix);
+ this.statsProvider = provider;
+ }
+
+ public void generate(String filename) throws Exception {
+ log.info("Processing classes under package {}", packagePrefix);
+ // get all classes annotated with `StatsDoc`
+ Set<Class<?>> annotatedClasses =
reflections.getTypesAnnotatedWith(StatsDoc.class);
+ log.info("Retrieve all `StatsDoc` annotated classes : {}",
annotatedClasses);
+
+ for (Class<?> annotatedClass : annotatedClasses) {
+ generateDocForAnnotatedClass(annotatedClass);
+ }
+ log.info("Successfully processed classes under package {}",
packagePrefix);
+ log.info("Writing stats doc to file {}", filename);
+ writeDoc(filename);
+ log.info("Successfully wrote stats doc to file {}", filename);
+ }
+
+ private void generateDocForAnnotatedClass(Class<?> annotatedClass) {
+ StatsDoc scopeStatsDoc =
annotatedClass.getDeclaredAnnotation(StatsDoc.class);
+ if (scopeStatsDoc == null) {
+ return;
+ }
+
+ log.info("Processing StatsDoc annotated class {} : {}",
annotatedClass, scopeStatsDoc);
+
+ Field[] fields = annotatedClass.getDeclaredFields();
+ for (Field field : fields) {
+ StatsDoc fieldStatsDoc =
field.getDeclaredAnnotation(StatsDoc.class);
+ if (null == fieldStatsDoc) {
+ // it is not a `StatsDoc` annotated field
+ continue;
+ }
+ generateDocForAnnotatedField(scopeStatsDoc, fieldStatsDoc, field);
+ }
+
+ log.info("Successfully processed StatsDoc annotated class {}.",
annotatedClass);
+ }
+
+ private NavigableMap<String, StatsDocEntry> getCategoryMap(String
category) {
+ NavigableMap<String, StatsDocEntry> categoryMap =
docEntries.get(category);
+ if (null == categoryMap) {
+ categoryMap = new TreeMap<>();
+ docEntries.put(category, categoryMap);
+ }
+ return categoryMap;
+ }
+
+ private void generateDocForAnnotatedField(StatsDoc scopedStatsDoc,
StatsDoc fieldStatsDoc, Field field) {
+ NavigableMap<String, StatsDocEntry> categoryMap =
getCategoryMap(scopedStatsDoc.category());
+
+ String statsName =
+ statsProvider.getStatsName(scopedStatsDoc.scope(),
scopedStatsDoc.name(), fieldStatsDoc.name());
+ StatsType statsType;
+ if (Counter.class.isAssignableFrom(field.getType())) {
+ statsType = StatsType.COUNTER;
+ } else if (Gauge.class.isAssignableFrom(field.getType())) {
+ statsType = StatsType.GAUGE;
+ } else if (OpStatsLogger.class.isAssignableFrom(field.getType())) {
+ statsType = StatsType.OPSTATS;
+ } else {
+ throw new IllegalArgumentException("Unknown stats field '" +
field.getName()
+ + "' is annotated with `StatsDoc`: " + field.getType());
+ }
+
+ String helpDesc = fieldStatsDoc.help();
+ StatsDocEntry docEntry = new StatsDocEntry(statsName, statsType,
helpDesc);
+ categoryMap.put(statsName, docEntry);
+ }
+
+ private void writeDoc(String file) throws IOException {
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(FlowStyle.BLOCK);
+ options.setDefaultScalarStyle(ScalarStyle.LITERAL);
+ Yaml yaml = new Yaml(options);
+ Writer writer;
+ if (Strings.isNullOrEmpty(file)) {
+ writer = new OutputStreamWriter(System.out, UTF_8);
+ } else {
+ writer = new OutputStreamWriter(new FileOutputStream(file), UTF_8);
+ }
+ try {
+ Map<String, Map<String, Map<String, String>>> docs =
docEntries.entrySet()
+ .stream()
+ .collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().entrySet()
+ .stream()
+ .collect(Collectors.toMap(
+ e1 -> e1.getKey(),
+ e1 -> e1.getValue().properties()
+ ))
+ ));
+ yaml.dump(docs, writer);
+ writer.flush();
+ } finally {
+ writer.close();
+ }
+ }
+
+ /**
+ * Args for stats generator.
+ */
+ private static class MainArgs {
+
+ @Parameter(
+ names = {
+ "-p", "--package"
+ },
+ description = "Package prefix of the classes to generate stats
doc")
+ String packagePrefix = "org.apache.bookkeeper";
+
+ @Parameter(
+ names = {
+ "-sp", "--stats-provider"
+ },
+ description = "The stats provider used for generating stats doc")
+ String statsProviderClass = "prometheus";
+
+ @Parameter(
+ names = {
+ "-o", "--output-yaml-file"
+ },
+ description = "The output yaml file to dump stats docs."
+ + " If omitted, the output goes to stdout."
+ )
+ String yamlFile = null;
+
+ @Parameter(
+ names = {
+ "-h", "--help"
+ },
+ description = "Show this help message")
+ boolean help = false;
+
+ }
+
+ public static void main(String[] args) throws Exception {
+ MainArgs mainArgs = new MainArgs();
+
+ JCommander commander = new JCommander();
+ try {
+ commander.setProgramName("stats-doc-gen");
+ commander.addObject(mainArgs);
+ commander.parse(args);
+ if (mainArgs.help) {
+ commander.usage();
+ Runtime.getRuntime().exit(0);
+ return;
+ }
+ } catch (Exception e) {
+ commander.usage();
+ Runtime.getRuntime().exit(-1);
+ return;
+ }
+
+
Stats.loadStatsProvider(getStatsProviderClass(mainArgs.statsProviderClass));
+ StatsProvider provider = Stats.get();
+
+ StatsDocGenerator docGen = new StatsDocGenerator(
+ mainArgs.packagePrefix,
+ provider
+ );
+ docGen.generate(mainArgs.yamlFile);
+ }
+
+ private static String getStatsProviderClass(String providerClass) {
+ switch (providerClass.toLowerCase()) {
+ case "ostrich":
+ return
"org.apache.bookkeeper.stats.twitter.ostrich.OstrichProvider";
+ case "prometheus":
+ return
"org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider";
+ case "finagle":
+ return
"org.apache.bookkeeper.stats.twitter.finagle.FinagleStatsProvider";
+ case "codahale":
+ return
"org.apache.bookkeeper.stats.codahale.CodahaleMetricsProvider";
+ default:
+ return providerClass;
+ }
+ }
+
+}
diff --git
a/stats/utils/src/main/java/org/apache/bookkeeper/stats/utils/package-info.java
b/stats/utils/src/main/java/org/apache/bookkeeper/stats/utils/package-info.java
new file mode 100644
index 0000000..c0b248f
--- /dev/null
+++
b/stats/utils/src/main/java/org/apache/bookkeeper/stats/utils/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Utilities for bookkeeper stats api.
+ */
+package org.apache.bookkeeper.stats.utils;
\ No newline at end of file
