This is an automated email from the ASF dual-hosted git repository.
frankchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 4574dea5e9 Use MXBeans to get GC metrics #12476 (#12481)
4574dea5e9 is described below
commit 4574dea5e92beb72d93e1d6900ce1a63627330d0
Author: Jianhuan Liu <[email protected]>
AuthorDate: Fri Jul 8 14:32:06 2022 +0800
Use MXBeans to get GC metrics #12476 (#12481)
* jvm gc to mxbeans
* add zgc and shenandoah #12476
* remove tryCreateGcCounter
* separate the space collector
* blend GcGenerationCollector into GcCollector
* add jdk surefire argLine
---
.../apache/druid/java/util/metrics/JvmMonitor.java | 304 ++++++++-------------
.../druid/java/util/metrics/JvmMonitorTest.java | 39 +--
pom.xml | 1 -
processing/pom.xml | 1 -
4 files changed, 114 insertions(+), 231 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java
b/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java
index 77f4033a3d..7c9f6be99d 100644
--- a/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java
+++ b/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java
@@ -22,19 +22,13 @@ package org.apache.druid.java.util.metrics;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
-import it.unimi.dsi.fastutil.ints.IntRBTreeSet;
-import it.unimi.dsi.fastutil.ints.IntSet;
-import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
-import org.gridkit.lab.jvm.perfdata.JStatData;
-import org.gridkit.lab.jvm.perfdata.JStatData.LongCounter;
-import org.gridkit.lab.jvm.perfdata.JStatData.StringCounter;
-import org.gridkit.lab.jvm.perfdata.JStatData.TickCounter;
import javax.annotation.Nullable;
import java.lang.management.BufferPoolMXBean;
+import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
import java.lang.management.MemoryType;
@@ -42,22 +36,15 @@ import java.lang.management.MemoryUsage;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
public class JvmMonitor extends FeedDefiningMonitor
{
- private static final Logger log = new Logger(JvmMonitor.class);
- private static final Pattern PATTERN_GC_GENERATION =
- Pattern.compile("^sun\\.gc\\.(?:generation|collector)\\.(\\d+)\\..*");
private final Map<String, String[]> dimensions;
- private final long pid;
@VisibleForTesting
@Nullable
- final GcCounters gcCounters;
+ final GcCollectors gcCollectors;
@Nullable
private final AllocationMetricCollector collector;
@@ -73,18 +60,12 @@ public class JvmMonitor extends FeedDefiningMonitor
}
public JvmMonitor(Map<String, String[]> dimensions, String feed)
- {
- this(dimensions, feed, JvmPidDiscoverer.instance());
- }
-
- public JvmMonitor(Map<String, String[]> dimensions, String feed,
PidDiscoverer pidDiscoverer)
{
super(feed);
Preconditions.checkNotNull(dimensions);
this.dimensions = ImmutableMap.copyOf(dimensions);
- this.pid = Preconditions.checkNotNull(pidDiscoverer).getPid();
this.collector = AllocationMetricCollectors.getAllocationMetricCollector();
- this.gcCounters = tryCreateGcCounters();
+ this.gcCollectors = new GcCollectors();
}
@Override
@@ -162,199 +143,152 @@ public class JvmMonitor extends FeedDefiningMonitor
private void emitGcMetrics(ServiceEmitter emitter)
{
- if (gcCounters != null) {
- gcCounters.emit(emitter, dimensions);
- }
- }
-
- @Nullable
- private GcCounters tryCreateGcCounters()
- {
- try {
- return new GcCounters();
- }
- catch (RuntimeException e) {
- // in JDK11 jdk.internal.perf.Perf is not accessible, unless
- // --add-exports java.base/jdk.internal.perf=ALL-UNNAMED is set
- log.warn("Cannot initialize GC counters. If running JDK11 and above,"
- + " add --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED"
- + " to the JVM arguments to enable GC counters.");
- }
- return null;
+ gcCollectors.emit(emitter, dimensions);
}
- @VisibleForTesting
- static IntSet getGcGenerations(final Set<String> jStatCounterNames)
+ private class GcCollectors
{
- final IntSet retVal = new IntRBTreeSet();
+ private final List<GcGenerationCollector> generationCollectors = new
ArrayList<>();
+ private final List<GcSpaceCollector> spaceCollectors = new ArrayList<>();
- for (final String counterName : jStatCounterNames) {
- final Matcher m = PATTERN_GC_GENERATION.matcher(counterName);
- if (m.matches()) {
- retVal.add(Integer.parseInt(m.group(1)));
+ GcCollectors()
+ {
+ List<GarbageCollectorMXBean> collectorMxBeans =
ManagementFactory.getGarbageCollectorMXBeans();
+ for (GarbageCollectorMXBean collectorMxBean : collectorMxBeans) {
+ generationCollectors.add(new GcGenerationCollector(collectorMxBean));
}
- }
- return retVal;
- }
+ List<MemoryPoolMXBean> memoryPoolMxBeans =
ManagementFactory.getMemoryPoolMXBeans();
+ for (MemoryPoolMXBean memoryPoolMxBean : memoryPoolMxBeans) {
+ MemoryUsage collectionUsage = memoryPoolMxBean.getCollectionUsage();
+ if (collectionUsage != null) {
+ spaceCollectors.add(new GcSpaceCollector(collectionUsage,
memoryPoolMxBean.getName()));
+ }
+ }
- @VisibleForTesting
- static String getGcGenerationName(final int genIndex)
- {
- switch (genIndex) {
- case 0:
- return "young";
- case 1:
- return "old";
- case 2:
- // Removed in Java 8 but still actual for previous Java versions
- return "perm";
- default:
- return String.valueOf(genIndex);
}
- }
- /**
- * The following GC-related code is partially based on
- *
https://github.com/aragozin/jvm-tools/blob/e0e37692648951440aa1a4ea5046261cb360df70/
- *
sjk-core/src/main/java/org/gridkit/jvmtool/PerfCounterGcCpuUsageMonitor.java
- */
- private class GcCounters
- {
- private final List<GcGeneration> generations = new ArrayList<>();
-
- GcCounters()
+ void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
{
- // connect to itself
- final JStatData jStatData = JStatData.connect(pid);
- final Map<String, JStatData.Counter<?>> jStatCounters =
jStatData.getAllCounters();
-
- for (int genIndex : getGcGenerations(jStatCounters.keySet())) {
- generations.add(new GcGeneration(jStatCounters, genIndex,
getGcGenerationName(genIndex)));
+ for (GcGenerationCollector generationCollector : generationCollectors) {
+ generationCollector.emit(emitter, dimensions);
}
- }
- void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
- {
- for (GcGeneration generation : generations) {
- generation.emit(emitter, dimensions);
+ for (GcSpaceCollector spaceCollector : spaceCollectors) {
+ spaceCollector.emit(emitter, dimensions);
}
}
}
- private class GcGeneration
+ private class GcGenerationCollector
{
- private final String name;
- @Nullable
- private final GcGenerationCollector collector;
- private final List<GcGenerationSpace> spaces = new ArrayList<>();
+ private final String generation;
+ private final String collectorName;
+ private final GarbageCollectorMXBean gcBean;
- GcGeneration(Map<String, JStatData.Counter<?>> jStatCounters, long
genIndex, String name)
- {
- this.name = StringUtils.toLowerCase(name);
+ private long lastInvocations = 0;
+ private long lastCpuNanos = 0;
- final String spacesCountKey =
StringUtils.format("sun.gc.generation.%d.spaces", genIndex);
+ private static final String GC_YOUNG_GENERATION_NAME = "young";
+ private static final String GC_OLD_GENERATION_NAME = "old";
+ private static final String GC_ZGC_GENERATION_NAME = "zgc";
- if (jStatCounters.containsKey(spacesCountKey)) {
- final long spacesCount = ((JStatData.LongCounter)
jStatCounters.get(spacesCountKey)).getLong();
- for (long spaceIndex = 0; spaceIndex < spacesCount; spaceIndex++) {
- spaces.add(new GcGenerationSpace(jStatCounters, genIndex,
spaceIndex));
- }
- }
+ private static final String CMS_COLLECTOR_NAME = "cms";
+ private static final String G1_COLLECTOR_NAME = "g1";
+ private static final String PARALLEL_COLLECTOR_NAME = "parallel";
+ private static final String SERIAL_COLLECTOR_NAME = "serial";
+ private static final String ZGC_COLLECTOR_NAME = "zgc";
+ private static final String SHENANDOAN_COLLECTOR_NAME = "shenandoah";
- if
(jStatCounters.containsKey(StringUtils.format("sun.gc.collector.%d.name",
genIndex))) {
- collector = new GcGenerationCollector(jStatCounters, genIndex);
- } else {
- collector = null;
- }
+ GcGenerationCollector(GarbageCollectorMXBean gcBean)
+ {
+ Pair<String, String> gcNamePair = getReadableName(gcBean.getName());
+ this.generation = gcNamePair.lhs;
+ this.collectorName = gcNamePair.rhs;
+ this.gcBean = gcBean;
}
- void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
+ private Pair<String, String> getReadableName(String name)
{
- ImmutableMap.Builder<String, String[]> dimensionsCopyBuilder =
ImmutableMap
- .<String, String[]>builder()
- .putAll(dimensions)
- .put("gcGen", new String[]{name});
-
- if (collector != null) {
- dimensionsCopyBuilder.put("gcName", new String[]{collector.name});
- }
+ switch (name) {
+ //CMS
+ case "ParNew":
+ return new Pair<>(GC_YOUNG_GENERATION_NAME, CMS_COLLECTOR_NAME);
+ case "ConcurrentMarkSweep":
+ return new Pair<>(GC_OLD_GENERATION_NAME, CMS_COLLECTOR_NAME);
+
+ // G1
+ case "G1 Young Generation":
+ return new Pair<>(GC_YOUNG_GENERATION_NAME, G1_COLLECTOR_NAME);
+ case "G1 Old Generation":
+ return new Pair<>(GC_OLD_GENERATION_NAME, G1_COLLECTOR_NAME);
+
+ // Parallel
+ case "PS Scavenge":
+ return new Pair<>(GC_YOUNG_GENERATION_NAME, PARALLEL_COLLECTOR_NAME);
+ case "PS MarkSweep":
+ return new Pair<>(GC_OLD_GENERATION_NAME, PARALLEL_COLLECTOR_NAME);
+
+ // Serial
+ case "Copy":
+ return new Pair<>(GC_YOUNG_GENERATION_NAME, SERIAL_COLLECTOR_NAME);
+ case "MarkSweepCompact":
+ return new Pair<>(GC_OLD_GENERATION_NAME, SERIAL_COLLECTOR_NAME);
- Map<String, String[]> dimensionsCopy = dimensionsCopyBuilder.build();
+ //zgc
+ case "ZGC":
+ return new Pair<>(GC_ZGC_GENERATION_NAME, ZGC_COLLECTOR_NAME);
- if (collector != null) {
- collector.emit(emitter, dimensionsCopy);
- }
+ //Shenandoah
+ case "Shenandoah Cycles":
+ return new Pair<>(GC_YOUNG_GENERATION_NAME,
SHENANDOAN_COLLECTOR_NAME);
+ case "Shenandoah Pauses":
+ return new Pair<>(GC_OLD_GENERATION_NAME, SHENANDOAN_COLLECTOR_NAME);
- for (GcGenerationSpace space : spaces) {
- space.emit(emitter, dimensionsCopy);
+ default:
+ return new Pair<>(name, name);
}
}
- }
- private class GcGenerationCollector
- {
- private final String name;
- private final LongCounter invocationsCounter;
- private final TickCounter cpuCounter;
- private long lastInvocations = 0;
- private long lastCpuNanos = 0;
-
- GcGenerationCollector(Map<String, JStatData.Counter<?>> jStatCounters,
long genIndex)
+ void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
{
- String collectorKeyPrefix = StringUtils.format("sun.gc.collector.%d",
genIndex);
+ ImmutableMap.Builder<String, String[]> dimensionsCopyBuilder =
ImmutableMap
+ .<String, String[]>builder()
+ .putAll(dimensions)
+ .put("gcGen", new String[]{generation});
- String nameKey = StringUtils.format("%s.name", collectorKeyPrefix);
- StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey);
- name = getReadableName(nameCounter.getString());
+ dimensionsCopyBuilder.put("gcName", new String[]{collectorName});
- invocationsCounter = (LongCounter)
jStatCounters.get(StringUtils.format("%s.invocations", collectorKeyPrefix));
- cpuCounter = (TickCounter)
jStatCounters.get(StringUtils.format("%s.time", collectorKeyPrefix));
- }
+ Map<String, String[]> dimensionsCopy = dimensionsCopyBuilder.build();
- void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
- {
final ServiceMetricEvent.Builder builder = builder();
- MonitorUtils.addDimensionsToBuilder(builder, dimensions);
+ MonitorUtils.addDimensionsToBuilder(builder, dimensionsCopy);
- long newInvocations = invocationsCounter.getLong();
+ long newInvocations = gcBean.getCollectionCount();
emitter.emit(builder.build("jvm/gc/count", newInvocations -
lastInvocations));
lastInvocations = newInvocations;
- long newCpuNanos = cpuCounter.getNanos();
+ long newCpuNanos = gcBean.getCollectionTime();
emitter.emit(builder.build("jvm/gc/cpu", newCpuNanos - lastCpuNanos));
lastCpuNanos = newCpuNanos;
+
}
+ }
- private String getReadableName(String name)
+ private class GcSpaceCollector
+ {
+
+ private final List<GcGenerationSpace> spaces = new ArrayList<>();
+
+ public GcSpaceCollector(MemoryUsage collectionUsage, String name)
{
- switch (name) {
- // Young gen
- case "Copy":
- return "serial";
- case "PSScavenge":
- return "parallel";
- case "PCopy":
- return "cms";
- case "G1 incremental collections":
- return "g1";
- case "Shenandoah partial":
- return "shenandoah";
-
- // Old gen
- case "MCS":
- return "serial";
- case "PSParallelCompact":
- return "parallel";
- case "CMS":
- return "cms";
- case "G1 stop-the-world full collections":
- return "g1";
- case "Shenandoah full":
- return "shenandoah";
+ spaces.add(new GcGenerationSpace(collectionUsage, name));
+ }
- default:
- return name;
+ void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
+ {
+ for (GcGenerationSpace space : spaces) {
+ space.emit(emitter, dimensions);
}
}
}
@@ -362,24 +296,12 @@ public class JvmMonitor extends FeedDefiningMonitor
private class GcGenerationSpace
{
private final String name;
+ private final MemoryUsage memoryUsage;
- private final LongCounter maxCounter;
- private final LongCounter capacityCounter;
- private final LongCounter usedCounter;
- private final LongCounter initCounter;
-
- GcGenerationSpace(Map<String, JStatData.Counter<?>> jStatCounters, long
genIndex, long spaceIndex)
+ public GcGenerationSpace(MemoryUsage memoryUsage, String name)
{
- String spaceKeyPrefix =
StringUtils.format("sun.gc.generation.%d.space.%d", genIndex, spaceIndex);
-
- String nameKey = StringUtils.format("%s.name", spaceKeyPrefix);
- StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey);
- name = StringUtils.toLowerCase(nameCounter.toString());
-
- maxCounter = (LongCounter)
jStatCounters.get(StringUtils.format("%s.maxCapacity", spaceKeyPrefix));
- capacityCounter = (LongCounter)
jStatCounters.get(StringUtils.format("%s.capacity", spaceKeyPrefix));
- usedCounter = (LongCounter)
jStatCounters.get(StringUtils.format("%s.used", spaceKeyPrefix));
- initCounter = (LongCounter)
jStatCounters.get(StringUtils.format("%s.initCapacity", spaceKeyPrefix));
+ this.memoryUsage = memoryUsage;
+ this.name = name;
}
void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
@@ -389,10 +311,10 @@ public class JvmMonitor extends FeedDefiningMonitor
builder.setDimension("gcGenSpaceName", name);
- emitter.emit(builder.build("jvm/gc/mem/max", maxCounter.getLong()));
- emitter.emit(builder.build("jvm/gc/mem/capacity",
capacityCounter.getLong()));
- emitter.emit(builder.build("jvm/gc/mem/used", usedCounter.getLong()));
- emitter.emit(builder.build("jvm/gc/mem/init", initCounter.getLong()));
+ emitter.emit(builder.build("jvm/gc/mem/max", memoryUsage.getMax()));
+ emitter.emit(builder.build("jvm/gc/mem/capacity",
memoryUsage.getCommitted()));
+ emitter.emit(builder.build("jvm/gc/mem/used", memoryUsage.getUsed()));
+ emitter.emit(builder.build("jvm/gc/mem/init", memoryUsage.getInit()));
}
}
}
diff --git
a/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java
b/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java
index d5a2890e08..24af5db4c5 100644
--- a/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java
+++ b/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java
@@ -19,8 +19,6 @@
package org.apache.druid.java.util.metrics;
-import com.google.common.collect.ImmutableSet;
-import it.unimi.dsi.fastutil.ints.IntSet;
import org.apache.druid.java.util.emitter.core.Emitter;
import org.apache.druid.java.util.emitter.core.Event;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
@@ -43,7 +41,7 @@ public class JvmMonitorTest
serviceEmitter.start();
final JvmMonitor jvmMonitor = new JvmMonitor();
// skip tests if gc counters fail to initialize with this JDK
- Assume.assumeNotNull(jvmMonitor.gcCounters);
+ Assume.assumeNotNull(jvmMonitor.gcCollectors);
while (true) {
// generate some garbage to see gc counters incremented
@@ -58,41 +56,6 @@ public class JvmMonitorTest
}
}
- @Test
- public void testGetGcGenerations()
- {
- Assert.assertEquals(
- IntSet.of(0, 1),
- JvmMonitor.getGcGenerations(
- ImmutableSet.of(
- "sun.gc.collector.0.name",
- "sun.gc.collector.1.name",
- "sun.gc.generation.1.spaces"
- )
- )
- );
-
- Assert.assertEquals(
- IntSet.of(1, 2),
- JvmMonitor.getGcGenerations(
- ImmutableSet.of(
- "sun.gc.generation.1.spaces",
- "sun.gc.collector.2.name",
- "sun.gc.somethingelse.3.name"
- )
- )
- );
- }
-
- @Test
- public void testGetGcGenerationName()
- {
- Assert.assertEquals("young", JvmMonitor.getGcGenerationName(0));
- Assert.assertEquals("old", JvmMonitor.getGcGenerationName(1));
- Assert.assertEquals("perm", JvmMonitor.getGcGenerationName(2));
- Assert.assertEquals("3", JvmMonitor.getGcGenerationName(3));
- }
-
private static class GcTrackingEmitter implements Emitter
{
private Number oldGcCount;
diff --git a/pom.xml b/pom.xml
index fce189172b..5d29c54cb8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1757,7 +1757,6 @@
<!-- locale settings must be set on the command
line before startup -->
<!-- set heap size to work around
https://github.com/travis-ci/travis-ci/issues/3396 -->
<argLine>
- ${jdk.surefire.argLine}
-Xmx768m -Duser.language=en -Duser.country=US
-Dfile.encoding=UTF-8
-XX:+ExitOnOutOfMemoryError
-XX:+HeapDumpOnOutOfMemoryError
diff --git a/processing/pom.xml b/processing/pom.xml
index 80a8c80016..da5e9a7638 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -300,7 +300,6 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>
- ${jdk.surefire.argLine}
-server -Xms3G -Xmx3G
-Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks
-XX:+ExitOnOutOfMemoryError
-XX:+HeapDumpOnOutOfMemoryError
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
