This is an automated email from the ASF dual-hosted git repository. mmuzaf pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new b6082be IGNITE-12053: Total time threads parked if checkpoint throttling occurred metric added (#7080) b6082be is described below commit b6082bed9e1244cbde55574538c34e18c43a381d Author: Nikita Amelchev <nsamelc...@gmail.com> AuthorDate: Fri Nov 29 19:52:48 2019 +0300 IGNITE-12053: Total time threads parked if checkpoint throttling occurred metric added (#7080) --- .../cache/persistence/DataRegionMetricsImpl.java | 13 ++++++++++ .../cache/persistence/pagemem/PageMemoryImpl.java | 8 +++++- .../pagemem/PagesWriteSpeedBasedThrottle.java | 2 ++ .../persistence/pagemem/PagesWriteThrottle.java | 4 +++ .../pagemem/IgniteThrottlingUnitTest.java | 15 +++++++++++ .../persistence/pagemem/PageMemoryImplTest.java | 10 ++++++++ .../pagemem/PagesWriteThrottleSmokeTest.java | 29 +++++++++++++++++++--- 7 files changed, 77 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java index 4335f70..c30846c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java @@ -97,6 +97,9 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { /** */ private final HitRateMetric pageReplaceAge; + /** Total throttling threads time in milliseconds. */ + private final LongAdderMetric totalThrottlingTime; + /** */ private final DataRegionConfiguration memPlcCfg; @@ -180,6 +183,10 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { dataRegionMetricsProvider::emptyDataPages, "Calculates empty data pages count for region. It counts only totally free pages that can be reused " + "(e. g. pages that are contained in reuse bucket of free list)."); + + totalThrottlingTime = mreg.longAdderMetric("TotalThrottlingTime", + "Total throttling threads time in milliseconds. The Ignite throttles threads that generate " + + "dirty pages during the ongoing checkpoint."); } /** {@inheritDoc} */ @@ -561,6 +568,12 @@ public class DataRegionMetricsImpl implements DataRegionMetrics { pageReplaceAge.reset(); } + /** @param time Time to add to {@code totalThrottlingTime} metric in milliseconds. */ + public void addThrottlingTime(long time) { + if (metricsEnabled) + totalThrottlingTime.add(time); + } + /** * Updates allocation rate metric. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 964d618..7adf1c5 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -283,7 +283,7 @@ public class PageMemoryImpl implements PageMemoryEx { private long[] sizes; /** Memory metrics to track dirty pages count and page replace rate. */ - private DataRegionMetricsImpl memMetrics; + private final DataRegionMetricsImpl memMetrics; /** * {@code False} if memory was not started or already stopped and is not supposed for any usage. @@ -316,6 +316,7 @@ public class PageMemoryImpl implements PageMemoryEx { ) { assert ctx != null; assert pageSize > 0; + assert memMetrics != null; log = ctx.logger(PageMemoryImpl.class); @@ -1881,6 +1882,11 @@ public class PageMemoryImpl implements PageMemoryEx { return U.safeAbs(hash) % segments; } + /** @return Data region metrics. */ + public DataRegionMetricsImpl metrics() { + return memMetrics; + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java index 9e4975c..f33b86d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java @@ -225,6 +225,8 @@ public class PagesWriteSpeedBasedThrottle implements PagesWriteThrottlePolicy { doPark(throttleParkTimeNs); } + pageMemory.metrics().addThrottlingTime(U.nanosToMillis(System.nanoTime() - curNanoTime)); + speedMarkAndAvgParkTime.addMeasurementForAverageCalculation(throttleParkTimeNs); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java index b2a0ec0..cc0f496 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java @@ -132,6 +132,8 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy { + " for timeout(ms)=" + (throttleParkTimeNs / 1_000_000)); } + long startTime = U.currentTimeMillis(); + if (isPageInCheckpoint) { cpBufThrottledThreads.put(curThread.getId(), curThread); @@ -149,6 +151,8 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy { } else LockSupport.parkNanos(throttleParkTimeNs); + + pageMemory.metrics().addThrottlingTime(U.currentTimeMillis() - startTime); } else { int oldCntr = cntr.getAndSet(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java index 4dad75c..0e86f2b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java @@ -24,16 +24,24 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.LockSupport; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker; import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier; +import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; +import org.apache.ignite.internal.processors.metric.GridMetricManager; import org.apache.ignite.logger.NullLogger; +import org.apache.ignite.spi.metric.noop.NoopMetricExporterSpi; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.GridTestKernalContext; +import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; import static java.lang.Thread.State.TIMED_WAITING; +import static org.apache.ignite.internal.processors.database.DataRegionMetricsSelfTest.NO_OP_METRICS; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -62,6 +70,13 @@ public class IgniteThrottlingUnitTest { { when(pageMemory2g.totalPages()).thenReturn((2L * 1024 * 1024 * 1024) / 4096); + + IgniteConfiguration cfg = new IgniteConfiguration().setMetricExporterSpi(new NoopMetricExporterSpi()); + + DataRegionMetricsImpl metrics = new DataRegionMetricsImpl(new DataRegionConfiguration(), + new GridMetricManager(new GridTestKernalContext(new GridTestLog4jLogger(), cfg)), NO_OP_METRICS); + + when(pageMemory2g.metrics()).thenReturn(metrics); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java index d061635..2fc8cbd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java @@ -50,11 +50,13 @@ import org.apache.ignite.internal.processors.cache.persistence.PageStoreWriter; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.failure.FailureProcessor; import org.apache.ignite.internal.processors.metric.GridMetricManager; +import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric; import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor; import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor; import org.apache.ignite.internal.util.GridMultiCollectionWrapper; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.lang.GridInClosure3X; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi; import org.apache.ignite.spi.metric.noop.NoopMetricExporterSpi; @@ -420,6 +422,12 @@ public class PageMemoryImplTest extends GridCommonAbstractTest { } memory.finishCheckpoint(); + + LongAdderMetric totalThrottlingTime = U.field(memory.metrics(), "totalThrottlingTime"); + + assertNotNull(totalThrottlingTime); + + assertTrue(totalThrottlingTime.value() > 0); } /** @@ -549,6 +557,8 @@ public class PageMemoryImplTest extends GridCommonAbstractTest { noThrottle ); + mem.metrics().enableMetrics(); + mem.start(); return mem; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java index 4f442e1..796db43 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java @@ -25,15 +25,14 @@ import java.nio.file.OpenOption; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; @@ -41,13 +40,18 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.metric.MetricRegistry; import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; +import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; +import static org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl.DATAREGION_METRICS_PREFIX; +import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName; + /** * */ @@ -120,7 +124,7 @@ public class PagesWriteThrottleSmokeTest extends GridCommonAbstractTest { startGrids(2).active(true); try { - Ignite ig = ignite(0); + IgniteEx ig = ignite(0); final int keyCnt = 2_000_000; @@ -200,6 +204,10 @@ public class PagesWriteThrottleSmokeTest extends GridCommonAbstractTest { fail("Put rate degraded to zero for at least 10 seconds"); } + + LongAdderMetric totalThrottlingTime = totalThrottlingTime(ig); + + assertTrue(totalThrottlingTime.value() > 0); } finally { stopAllGrids(); @@ -207,6 +215,21 @@ public class PagesWriteThrottleSmokeTest extends GridCommonAbstractTest { } /** + * @param ignite Ignite instance. + * @return {@code totalThrottlingTime} metric for the default region. + */ + private LongAdderMetric totalThrottlingTime(IgniteEx ignite) { + MetricRegistry mreg = ignite.context().metric().registry(metricName(DATAREGION_METRICS_PREFIX, + ignite.configuration().getDataStorageConfiguration().getDefaultDataRegionConfiguration().getName())); + + LongAdderMetric totalThrottlingTime = mreg.findMetric("TotalThrottlingTime"); + + assertNotNull(totalThrottlingTime); + + return totalThrottlingTime; + } + + /** * */ private static class TestValue implements Serializable {