This is an automated email from the ASF dual-hosted git repository.
namelchev pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 15abfae9e05 IGNITE-17587 Fixed the "io.datastorage.StorageSize" metric
in case of multiple persistence dataregions (#10225)
15abfae9e05 is described below
commit 15abfae9e054fdd606901378c412881543eb46df
Author: Nikita Amelchev <[email protected]>
AuthorDate: Wed Aug 31 12:16:22 2022 +0300
IGNITE-17587 Fixed the "io.datastorage.StorageSize" metric in case of
multiple persistence dataregions (#10225)
---
.../cache/persistence/DataStorageMetricsImpl.java | 22 +++------
.../GridCacheDatabaseSharedManager.java | 18 ++++++--
.../cache/persistence/GridCacheOffheapManager.java | 45 +-----------------
.../cache/persistence/checkpoint/Checkpointer.java | 7 ++-
.../persistence/file/FilePageStoreManager.java | 2 +-
.../IgniteDataStorageMetricsSelfTest.java | 54 ++++++++++++++++++----
6 files changed, 74 insertions(+), 74 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
index 73c8dc98203..3f6f1ab7e23 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
@@ -739,6 +739,8 @@ public class DataStorageMetricsImpl implements
DataStorageMetricsMXBean {
* @param totalPages Total number of all pages in checkpoint.
* @param dataPages Total number of data pages in checkpoint.
* @param cowPages Total number of COW-ed pages in checkpoint.
+ * @param storageSize Storage space allocated, in bytes.
+ * @param sparseStorageSize Storage space allocated adjusted for possible
sparsity, in bytes.
*/
public void onCheckpoint(
long beforeLockDuration,
@@ -755,7 +757,9 @@ public class DataStorageMetricsImpl implements
DataStorageMetricsMXBean {
long start,
long totalPages,
long dataPages,
- long cowPages
+ long cowPages,
+ long storageSize,
+ long sparseStorageSize
) {
if (metricsEnabled) {
lastCpBeforeLockDuration.value(beforeLockDuration);
@@ -773,6 +777,8 @@ public class DataStorageMetricsImpl implements
DataStorageMetricsMXBean {
lastCpTotalPages.value(totalPages);
lastCpDataPages.value(dataPages);
lastCpCowPages.value(cowPages);
+ this.storageSize.value(storageSize);
+ this.sparseStorageSize.value(sparseStorageSize);
totalCheckpointTime.add(duration);
@@ -790,20 +796,6 @@ public class DataStorageMetricsImpl implements
DataStorageMetricsMXBean {
}
}
- /**
- * @param sparseStorageSize Sparse storage size.
- * @param storageSize Storage size.
- */
- public void onStorageSizeChanged(
- long storageSize,
- long sparseStorageSize
- ) {
- if (metricsEnabled) {
- this.storageSize.value(storageSize);
- this.sparseStorageSize.value(sparseStorageSize);
- }
- }
-
/**
* Callback on logging a record to a WAL.
*
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 2ca91ab167b..445e1062d8a 100755
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -2053,6 +2053,19 @@ public class GridCacheDatabaseSharedManager extends
IgniteCacheDatabaseSharedMan
return res;
}
+ /**
+ * @param f Consumer.
+ * @return Accumulated result for all page stores.
+ */
+ public long forAllPageStores(ToLongFunction<PageStore> f) {
+ long res = 0;
+
+ for (CacheGroupContext gctx : cacheGroupContexts())
+ res += forGroupPageStores(gctx, f);
+
+ return res;
+ }
+
/**
* Calculates tail pointer for WAL at the end of logical recovery.
*
@@ -3169,11 +3182,6 @@ public class GridCacheDatabaseSharedManager extends
IgniteCacheDatabaseSharedMan
return new DataStorageMetricsSnapshot(dsMetrics);
}
- /** {@inheritDoc} */
- @Override public DataStorageMetricsImpl dataStorageMetricsImpl() {
- return dsMetrics;
- }
-
/** {@inheritDoc} */
@Override public MetaStorage metaStorage() {
return metaStorage;
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 4668f482620..45bc6c55c8e 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -33,7 +33,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.ToLongFunction;
import java.util.stream.Stream;
import javax.cache.processor.EntryProcessor;
import org.apache.ignite.IgniteCheckedException;
@@ -50,7 +49,6 @@ import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageSupport;
import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
-import org.apache.ignite.internal.pagemem.store.PageStore;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
@@ -82,7 +80,6 @@ import
org.apache.ignite.internal.processors.cache.distributed.dht.topology.Grid
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion;
import
org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener;
-import
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
import
org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList;
import
org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow;
@@ -126,6 +123,7 @@ import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.Nullable;
+
import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
import static
org.apache.ignite.internal.processors.cache.GridCacheTtlManager.DFLT_UNWIND_THROTTLING_TIMEOUT;
import static
org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
@@ -279,47 +277,6 @@ public class GridCacheOffheapManager extends
IgniteCacheOffheapManagerImpl imple
syncMetadata(ctx, ctx.executor(), false);
}
- /** {@inheritDoc} */
- @Override public void afterCheckpointEnd(Context ctx) throws
IgniteCheckedException {
- persStoreMetrics.onStorageSizeChanged(
- forAllPageStores(PageStore::size),
- forAllPageStores(PageStore::getSparseSize)
- );
- }
-
- /**
- * @param f Consumer.
- * @return Accumulated result for all page stores.
- */
- private long forAllPageStores(ToLongFunction<PageStore> f) {
- return forGroupPageStores(grp, f);
- }
-
- /**
- * @param gctx Group context.
- * @param f Consumer.
- * @return Accumulated result for all page stores.
- */
- private long forGroupPageStores(CacheGroupContext gctx,
ToLongFunction<PageStore> f) {
- int groupId = gctx.groupId();
-
- long res = 0;
-
- try {
- Collection<PageStore> stores =
((FilePageStoreManager)ctx.cache().context().pageStore()).getStores(groupId);
-
- if (stores != null) {
- for (PageStore store : stores)
- res += f.applyAsLong(store);
- }
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
-
- return res;
- }
-
/**
* Syncs and saves meta-information of all data structures to page memory.
*
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/Checkpointer.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/Checkpointer.java
index 457c668964f..68c993fc5e2 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/Checkpointer.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/Checkpointer.java
@@ -44,6 +44,7 @@ import
org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointState;
import
org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl;
+import
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import
org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager;
import
org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
import
org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
@@ -622,6 +623,8 @@ public class Checkpointer extends GridWorker {
}
if (persStoreMetrics.metricsEnabled()) {
+ GridCacheDatabaseSharedManager dbMgr =
(GridCacheDatabaseSharedManager)cacheProcessor.context().database();
+
persStoreMetrics.onCheckpoint(
tracker.beforeLockDuration(),
tracker.lockWaitDuration(),
@@ -637,7 +640,9 @@ public class Checkpointer extends GridWorker {
tracker.checkpointStartTime(),
chp.pagesSize,
tracker.dataPagesWritten(),
- tracker.cowPagesWritten()
+ tracker.cowPagesWritten(),
+ dbMgr.forAllPageStores(PageStore::size),
+ dbMgr.forAllPageStores(PageStore::getSparseSize)
);
}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 02c1304a320..cf6f23515ca 100755
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -1084,7 +1084,7 @@ public class FilePageStoreManager extends
GridCacheSharedManagerAdapter implemen
}
/**
- * Return cache store holedr.
+ * Return cache store holder.
*
* @param grpId Cache group ID.
* @return Cache store holder.
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
index f767fdd634d..160be0a9bd8 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
@@ -20,11 +20,13 @@ package
org.apache.ignite.internal.processors.cache.persistence;
import java.io.File;
import java.io.Serializable;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Objects;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
+import java.util.function.ToLongFunction;
import java.util.function.UnaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -54,11 +56,13 @@ import
org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric;
import org.apache.ignite.internal.processors.metric.impl.LongGauge;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.PAX;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
import org.apache.ignite.spi.metric.HistogramMetric;
+import org.apache.ignite.spi.metric.LongMetric;
import org.apache.ignite.testframework.ListeningTestLogger;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
@@ -68,8 +72,10 @@ import static
org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static
org.apache.ignite.internal.processors.cache.CacheGroupMetricsImpl.CACHE_GROUP_METRICS_PREFIX;
import static
org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl.DATASTORAGE_METRIC_PREFIX;
import static
org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE;
+import static
org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
import static org.apache.ignite.testframework.GridTestUtils.setFieldValue;
import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
@@ -80,9 +86,18 @@ public class IgniteDataStorageMetricsSelfTest extends
GridCommonAbstractTest {
/** */
private static final String GROUP1 = "grp1";
+ /** */
+ private static final String GROUP2 = "grp2";
+
/** */
private static final String NO_PERSISTENCE = "no-persistence";
+ /** */
+ private static final String PERSISTENCE_REGION_1 = "persistence-1";
+
+ /** */
+ private static final String PERSISTENCE_REGION_2 = "persistence-2";
+
/** */
private final ListeningTestLogger listeningLog = new
ListeningTestLogger(log);
@@ -111,12 +126,18 @@ public class IgniteDataStorageMetricsSelfTest extends
GridCommonAbstractTest {
.setMaxSize(maxRegionSize)
.setPersistenceEnabled(true)
.setMetricsEnabled(true)
- .setName("dflt-plc"))
- .setDataRegionConfigurations(new DataRegionConfiguration()
- .setMaxSize(maxRegionSize)
- .setPersistenceEnabled(false)
- .setMetricsEnabled(true)
- .setName(NO_PERSISTENCE))
+ .setName(PERSISTENCE_REGION_1))
+ .setDataRegionConfigurations(
+ new DataRegionConfiguration()
+ .setMaxSize(maxRegionSize)
+ .setPersistenceEnabled(true)
+ .setMetricsEnabled(true)
+ .setName(PERSISTENCE_REGION_2),
+ new DataRegionConfiguration()
+ .setMaxSize(maxRegionSize)
+ .setPersistenceEnabled(false)
+ .setMetricsEnabled(true)
+ .setName(NO_PERSISTENCE))
.setWalMode(WALMode.LOG_ONLY)
.setMetricsEnabled(true);
@@ -126,6 +147,7 @@ public class IgniteDataStorageMetricsSelfTest extends
GridCommonAbstractTest {
cfg.setCacheConfiguration(
cacheConfiguration(GROUP1, "cache", PARTITIONED, ATOMIC, 1, null),
+ cacheConfiguration(GROUP2, "cache2", PARTITIONED, ATOMIC, 1,
PERSISTENCE_REGION_2),
cacheConfiguration(null, "cache-np", PARTITIONED, ATOMIC, 1,
NO_PERSISTENCE));
cfg.setGridLogger(listeningLog);
@@ -186,16 +208,19 @@ public class IgniteDataStorageMetricsSelfTest extends
GridCommonAbstractTest {
try {
IgniteCache<Object, Object> cache = ig.cache("cache");
+ IgniteCache<Object, Object> cache2 = ig.cache("cache2");
- for (int i = 0; i < 10; i++)
+ for (int i = 0; i < 10; i++) {
cache.put(i, new Person("first-" + i, "last-" + i));
+ cache2.put(i, new Person("first-" + i, "last-" + i));
+ }
IgniteCache<Object, Object> cacheNp = ig.cache("cache-np");
for (int i = 0; i < 10; i++)
cacheNp.put(i, new Person("first-" + i, "last-" + i));
- DataRegionMetrics memMetrics = ig.dataRegionMetrics("dflt-plc");
+ DataRegionMetrics memMetrics =
ig.dataRegionMetrics(PERSISTENCE_REGION_1);
assertNotNull(memMetrics);
assertTrue(memMetrics.getDirtyPages() > 0);
@@ -219,6 +244,19 @@ public class IgniteDataStorageMetricsSelfTest extends
GridCommonAbstractTest {
pMetrics.getLastCheckpointDataPagesNumber() != 0;
}
}, 10_000));
+
+ Collection<MetricRegistry> grpRegs =
F.viewReadOnly(ig.context().cache().cacheGroups(),
+ ctx ->
ig.context().metric().registry(metricName(CACHE_GROUP_METRICS_PREFIX,
ctx.cacheOrGroupName())));
+
+ ToLongFunction<String> sumByGroups = metric -> grpRegs.stream()
+ .map(grpReg ->
grpReg.<LongMetric>findMetric(metric).value()).mapToLong(v -> v)
+ .sum();
+
+ long storageSize =
dsMetricRegistry(ig).<LongMetric>findMetric("StorageSize").value();
+ long sparseStorageSize =
dsMetricRegistry(ig).<LongMetric>findMetric("SparseStorageSize").value();
+
+ assertEquals(sumByGroups.applyAsLong("StorageSize"), storageSize);
+ assertEquals(sumByGroups.applyAsLong("SparseStorageSize"),
sparseStorageSize);
}
finally {
stopAllGrids();
