IGNITE-4536 - Update metrics for memory policies.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3f90a9c0 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3f90a9c0 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3f90a9c0 Branch: refs/heads/ignite-3477-master Commit: 3f90a9c0795c2ba61942f2f379e379e8bcd36d08 Parents: 6b62a20 Author: Sergey Chugunov <[email protected]> Authored: Fri Apr 7 15:39:22 2017 +0300 Committer: Alexey Goncharuk <[email protected]> Committed: Fri Apr 7 15:39:22 2017 +0300 ---------------------------------------------------------------------- .../src/main/java/org/apache/ignite/Ignite.java | 8 + .../java/org/apache/ignite/MemoryMetrics.java | 95 ++++++ .../org/apache/ignite/cache/CacheMetrics.java | 70 ----- .../configuration/MemoryConfiguration.java | 4 +- .../apache/ignite/internal/IgniteKernal.java | 7 +- .../pagemem/impl/PageMemoryNoStoreImpl.java | 11 +- .../cache/CacheClusterMetricsMXBeanImpl.java | 50 --- .../cache/CacheLocalMetricsMXBeanImpl.java | 50 --- .../processors/cache/CacheMetricsImpl.java | 125 +------- .../processors/cache/CacheMetricsSnapshot.java | 120 ------- .../processors/cache/GridCacheProcessor.java | 3 + .../IgniteCacheDatabaseSharedManager.java | 98 +++++- .../cache/database/IgniteMemoryPoolMetrics.java | 33 -- .../cache/database/MemoryMetricsImpl.java | 314 +++++++++++++++++++ .../processors/cache/database/MemoryPolicy.java | 18 +- .../cache/database/freelist/FreeListImpl.java | 35 +++ .../platform/cache/PlatformCache.java | 10 - .../visor/node/VisorMemoryConfiguration.java | 1 - .../ignite/mxbean/CacheMetricsMXBean.java | 40 --- .../ignite/mxbean/MemoryMetricsMXBean.java | 89 ++++++ .../pagemem/impl/PageMemoryNoLoadSelfTest.java | 3 +- .../cache/CacheConfigurationLeakTest.java | 3 +- ...LocalCacheOffHeapAndSwapMetricsSelfTest.java | 182 +---------- .../processors/database/BPlusTreeSelfTest.java | 3 +- .../database/FreeListImplSelfTest.java | 5 +- .../database/MemoryMetricsSelfTest.java | 313 ++++++++++++++++++ .../database/MetadataStorageSelfTest.java | 3 +- .../processors/igfs/IgfsIgniteMock.java | 8 + .../platform/PlatformCacheWriteMetricsTask.java | 126 +++----- .../ignite/testframework/junits/IgniteMock.java | 6 + .../junits/multijvm/IgniteProcessProxy.java | 6 + .../ignite/testsuites/IgniteBasicTestSuite.java | 2 + .../h2/database/InlineIndexHelperTest.java | 4 + .../cpp/core-test/config/cache-test.xml | 2 +- .../platforms/cpp/core-test/src/cache_test.cpp | 2 +- .../org/apache/ignite/IgniteSpringBean.java | 7 + .../ant/beautifier/GridJavadocAntTask.java | 18 +- 37 files changed, 1092 insertions(+), 782 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/Ignite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java index c97186e..aba224f 100644 --- a/modules/core/src/main/java/org/apache/ignite/Ignite.java +++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java @@ -615,4 +615,12 @@ public interface Ignite extends AutoCloseable { * Clears partition's lost state and moves caches to a normal mode. */ public void resetLostPartitions(Collection<String> cacheNames); + + + /** + * Returns collection {@link MemoryMetrics} objects providing information about memory usage in current Ignite instance. + * + * @return Collection of {@link MemoryMetrics} + */ + public Collection<MemoryMetrics> memoryMetrics(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/MemoryMetrics.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/MemoryMetrics.java b/modules/core/src/main/java/org/apache/ignite/MemoryMetrics.java new file mode 100644 index 0000000..b4097d1 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/MemoryMetrics.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite; + +/** + * Interface provides methods to access metrics of memory usage on local instance of Ignite. + */ +public interface MemoryMetrics { + /** + * @return Memory policy name. + */ + public String getName(); + + /** + * @return Returns size (in MBytes) of MemoryPolicy observed by this MemoryMetrics MBean. + */ + public int getSize(); + + /** + * @return Path of memory-mapped file used to swap PageMemory pages to disk. + */ + public String getSwapFilePath(); + + /** + * Enables collecting memory metrics. + */ + public void enableMetrics(); + + /** + * Disables collecting memory metrics. + */ + public void disableMetrics(); + + /** + * @return Total number of allocated pages. + */ + public long getTotalAllocatedPages(); + + /** + * @return Number of allocated pages per second within PageMemory. + */ + public float getAllocationRate(); + + /** + * @return Number of evicted pages per second within PageMemory. + */ + public float getEvictionRate(); + + /** + * Large entities bigger than page are split into fragments so each fragment can fit into a page. + * + * @return Percentage of pages fully occupied by large entities. + */ + public float getLargeEntriesPagesPercentage(); + + /** + * @return Free space to overall size ratio across all pages in FreeList. + */ + public float getPagesFillFactor(); + + /** + * Sets interval of time (in seconds) to monitor allocation rate. + * + * E.g. after setting rateTimeInterval to 60 seconds subsequent calls to {@link #getAllocationRate()} + * will return average allocation rate (pages per second) for the last minute. + * + * @param rateTimeInterval Time interval used to calculate allocation/eviction rate. + */ + public void rateTimeInterval(int rateTimeInterval); + + /** + * Sets number of subintervals the whole rateTimeInterval will be split into to calculate allocation rate, + * 5 by default. + * Setting it to bigger number allows more precise calculation and smaller drops of allocationRate metric + * when next subinterval has to be recycled but introduces bigger calculation overhead. + * + * @param subInts Number of subintervals. + */ + public void subIntervals(int subInts); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java index 669c427..c0eb98e 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java @@ -146,13 +146,6 @@ public interface CacheMetrics { public String name(); /** - * Gets number of entries that was swapped to disk. - * - * @return Number of entries that was swapped to disk. - */ - public long getOverflowSize(); - - /** * The total number of get requests to the off-heap memory. * * @return The number of gets. @@ -244,69 +237,6 @@ public interface CacheMetrics { public long getOffHeapMaxSize(); /** - * The total number of get requests to the swap. - * - * @return The number of gets. - */ - public long getSwapGets(); - - /** - * The total number of put requests to the swap. - * - * @return The number of puts. - */ - public long getSwapPuts(); - - /** - * The total number of removals from the swap. - * - * @return The number of removals. - */ - public long getSwapRemovals(); - - /** - * The number of get requests that were satisfied by the swap. - * - * @return The swap hits number. - */ - public long getSwapHits(); - - /** - * A miss is a get request that is not satisfied by swap. - * - * @return The swap misses number. - */ - public long getSwapMisses(); - - /** - * Gets number of entries stored in swap. - * - * @return Number of entries stored in swap. - */ - public long getSwapEntriesCount(); - - /** - * Gets size of swap. - * - * @return Size of swap. - */ - public long getSwapSize(); - - /** - * Gets the percentage of hits on swap. - * - * @return The percentage of hits on swap. - */ - public float getSwapHitPercentage(); - - /** - * Gets the percentage of misses on swap. - * - * @return The percentage of misses on swap. - */ - public float getSwapMissPercentage(); - - /** * Gets number of non-{@code null} values in the cache. * * @return Number of non-{@code null} values in the cache. http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java index 0f16ada..3ef876b 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java @@ -112,7 +112,7 @@ public class MemoryConfiguration implements Serializable { } /** - * @return Array of MemoryPolicyConfiguration objects. + * @return array of MemoryPolicyConfiguration objects. */ public MemoryPolicyConfiguration[] getMemoryPolicies() { return memPlcs; @@ -151,7 +151,7 @@ public class MemoryConfiguration implements Serializable { */ public MemoryConfiguration setConcurrencyLevel(int concLvl) { this.concLvl = concLvl; - + return this; } http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 3ab7f8e..9c661df 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -73,6 +73,7 @@ import org.apache.ignite.IgniteSet; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.IgniteTransactions; import org.apache.ignite.Ignition; +import org.apache.ignite.MemoryMetrics; import org.apache.ignite.cache.affinity.Affinity; import org.apache.ignite.cluster.ClusterGroup; import org.apache.ignite.cluster.ClusterMetrics; @@ -99,7 +100,6 @@ import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; import org.apache.ignite.internal.managers.failover.GridFailoverManager; import org.apache.ignite.internal.managers.indexing.GridIndexingManager; import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager; -import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.processors.GridProcessor; import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; @@ -3371,6 +3371,11 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { } /** {@inheritDoc} */ + @Override public Collection<MemoryMetrics> memoryMetrics() { + return ctx.cache().context().database().memoryMetrics(); + } + + /** {@inheritDoc} */ @Nullable @Override public IgniteAtomicSequence atomicSequence(String name, long initVal, boolean create) { guard(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java index 0b65ce6..f24113c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java @@ -31,6 +31,7 @@ import org.apache.ignite.internal.mem.OutOfMemoryException; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl; import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.OffheapReadWriteLock; @@ -104,6 +105,9 @@ public class PageMemoryNoStoreImpl implements PageMemory { /** Direct memory allocator. */ private final DirectMemoryProvider directMemoryProvider; + /** Object to collect memory usage metrics. */ + private final MemoryMetricsImpl memMetrics; + /** Segments array. */ private Segment[] segments; @@ -133,6 +137,7 @@ public class PageMemoryNoStoreImpl implements PageMemory { * @param directMemoryProvider Memory allocator to use. * @param sharedCtx Cache shared context. * @param pageSize Page size. + * @param memMetrics Memory Metrics. * @param trackAcquiredPages If {@code true} tracks number of allocated pages (for tests purpose only). */ public PageMemoryNoStoreImpl( @@ -140,6 +145,7 @@ public class PageMemoryNoStoreImpl implements PageMemory { DirectMemoryProvider directMemoryProvider, GridCacheSharedContext<?, ?> sharedCtx, int pageSize, + MemoryMetricsImpl memMetrics, boolean trackAcquiredPages ) { assert log != null || sharedCtx != null; @@ -148,6 +154,7 @@ public class PageMemoryNoStoreImpl implements PageMemory { this.log = sharedCtx != null ? sharedCtx.logger(PageMemoryNoStoreImpl.class) : log; this.directMemoryProvider = directMemoryProvider; this.trackAcquiredPages = trackAcquiredPages; + this.memMetrics = memMetrics; sysPageSize = pageSize + PAGE_OVERHEAD; @@ -210,6 +217,8 @@ public class PageMemoryNoStoreImpl implements PageMemory { /** {@inheritDoc} */ @Override public long allocatePage(int cacheId, int partId, byte flags) { + memMetrics.incrementTotalAllocatedPages(); + long relPtr = INVALID_REL_PTR; long absPtr = 0; @@ -607,9 +616,9 @@ public class PageMemoryNoStoreImpl implements PageMemory { } /** + * @param tag Tag to initialize RW lock. * @return Relative pointer of the allocated page. * @throws GridOffHeapOutOfMemoryException If failed to allocate. - * @param tag Tag to initialize RW lock. */ private long allocateFreePage(int tag) throws GridOffHeapOutOfMemoryException { long limit = region.address() + region.size(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java index 519d07b..c633fde 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java @@ -44,11 +44,6 @@ class CacheClusterMetricsMXBeanImpl implements CacheMetricsMXBean { } /** {@inheritDoc} */ - @Override public long getOverflowSize() { - return cache.clusterMetrics().getOverflowSize(); - } - - /** {@inheritDoc} */ @Override public long getOffHeapGets() { return cache.clusterMetrics().getOffHeapGets(); } @@ -114,51 +109,6 @@ class CacheClusterMetricsMXBeanImpl implements CacheMetricsMXBean { } /** {@inheritDoc} */ - @Override public long getSwapGets() { - return cache.clusterMetrics().getSwapGets(); - } - - /** {@inheritDoc} */ - @Override public long getSwapPuts() { - return cache.clusterMetrics().getSwapPuts(); - } - - /** {@inheritDoc} */ - @Override public long getSwapRemovals() { - return cache.clusterMetrics().getSwapRemovals(); - } - - /** {@inheritDoc} */ - @Override public long getSwapHits() { - return cache.clusterMetrics().getSwapHits(); - } - - /** {@inheritDoc} */ - @Override public long getSwapMisses() { - return cache.clusterMetrics().getSwapMisses(); - } - - /** {@inheritDoc} */ - @Override public float getSwapHitPercentage() { - return cache.clusterMetrics().getSwapHitPercentage(); - } - - /** {@inheritDoc} */ - @Override public float getSwapMissPercentage() { - return cache.clusterMetrics().getSwapMissPercentage(); - } - - /** {@inheritDoc} */ - @Override public long getSwapEntriesCount() { - return cache.clusterMetrics().getSwapEntriesCount(); - } - - /** {@inheritDoc} */ - @Override public long getSwapSize() { - return cache.clusterMetrics().getSwapSize(); - } - - /** {@inheritDoc} */ @Override public int getSize() { return cache.clusterMetrics().getSize(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java index 5284ca0..cdab58a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java @@ -44,11 +44,6 @@ class CacheLocalMetricsMXBeanImpl implements CacheMetricsMXBean { } /** {@inheritDoc} */ - @Override public long getOverflowSize() { - return cache.metrics0().getOverflowSize(); - } - - /** {@inheritDoc} */ @Override public long getOffHeapGets() { return cache.metrics0().getOffHeapGets(); } @@ -114,51 +109,6 @@ class CacheLocalMetricsMXBeanImpl implements CacheMetricsMXBean { } /** {@inheritDoc} */ - @Override public long getSwapGets() { - return cache.metrics0().getSwapGets(); - } - - /** {@inheritDoc} */ - @Override public long getSwapPuts() { - return cache.metrics0().getSwapPuts(); - } - - /** {@inheritDoc} */ - @Override public long getSwapRemovals() { - return cache.metrics0().getSwapRemovals(); - } - - /** {@inheritDoc} */ - @Override public long getSwapHits() { - return cache.metrics0().getSwapHits(); - } - - /** {@inheritDoc} */ - @Override public long getSwapMisses() { - return cache.metrics0().getSwapMisses(); - } - - /** {@inheritDoc} */ - @Override public float getSwapHitPercentage() { - return cache.metrics0().getSwapHitPercentage(); - } - - /** {@inheritDoc} */ - @Override public float getSwapMissPercentage() { - return cache.metrics0().getSwapMissPercentage(); - } - - /** {@inheritDoc} */ - @Override public long getSwapEntriesCount() { - return cache.metrics0().getSwapEntriesCount(); - } - - /** {@inheritDoc} */ - @Override public long getSwapSize() { - return cache.metrics0().getSwapSize(); - } - - /** {@inheritDoc} */ @Override public int getSize() { return cache.metrics0().getSize(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java index 1613af7..eb63a60 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java @@ -149,11 +149,6 @@ public class CacheMetricsImpl implements CacheMetrics { } /** {@inheritDoc} */ - @Override public long getOverflowSize() { - return 0; - } - - /** {@inheritDoc} */ @Override public long getOffHeapGets() { return offHeapGets.get(); } @@ -245,63 +240,6 @@ public class CacheMetricsImpl implements CacheMetrics { } /** {@inheritDoc} */ - @Override public long getSwapGets() { - return swapGets.get(); - } - - /** {@inheritDoc} */ - @Override public long getSwapPuts() { - return swapPuts.get(); - } - - /** {@inheritDoc} */ - @Override public long getSwapRemovals() { - return swapRemoves.get(); - } - - /** {@inheritDoc} */ - @Override public long getSwapHits() { - return swapHits.get(); - } - - /** {@inheritDoc} */ - @Override public long getSwapMisses() { - return swapMisses.get(); - } - - /** {@inheritDoc} */ - @Override public long getSwapEntriesCount() { - return 0; - } - - /** {@inheritDoc} */ - @Override public long getSwapSize() { - return 0; - } - - /** {@inheritDoc} */ - @Override public float getSwapHitPercentage() { - long hits0 = swapHits.get(); - long gets0 = swapGets.get(); - - if (hits0 == 0) - return 0; - - return (float) hits0 / gets0 * 100.0f; - } - - /** {@inheritDoc} */ - @Override public float getSwapMissPercentage() { - long misses0 = swapMisses.get(); - long reads0 = swapGets.get(); - - if (misses0 == 0) - return 0; - - return (float) misses0 / reads0 * 100.0f; - } - - /** {@inheritDoc} */ @Override public int getSize() { GridCacheAdapter<?, ?> cache = cctx.cache(); @@ -782,6 +720,14 @@ public class CacheMetricsImpl implements CacheMetrics { return ccfg != null && ccfg.isManagementEnabled(); } + public long getTotalAllocatedPages() { + return 0; + } + + public long getTotalEvictedPages() { + return 0; + } + /** * Off-heap read callback. * @@ -829,61 +775,6 @@ public class CacheMetricsImpl implements CacheMetrics { delegate.onOffHeapEvict(); } - /** - * Swap read callback. - * - * @param hit Hit or miss flag. - */ - public void onSwapRead(boolean hit) { - swapGets.incrementAndGet(); - - if (hit) - swapHits.incrementAndGet(); - else - swapMisses.incrementAndGet(); - - if (delegate != null) - delegate.onSwapRead(hit); - } - - /** - * Swap write callback. - */ - public void onSwapWrite() { - onSwapWrite(1); - } - - /** - * Swap write callback. - * - * @param cnt Amount of entries. - */ - public void onSwapWrite(int cnt) { - swapPuts.addAndGet(cnt); - - if (delegate != null) - delegate.onSwapWrite(cnt); - } - - /** - * Swap remove callback. - */ - public void onSwapRemove() { - onSwapRemove(1); - } - - /** - * Swap remove callback. - * - * @param cnt Amount of entries. - */ - public void onSwapRemove(int cnt) { - swapRemoves.addAndGet(cnt); - - if (delegate != null) - delegate.onSwapRemove(cnt); - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(CacheMetricsImpl.class, this); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java index 1971d5b..a8d7693 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java @@ -74,9 +74,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { /** Cache name */ private String cacheName; - /** Number of entries that was swapped to disk. */ - private long overflowSize; - /** Number of reads from off-heap. */ private long offHeapGets; @@ -110,27 +107,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { /** Off-heap memory maximum size*/ private long offHeapMaxSize; - /** Number of reads from swap. */ - private long swapGets; - - /** Number of writes to swap. */ - private long swapPuts; - - /** Number of removed entries from swap. */ - private long swapRemoves; - - /** Number of entries stored in swap. */ - private long swapEntriesCnt; - - /** Swap hits number. */ - private long swapHits; - - /** Swap misses number. */ - private long swapMisses; - - /** Swap size. */ - private long swapSize; - /** Number of non-{@code null} values in the cache. */ private int size; @@ -262,7 +238,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { rollbackAvgTimeNanos = m.getAverageTxRollbackTime(); cacheName = m.name(); - overflowSize = m.getOverflowSize(); offHeapGets = m.getOffHeapGets(); offHeapPuts = m.getOffHeapPuts(); @@ -276,14 +251,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { offHeapAllocatedSize = m.getOffHeapAllocatedSize(); offHeapMaxSize = m.getOffHeapMaxSize(); - swapGets = m.getSwapGets(); - swapPuts = m.getSwapPuts(); - swapRemoves = m.getSwapRemovals(); - swapHits = m.getSwapHits(); - swapMisses = m.getSwapMisses(); - swapEntriesCnt = m.getSwapEntriesCount(); - swapSize = m.getSwapSize(); - size = m.getSize(); keySize = m.getKeySize(); isEmpty = m.isEmpty(); @@ -365,11 +332,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { commitAvgTimeNanos += e.getAverageTxCommitTime(); rollbackAvgTimeNanos += e.getAverageTxRollbackTime(); - if (e.getOverflowSize() > -1) - overflowSize += e.getOverflowSize(); - else - overflowSize = -1; - offHeapGets += e.getOffHeapGets(); offHeapPuts += e.getOffHeapPuts(); offHeapRemoves += e.getOffHeapRemovals(); @@ -381,14 +343,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { offHeapBackupEntriesCnt += e.getOffHeapBackupEntriesCount(); offHeapAllocatedSize += e.getOffHeapAllocatedSize(); - swapGets += e.getSwapGets(); - swapPuts += e.getSwapPuts(); - swapRemoves += e.getSwapRemovals(); - swapHits += e.getSwapHits(); - swapMisses += e.getSwapMisses(); - swapEntriesCnt += e.getSwapEntriesCount(); - swapSize += e.getSwapSize(); - if (e.getDhtEvictQueueCurrentSize() > -1) dhtEvictQueueCurrSize += e.getDhtEvictQueueCurrentSize(); else @@ -551,11 +505,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { } /** {@inheritDoc} */ - @Override public long getOverflowSize() { - return overflowSize; - } - - /** {@inheritDoc} */ @Override public long getOffHeapGets() { return offHeapGets; } @@ -626,57 +575,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { } /** {@inheritDoc} */ - @Override public long getSwapGets() { - return swapGets; - } - - /** {@inheritDoc} */ - @Override public long getSwapPuts() { - return swapPuts; - } - - /** {@inheritDoc} */ - @Override public long getSwapRemovals() { - return swapRemoves; - } - - /** {@inheritDoc} */ - @Override public long getSwapHits() { - return swapHits; - } - - /** {@inheritDoc} */ - @Override public long getSwapMisses() { - return swapMisses; - } - - /** {@inheritDoc} */ - @Override public float getSwapHitPercentage() { - if (swapHits == 0 || swapGets == 0) - return 0; - - return (float) swapHits / swapGets * 100.0f; - } - - /** {@inheritDoc} */ - @Override public float getSwapMissPercentage() { - if (swapMisses == 0 || swapGets == 0) - return 0; - - return (float) swapMisses / swapGets * 100.0f; - } - - /** {@inheritDoc} */ - @Override public long getSwapEntriesCount() { - return swapEntriesCnt; - } - - /** {@inheritDoc} */ - @Override public long getSwapSize() { - return swapSize; - } - - /** {@inheritDoc} */ @Override public int getSize() { return size; } @@ -868,7 +766,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { out.writeFloat(commitAvgTimeNanos); out.writeFloat(rollbackAvgTimeNanos); - out.writeLong(overflowSize); out.writeLong(offHeapGets); out.writeLong(offHeapPuts); out.writeLong(offHeapRemoves); @@ -881,14 +778,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { out.writeLong(offHeapAllocatedSize); out.writeLong(offHeapMaxSize); - out.writeLong(swapGets); - out.writeLong(swapPuts); - out.writeLong(swapRemoves); - out.writeLong(swapHits); - out.writeLong(swapMisses); - out.writeLong(swapEntriesCnt); - out.writeLong(swapSize); - out.writeInt(dhtEvictQueueCurrSize); out.writeInt(txThreadMapSize); out.writeInt(txXidMapSize); @@ -926,7 +815,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { commitAvgTimeNanos = in.readFloat(); rollbackAvgTimeNanos = in.readFloat(); - overflowSize = in.readLong(); offHeapGets = in.readLong(); offHeapPuts = in.readLong(); offHeapRemoves = in.readLong(); @@ -939,14 +827,6 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable { offHeapAllocatedSize = in.readLong(); offHeapMaxSize = in.readLong(); - swapGets = in.readLong(); - swapPuts = in.readLong(); - swapRemoves = in.readLong(); - swapHits = in.readLong(); - swapMisses = in.readLong(); - swapEntriesCnt = in.readLong(); - swapSize = in.readLong(); - dhtEvictQueueCurrSize = in.readInt(); txThreadMapSize = in.readInt(); txXidMapSize = in.readInt(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 3ddd7ac..ab5db28 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -716,6 +716,9 @@ public class GridCacheProcessor extends GridProcessorAdapter { if (cacheType != CacheType.USER) cfg.setMemoryPolicyName(sharedCtx.database().systemMemoryPolicyName()); + if (cacheType != CacheType.USER) + cfg.setMemoryPolicyName(sharedCtx.database().systemMemoryPolicyName()); + boolean template = cfg.getName() != null && cfg.getName().endsWith("*"); DynamicCacheDescriptor desc = new DynamicCacheDescriptor(ctx, http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java index c1c2e67..300e935 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java @@ -23,9 +23,12 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.UUID; +import javax.management.JMException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.MemoryMetrics; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.MemoryConfiguration; import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.internal.GridKernalContext; @@ -44,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; +import org.apache.ignite.mxbean.MemoryMetricsMXBean; import org.jetbrains.annotations.Nullable; /** @@ -60,6 +64,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap protected Map<String, MemoryPolicy> memPlcMap; /** */ + protected Map<String, MemoryMetrics> memMetricsMap; + + /** */ protected MemoryPolicy dfltMemPlc; /** */ @@ -110,14 +117,19 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap for (MemoryPolicy memPlc : memPlcMap.values()) { MemoryPolicyConfiguration memPlcCfg = memPlc.config(); + MemoryMetricsImpl memMetrics = (MemoryMetricsImpl) memMetricsMap.get(memPlcCfg.getName()); + FreeListImpl freeList = new FreeListImpl(0, cctx.igniteInstanceName(), memPlc.pageMemory(), + memMetrics, null, cctx.wal(), 0L, true); + memMetrics.freeList(freeList); + freeListMap.put(memPlcCfg.getName(), freeList); } @@ -149,8 +161,18 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap //reserve place for default and system memory policies memPlcMap = U.newHashMap(2); - dfltMemPlc = createDefaultMemoryPolicy(dbCfg); + memMetricsMap = U.newHashMap(2); + + MemoryPolicyConfiguration dfltPlcCfg = dbCfg.createDefaultPolicyConfig(); + + MemoryMetricsImpl memMetrics = new MemoryMetricsImpl(dfltPlcCfg); + + registerMetricsMBean(memMetrics); + + dfltMemPlc = createDefaultMemoryPolicy(dbCfg, dfltPlcCfg, memMetrics); + memPlcMap.put(null, dfltMemPlc); + memMetricsMap.put(null, memMetrics); log.warning("No user-defined default MemoryPolicy found; system default of 1GB size will be used."); } @@ -160,23 +182,35 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap if (dfltMemPlcName == null) { //reserve additional place for default and system memory policies memPlcMap = U.newHashMap(memPlcsCfgs.length + 2); + memMetricsMap = U.newHashMap(memPlcsCfgs.length + 2); + + MemoryPolicyConfiguration dfltPlcCfg = dbCfg.createDefaultPolicyConfig(); + + MemoryMetricsImpl memMetrics = new MemoryMetricsImpl(dfltPlcCfg); - dfltMemPlc = createDefaultMemoryPolicy(dbCfg); + dfltMemPlc = createDefaultMemoryPolicy(dbCfg, dfltPlcCfg, memMetrics); memPlcMap.put(null, dfltMemPlc); + memMetricsMap.put(null, memMetrics); log.warning("No user-defined default MemoryPolicy found; system default of 1GB size will be used."); } - else + else { //reserve additional place for system memory policy only memPlcMap = U.newHashMap(memPlcsCfgs.length + 1); + memMetricsMap = U.newHashMap(memPlcsCfgs.length + 1);; + } for (MemoryPolicyConfiguration memPlcCfg : memPlcsCfgs) { - PageMemory pageMem = initMemory(dbCfg, memPlcCfg); + MemoryMetricsImpl memMetrics = new MemoryMetricsImpl(memPlcCfg); + + PageMemory pageMem = initMemory(dbCfg, memPlcCfg, memMetrics); - MemoryPolicy memPlc = new MemoryPolicy(pageMem, memPlcCfg); + MemoryPolicy memPlc = new MemoryPolicy(pageMem, memMetrics, memPlcCfg); memPlcMap.put(memPlcCfg.getName(), memPlc); + memMetricsMap.put(memPlcCfg.getName(), memMetrics); + if (memPlcCfg.getName().equals(dfltMemPlcName)) dfltMemPlc = memPlc; } @@ -184,18 +218,42 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap MemoryPolicyConfiguration sysPlcCfg = createSystemMemoryPolicy(dbCfg.getSystemCacheMemorySize()); - memPlcMap.put(SYSTEM_MEMORY_POLICY_NAME, new MemoryPolicy(initMemory(dbCfg, sysPlcCfg), sysPlcCfg)); + MemoryMetricsImpl sysMemMetrics = new MemoryMetricsImpl(sysPlcCfg); + + memPlcMap.put(SYSTEM_MEMORY_POLICY_NAME, new MemoryPolicy(initMemory(dbCfg, sysPlcCfg, sysMemMetrics), sysMemMetrics, sysPlcCfg)); + + memMetricsMap.put(SYSTEM_MEMORY_POLICY_NAME, sysMemMetrics); } /** - * @param dbCfg Database config. + * @param memMetrics Mem metrics. */ - private MemoryPolicy createDefaultMemoryPolicy(MemoryConfiguration dbCfg) { - MemoryPolicyConfiguration dfltPlc = dbCfg.createDefaultPolicyConfig(); + private void registerMetricsMBean(MemoryMetricsImpl memMetrics) { + IgniteConfiguration cfg = cctx.gridConfig(); - PageMemory pageMem = initMemory(dbCfg, dfltPlc); + try { + U.registerMBean( + cfg.getMBeanServer(), + cfg.getIgniteInstanceName(), + "MemoryMetrics", + memMetrics.getName(), + memMetrics, + MemoryMetricsMXBean.class); + } + catch (JMException e) { + log.warning("Failed to register MBean for MemoryMetrics with name: '" + memMetrics.getName() + "'"); + } + } - return new MemoryPolicy(pageMem, dfltPlc); + /** + * @param dbCfg Database configuration. + * @param memPlcCfg MemoryPolicy configuration. + * @param memMetrics MemoryMetrics instance. + */ + private MemoryPolicy createDefaultMemoryPolicy(MemoryConfiguration dbCfg, MemoryPolicyConfiguration memPlcCfg, MemoryMetricsImpl memMetrics) { + PageMemory pageMem = initMemory(dbCfg, memPlcCfg, memMetrics); + + return new MemoryPolicy(pageMem, memMetrics, memPlcCfg); } /** @@ -295,6 +353,13 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap } /** + * @return MemoryMetrics for all MemoryPolicies configured in Ignite instance. + */ + public Collection<MemoryMetrics> memoryMetrics() { + return memMetricsMap != null ? memMetricsMap.values() : null; + } + + /** * @param memPlcName Memory policy name. * @return {@link MemoryPolicy} instance associated with a given {@link MemoryPolicyConfiguration}. * @throws IgniteCheckedException in case of request for unknown MemoryPolicy. @@ -447,9 +512,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** * @param dbCfg memory configuration with common parameters. * @param plc memory policy with PageMemory specific parameters. + * @param memMetrics {@link MemoryMetrics} object to collect memory usage metrics. * @return Page memory instance. */ - private PageMemory initMemory(MemoryConfiguration dbCfg, MemoryPolicyConfiguration plc) { + private PageMemory initMemory(MemoryConfiguration dbCfg, MemoryPolicyConfiguration plc, MemoryMetricsImpl memMetrics) { long[] sizes = calculateFragmentSizes( dbCfg.getConcurrencyLevel(), plc.getSize()); @@ -464,7 +530,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap true, sizes); - return createPageMemory(memProvider, dbCfg.getPageSize()); + return createPageMemory(memProvider, dbCfg.getPageSize(), memMetrics); } /** @@ -512,9 +578,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap * * @param memProvider Memory provider. * @param pageSize Page size. + * @param memMetrics MemoryMetrics to collect memory usage metrics. + * @return PageMemory instance. */ - protected PageMemory createPageMemory(DirectMemoryProvider memProvider, int pageSize) { - return new PageMemoryNoStoreImpl(log, memProvider, cctx, pageSize, false); + protected PageMemory createPageMemory(DirectMemoryProvider memProvider, int pageSize, MemoryMetricsImpl memMetrics) { + return new PageMemoryNoStoreImpl(log, memProvider, cctx, pageSize, memMetrics, false); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteMemoryPoolMetrics.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteMemoryPoolMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteMemoryPoolMetrics.java deleted file mode 100644 index ca11232..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteMemoryPoolMetrics.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.database; - -/** - * - */ -public interface IgniteMemoryPoolMetrics { - /** - * @return Memory pool name. - */ - public String name(); - - /** - * @return Total number of pages available. - */ - public long totalPages(); -} http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java new file mode 100644 index 0000000..ed4cae0 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.database; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.configuration.MemoryPolicyConfiguration; +import org.apache.ignite.internal.processors.cache.database.freelist.FreeListImpl; +import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.mxbean.MemoryMetricsMXBean; +import org.jsr166.LongAdder8; + +/** + * + */ +public class MemoryMetricsImpl implements MemoryMetricsMXBean { + /** */ + private FreeListImpl freeList; + + /** */ + private final LongAdder8 totalAllocatedPages = new LongAdder8(); + + /** + * Counter for number of pages occupied by large entries (one entry is larger than one page). + */ + private final LongAdder8 largeEntriesPages = new LongAdder8(); + + /** */ + private volatile boolean metricsEnabled; + + /** */ + private volatile int subInts = 5; + + /** */ + private volatile LongAdder8[] allocRateCounters = new LongAdder8[subInts]; + + /** */ + private final AtomicInteger counterIdx = new AtomicInteger(0); + + /** */ + private final AtomicLong lastUpdTime = new AtomicLong(0); + + /** */ + private final MemoryPolicyConfiguration memPlcCfg; + + /** Time interval (in seconds) when allocations/evictions are counted to calculate rate. + * Default value is 60 seconds. */ + private volatile int rateTimeInterval = 60; + + /** + * @param memPlcCfg MemoryPolicyConfiguration. + */ + public MemoryMetricsImpl(MemoryPolicyConfiguration memPlcCfg) { + this.memPlcCfg = memPlcCfg; + + for (int i = 0; i < subInts; i++) + allocRateCounters[i] = new LongAdder8(); + } + + /** {@inheritDoc} */ + @Override public String getName() { + return U.maskName(memPlcCfg.getName()); + } + + /** {@inheritDoc} */ + @Override public int getSize() { + return (int) (memPlcCfg.getSize() / (1024 * 1024)); + } + + /** {@inheritDoc} */ + @Override public String getSwapFilePath() { + return memPlcCfg.getSwapFilePath(); + } + + /** {@inheritDoc} */ + @Override public long getTotalAllocatedPages() { + return metricsEnabled ? totalAllocatedPages.longValue() : 0; + } + + /** {@inheritDoc} */ + @Override public float getAllocationRate() { + if (!metricsEnabled) + return 0; + + float res = 0; + + for (int i = 0; i < subInts; i++) + res += allocRateCounters[i].floatValue(); + + return res / rateTimeInterval; + } + + /** {@inheritDoc} */ + @Override public float getEvictionRate() { + return 0; + } + + /** {@inheritDoc} */ + @Override public float getLargeEntriesPagesPercentage() { + if (!metricsEnabled) + return 0; + + return totalAllocatedPages.longValue() != 0 ? + (float) largeEntriesPages.doubleValue() / totalAllocatedPages.longValue() + : 0; + } + + /** {@inheritDoc} */ + @Override public float getPagesFillFactor() { + if (!metricsEnabled || freeList == null) + return 0; + + return freeList.fillFactor(); + } + + /** + * Increments totalAllocatedPages counter. + */ + public void incrementTotalAllocatedPages() { + if (metricsEnabled) { + totalAllocatedPages.increment(); + + try { + updateAllocationRateMetrics(); + } + catch (ArrayIndexOutOfBoundsException | NullPointerException ignored) { + // No-op. + } + } + } + + /** + * + */ + private void updateAllocationRateMetrics() { + if (subInts != allocRateCounters.length) + return; + + long lastUpdT = lastUpdTime.get(); + long currT = IgniteUtils.currentTimeMillis(); + + int currIdx = counterIdx.get(); + + long deltaT = currT - lastUpdT; + + int subInts = this.subInts; + + LongAdder8[] rateCntrs = allocRateCounters; + + for (int i = 1; i <= subInts; i++) { + if (deltaT < subInt(i)) { + if (i > 1) { + if (!lastUpdTime.compareAndSet(lastUpdT, currT)) { + rateCntrs[counterIdx.get()].increment(); + + break; + } + } + + if (rotateIndex(currIdx, i - 1)) { + currIdx = counterIdx.get(); + + resetCounters(currIdx, i - 1); + + rateCntrs[currIdx].increment(); + + break; + } + else { + rateCntrs[counterIdx.get()].increment(); + + break; + } + } + else if (i == subInts && lastUpdTime.compareAndSet(lastUpdT, currT)) + resetAll(); + + if (currIdx != counterIdx.get()) { + rateCntrs[counterIdx.get()].increment(); + + break; + } + } + } + + /** + * @param intervalNum Interval number. + */ + private int subInt(int intervalNum) { + return (rateTimeInterval * 1000 * intervalNum) / subInts; + } + + /** + * @param idx Index. + * @param rotateStep Rotate step. + */ + private boolean rotateIndex(int idx, int rotateStep) { + if (rotateStep == 0) + return true; + + int subInts = this.subInts; + + assert rotateStep < subInts; + + int nextIdx = (idx + rotateStep) % subInts; + + return counterIdx.compareAndSet(idx, nextIdx); + } + + /** + * + */ + private void resetAll() { + LongAdder8[] cntrs = allocRateCounters; + + for (LongAdder8 cntr : cntrs) + cntr.reset(); + } + + /** + * @param currIdx Current index. + * @param resettingCntrs Resetting allocRateCounters. + */ + private void resetCounters(int currIdx, int resettingCntrs) { + if (resettingCntrs == 0) + return; + + for (int j = 0; j < resettingCntrs; j++) { + int cleanIdx = currIdx - j >= 0 ? currIdx - j : currIdx - j + subInts; + + allocRateCounters[cleanIdx].reset(); + } + } + + /** + * + */ + public void incrementLargeEntriesPages() { + if (metricsEnabled) + largeEntriesPages.increment(); + } + + /** + * + */ + public void decrementLargeEntriesPages() { + if (metricsEnabled) + largeEntriesPages.decrement(); + } + + /** {@inheritDoc} */ + @Override public void enableMetrics() { + metricsEnabled = true; + } + + /** {@inheritDoc} */ + @Override public void disableMetrics() { + metricsEnabled = false; + } + + /** + * @param rateTimeInterval Time interval used to calculate allocation/eviction rate. + */ + @Override public void rateTimeInterval(int rateTimeInterval) { + this.rateTimeInterval = rateTimeInterval; + } + + /** + * Sets number of subintervals the whole rateTimeInterval will be split into to calculate allocation rate. + * + * @param subInts Number of subintervals. + */ + @Override public void subIntervals(int subInts) { + assert subInts > 0; + + if (this.subInts == subInts) + return; + + int rateIntervalMs = rateTimeInterval * 1000; + + if (rateIntervalMs / subInts < 10) + subInts = rateIntervalMs / 10; + + LongAdder8[] newCounters = new LongAdder8[subInts]; + + for (int i = 0; i < subInts; i++) + newCounters[i] = new LongAdder8(); + + this.subInts = subInts; + allocRateCounters = newCounters; + } + + /** + * @param freeList Free list. + */ + void freeList(FreeListImpl freeList) { + this.freeList = freeList; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java index 0978d10..be23b38 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.internal.processors.cache.database; +import org.apache.ignite.MemoryMetrics; import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.internal.pagemem.PageMemory; @@ -27,13 +28,19 @@ public class MemoryPolicy { private final PageMemory pageMem; /** */ + private final MemoryMetrics memMetrics; + + /** */ private final MemoryPolicyConfiguration cfg; /** - * @param pageMem Page mem. + * @param pageMem PageMemory instance. + * @param memMetrics MemoryMetrics instance. + * @param cfg Configuration of given MemoryPolicy. */ - public MemoryPolicy(PageMemory pageMem, MemoryPolicyConfiguration cfg) { + public MemoryPolicy(PageMemory pageMem, MemoryMetrics memMetrics, MemoryPolicyConfiguration cfg) { this.pageMem = pageMem; + this.memMetrics = memMetrics; this.cfg = cfg; } @@ -50,4 +57,11 @@ public class MemoryPolicy { public MemoryPolicyConfiguration config() { return cfg; } + + /** + * @return Memory Metrics. + */ + public MemoryMetrics memoryMetrics() { + return memMetrics; + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java index b74c85e..d433172 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord; import org.apache.ignite.internal.processors.cache.database.CacheDataRow; +import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl; import org.apache.ignite.internal.processors.cache.database.tree.io.CacheVersionIO; import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO; import org.apache.ignite.internal.processors.cache.database.tree.io.DataPagePayload; @@ -72,6 +73,9 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList { /** */ private final PageHandler<CacheDataRow, Boolean> updateRow = new UpdateRowHandler(); + /** */ + private final MemoryMetricsImpl memMetrics; + /** * */ @@ -294,6 +298,7 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList { * @param cacheId Cache ID. * @param name Name (for debug purpose). * @param pageMem Page memory. + * @param memMetrics Memory metrics. * @param reuseList Reuse list or {@code null} if this free list will be a reuse list for itself. * @param wal Write ahead log manager. * @param metaPageId Metadata page ID. @@ -304,6 +309,7 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList { int cacheId, String name, PageMemory pageMem, + MemoryMetricsImpl memMetrics, ReuseList reuseList, IgniteWriteAheadLogManager wal, long metaPageId, @@ -329,9 +335,33 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList { this.shift = shift; + this.memMetrics = memMetrics; + init(metaPageId, initNew); } + /** + * Calculates average fill factor over FreeListImpl instance. + */ + public float fillFactor() { + long pageSize = pageSize(); + + long totalSize = 0; + long loadSize = 0; + + for (int b = BUCKETS - 2; b > 0; b--) { + long bsize = pageSize - ((REUSE_BUCKET - b) << shift); + + long pages = bucketsSize[b].longValue(); + + loadSize += pages * (pageSize - bsize); + + totalSize += pages * pageSize; + } + + return totalSize == 0 ? -1L : ((float) loadSize / totalSize); + } + /** {@inheritDoc} */ @Override public void dumpStatistics(IgniteLogger log) { long dataPages = 0; @@ -411,6 +441,9 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList { int written = 0; do { + if (written != 0) + memMetrics.incrementLargeEntriesPages(); + int freeSpace = Math.min(MIN_SIZE_FOR_DATA_PAGE, rowSize - written); int bucket = bucket(freeSpace, false); @@ -472,6 +505,8 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList { assert nextLink != FAIL_L; // Can't fail here. while (nextLink != 0L) { + memMetrics.decrementLargeEntriesPages(); + itemId = PageIdUtils.itemId(nextLink); pageId = PageIdUtils.pageId(nextLink); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java index 72f5d62..a60d364 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java @@ -1400,7 +1400,6 @@ public class PlatformCache extends PlatformAbstractTarget { writer.writeLong(metrics.getCacheTxCommits()); writer.writeLong(metrics.getCacheTxRollbacks()); writer.writeString(metrics.name()); - writer.writeLong(metrics.getOverflowSize()); writer.writeLong(metrics.getOffHeapGets()); writer.writeLong(metrics.getOffHeapPuts()); writer.writeLong(metrics.getOffHeapRemovals()); @@ -1414,15 +1413,6 @@ public class PlatformCache extends PlatformAbstractTarget { writer.writeLong(metrics.getOffHeapBackupEntriesCount()); writer.writeLong(metrics.getOffHeapAllocatedSize()); writer.writeLong(metrics.getOffHeapMaxSize()); - writer.writeLong(metrics.getSwapGets()); - writer.writeLong(metrics.getSwapPuts()); - writer.writeLong(metrics.getSwapRemovals()); - writer.writeLong(metrics.getSwapHits()); - writer.writeLong(metrics.getSwapMisses()); - writer.writeLong(metrics.getSwapEntriesCount()); - writer.writeLong(metrics.getSwapSize()); - writer.writeFloat(metrics.getSwapHitPercentage()); - writer.writeFloat(metrics.getSwapMissPercentage()); writer.writeInt(metrics.getSize()); writer.writeInt(metrics.getKeySize()); writer.writeBoolean(metrics.isEmpty()); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryConfiguration.java index 2080105..f380394 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryConfiguration.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.List; import org.apache.ignite.configuration.MemoryConfiguration; import org.apache.ignite.configuration.MemoryPolicyConfiguration; -import org.apache.ignite.internal.LessNamingBean; import org.apache.ignite.internal.util.typedef.F; /** http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java index 0b6c8a8..61d2984 100644 --- a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java +++ b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java @@ -96,10 +96,6 @@ public interface CacheMetricsMXBean extends CacheStatisticsMXBean, CacheMXBean, public String name(); /** {@inheritDoc} */ - @MXBeanDescription("Number of entries that was swapped to disk.") - public long getOverflowSize(); - - /** {@inheritDoc} */ @MXBeanDescription("Number of gets from off-heap memory.") public long getOffHeapGets(); @@ -152,42 +148,6 @@ public interface CacheMetricsMXBean extends CacheStatisticsMXBean, CacheMXBean, public long getOffHeapMaxSize(); /** {@inheritDoc} */ - @MXBeanDescription("Number of gets from swap.") - public long getSwapGets(); - - /** {@inheritDoc} */ - @MXBeanDescription("Number of puts to swap.") - public long getSwapPuts(); - - /** {@inheritDoc} */ - @MXBeanDescription("Number of removed entries from swap.") - public long getSwapRemovals(); - - /** {@inheritDoc} */ - @MXBeanDescription("Number of hits on swap.") - public long getSwapHits(); - - /** {@inheritDoc} */ - @MXBeanDescription("Number of misses on swap.") - public long getSwapMisses(); - - /** {@inheritDoc} */ - @MXBeanDescription("Percentage of hits on swap.") - public float getSwapHitPercentage(); - - /** {@inheritDoc} */ - @MXBeanDescription("Percentage of misses on swap.") - public float getSwapMissPercentage(); - - /** {@inheritDoc} */ - @MXBeanDescription("Number of entries stored in swap.") - public long getSwapEntriesCount(); - - /** {@inheritDoc} */ - @MXBeanDescription("Size of swap.") - public long getSwapSize(); - - /** {@inheritDoc} */ @MXBeanDescription("Number of non-null values in the cache.") public int getSize(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/main/java/org/apache/ignite/mxbean/MemoryMetricsMXBean.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/MemoryMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/MemoryMetricsMXBean.java new file mode 100644 index 0000000..db75f57 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/mxbean/MemoryMetricsMXBean.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.mxbean; + +import org.apache.ignite.MemoryMetrics; + +/** + * This interface defines JMX view on {@link MemoryMetrics}. + */ +@MXBeanDescription("MBean that provides access to MemoryMetrics of current Ignite node.") +public interface MemoryMetricsMXBean extends MemoryMetrics { + /** {@inheritDoc} */ + @MXBeanDescription("Name of PageMemory metrics are collected for.") + @Override public String getName(); + + /** {@inheritDoc} */ + @MXBeanDescription("Size of PageMemory in MBytes.") + @Override public int getSize(); + + /** {@inheritDoc} */ + @MXBeanDescription("File path of memory-mapped swap file.") + @Override public String getSwapFilePath(); + + /** {@inheritDoc} */ + @MXBeanDescription("Enables metrics gathering.") + @Override public void enableMetrics(); + + /** {@inheritDoc} */ + @MXBeanDescription("Disables metrics gathering.") + @Override public void disableMetrics(); + + /** {@inheritDoc} */ + @MXBeanDescription("Total number of allocated pages.") + @Override public long getTotalAllocatedPages(); + + /** {@inheritDoc} */ + @MXBeanDescription("Allocation rate (pages per second) averaged across rateTimeInternal.") + @Override public float getAllocationRate(); + + /** {@inheritDoc} */ + @MXBeanDescription("Eviction rate (pages per second).") + @Override public float getEvictionRate(); + + /** {@inheritDoc} */ + @MXBeanDescription("Percentage of pages fully occupied by large entities' fragments.") + @Override public float getLargeEntriesPagesPercentage(); + + /** {@inheritDoc} */ + @MXBeanDescription("Pages fill factor: size of all entries in cache over size of all allocated pages.") + @Override public float getPagesFillFactor(); + + /** {@inheritDoc} */ + @MXBeanDescription( + "Sets time interval average allocation rate (pages per second) is calculated over." + ) + @MXBeanParametersNames( + "rateTimeInterval" + ) + @MXBeanParametersDescriptions( + "Time interval (in seconds) to set." + ) + @Override public void rateTimeInterval(int rateTimeInterval); + + /** {@inheritDoc} */ + @MXBeanDescription( + "Sets number of subintervals to calculate allocationRate metrics." + ) + @MXBeanParametersNames( + "subInts" + ) + @MXBeanParametersDescriptions( + "Number of subintervals to set." + ) + @Override public void subIntervals(int subInts); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java index 8362ace..c1ce8b7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.PageUtils; +import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl; import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -286,7 +287,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest { DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), memDir, true, sizes); - return new PageMemoryNoStoreImpl(log(), provider, null, PAGE_SIZE, true); + return new PageMemoryNoStoreImpl(log(), provider, null, PAGE_SIZE, new MemoryMetricsImpl(null), true); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/3f90a9c0/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConfigurationLeakTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConfigurationLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConfigurationLeakTest.java index b51932f..5c85d01 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConfigurationLeakTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConfigurationLeakTest.java @@ -47,9 +47,10 @@ public class CacheConfigurationLeakTest extends GridCommonAbstractTest { MemoryPolicyConfiguration plc = new MemoryPolicyConfiguration(); - plc.setName("dfltMemPlc"); + plc.setName("dfltPlc"); plc.setSize(MemoryConfiguration.DFLT_MEMORY_POLICY_SIZE * 10); + memCfg.setDefaultMemoryPolicyName("dfltPlc"); memCfg.setMemoryPolicies(plc); cfg.setMemoryConfiguration(memCfg);
