IGNITE-5461 Added collecting of memory metrics.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/29c26936 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/29c26936 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/29c26936 Branch: refs/heads/ignite-2.1.2-exchange Commit: 29c26936872027cbf1256648b81ebdb796448fce Parents: 57bfba7 Author: Alexey Kuznetsov <[email protected]> Authored: Fri Jun 16 21:23:34 2017 +0700 Committer: Alexey Kuznetsov <[email protected]> Committed: Fri Jun 16 21:23:34 2017 +0700 ---------------------------------------------------------------------- .../ignite/internal/visor/cache/VisorCache.java | 15 -- .../internal/visor/cache/VisorCacheMetrics.java | 13 ++ .../visor/cache/VisorMemoryMetrics.java | 175 +++++++++++++++++++ .../visor/node/VisorNodeDataCollectorJob.java | 41 ++++- .../node/VisorNodeDataCollectorJobResult.java | 32 ++++ .../visor/node/VisorNodeDataCollectorTask.java | 6 + .../node/VisorNodeDataCollectorTaskResult.java | 27 +++ 7 files changed, 287 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/29c26936/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java index e1d3a87..aa4f271 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java @@ -71,9 +71,6 @@ public class VisorCache extends VisorDataTransferObject { /** Number of backup entries in cache. */ private long backupSize; - /** Number of cache entries stored in heap memory. */ - private long onHeapEntriesCnt; - /** Number of partitions. */ private int partitions; @@ -112,8 +109,6 @@ public class VisorCache extends VisorDataTransferObject { partitions = ca.affinity().partitions(); near = cctx.isNear(); - onHeapEntriesCnt = 0; // TODO GG-11148 How to get this metric? - metrics = new VisorCacheMetrics(ignite, name); } @@ -131,7 +126,6 @@ public class VisorCache extends VisorDataTransferObject { c.nearSize = nearSize; c.backupSize = backupSize; c.primarySize = primarySize; - c.onHeapEntriesCnt = onHeapEntriesCnt; c.partitions = partitions; c.metrics = metrics; c.near = near; @@ -212,13 +206,6 @@ public class VisorCache extends VisorDataTransferObject { } /** - * @return Number of cache entries stored in heap memory. - */ - public long getOnHeapEntriesCount() { - return onHeapEntriesCnt; - } - - /** * @return Number of partitions. */ public int getPartitions() { @@ -250,7 +237,6 @@ public class VisorCache extends VisorDataTransferObject { out.writeInt(nearSize); out.writeLong(primarySize); out.writeLong(backupSize); - out.writeLong(onHeapEntriesCnt); out.writeInt(partitions); out.writeBoolean(near); out.writeObject(metrics); @@ -267,7 +253,6 @@ public class VisorCache extends VisorDataTransferObject { nearSize = in.readInt(); primarySize = in.readLong(); backupSize = in.readLong(); - onHeapEntriesCnt = in.readLong(); partitions = in.readInt(); near = in.readBoolean(); metrics = (VisorCacheMetrics)in.readObject(); http://git-wip-us.apache.org/repos/asf/ignite/blob/29c26936/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java index 5c1dfcb..e67f5f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java @@ -157,6 +157,9 @@ public class VisorCacheMetrics extends VisorDataTransferObject { /** Number of cached rolled back DHT transaction IDs. */ private int txDhtRolledbackVersionsSize; + /** Number of cache entries stored in heap memory. */ + private long heapEntriesCnt; + /** Memory size allocated in off-heap. */ private long offHeapAllocatedSize; @@ -259,6 +262,7 @@ public class VisorCacheMetrics extends VisorDataTransferObject { txDhtCommittedVersionsSize = m.getTxDhtCommittedVersionsSize(); txDhtRolledbackVersionsSize = m.getTxDhtRolledbackVersionsSize(); + heapEntriesCnt = m.getHeapEntriesCount(); offHeapAllocatedSize = m.getOffHeapAllocatedSize(); offHeapEntriesCnt = m.getOffHeapEntriesCount(); @@ -561,6 +565,13 @@ public class VisorCacheMetrics extends VisorDataTransferObject { } /** + * @return Number of entries in heap memory. + */ + public long getHeapEntriesCount() { + return heapEntriesCnt; + } + + /** * @return Memory size allocated in off-heap. */ public long getOffHeapAllocatedSize() { @@ -653,6 +664,7 @@ public class VisorCacheMetrics extends VisorDataTransferObject { out.writeInt(txDhtCommittedVersionsSize); out.writeInt(txDhtRolledbackVersionsSize); + out.writeLong(heapEntriesCnt); out.writeLong(offHeapAllocatedSize); out.writeLong(offHeapEntriesCnt); @@ -708,6 +720,7 @@ public class VisorCacheMetrics extends VisorDataTransferObject { txDhtCommittedVersionsSize = in.readInt(); txDhtRolledbackVersionsSize = in.readInt(); + heapEntriesCnt = in.readLong(); offHeapAllocatedSize = in.readLong(); offHeapEntriesCnt = in.readLong(); http://git-wip-us.apache.org/repos/asf/ignite/blob/29c26936/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java new file mode 100644 index 0000000..14eaba6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.cache; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.MemoryMetrics; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * Data transfer object for {@link MemoryMetrics} + */ +public class VisorMemoryMetrics extends VisorDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private String name; + + /** */ + private long totalAllocatedPages; + + /** */ + private float allocationRate; + + /** */ + private float evictionRate; + + /** */ + private float largeEntriesPagesPercentage; + + /** */ + private float pagesFillFactor; + + /** */ + private long dirtyPages; + + /** */ + private float pageReplaceRate; + + /** */ + private long physicalMemoryPages; + + /** + * Default constructor. + */ + public VisorMemoryMetrics() { + // No-op. + } + + /** + * @param m Metrics instance to create DTO. + */ + public VisorMemoryMetrics(MemoryMetrics m) { + name = m.getName(); + totalAllocatedPages = m.getTotalAllocatedPages(); + allocationRate = m.getAllocationRate(); + evictionRate = m.getEvictionRate(); + largeEntriesPagesPercentage = m.getLargeEntriesPagesPercentage(); + pagesFillFactor = m.getPagesFillFactor(); + dirtyPages = m.getDirtyPages(); + pageReplaceRate = m.getPagesReplaceRate(); + physicalMemoryPages = m.getPhysicalMemoryPages(); + } + + /** + * @return Name of the memory region. + */ + public String getName() { + return name; + } + + /** + * @return Total number of allocated pages. + */ + public long getTotalAllocatedPages() { + return totalAllocatedPages; + } + + /** + * @return Number of allocated pages per second. + */ + public float getAllocationRate() { + return allocationRate; + } + + /** {@inheritDoc} */ + public float getEvictionRate() { + return evictionRate; + } + + /** + * @return Number of evicted pages per second. + */ + public float getLargeEntriesPagesPercentage() { + return largeEntriesPagesPercentage; + } + + /** + * @return Percentage of pages fully occupied by large entities. + */ + public float getPagesFillFactor() { + return pagesFillFactor; + } + + /** + * @return Current number of dirty pages. + */ + public long getDirtyPages() { + return dirtyPages; + } + + /** + * @return Pages per second replace rate. + */ + public float getPagesReplaceRate() { + return pageReplaceRate; + } + + /** + * @return Total number of pages loaded to RAM. + */ + public long getPhysicalMemoryPages() { + return physicalMemoryPages; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeString(out, name); + out.writeLong(totalAllocatedPages); + out.writeFloat(allocationRate); + out.writeFloat(evictionRate); + out.writeFloat(largeEntriesPagesPercentage); + out.writeFloat(pagesFillFactor); + out.writeLong(dirtyPages); + out.writeFloat(pageReplaceRate); + out.writeLong(physicalMemoryPages); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + name = U.readString(in); + totalAllocatedPages = in.readLong(); + allocationRate = in.readFloat(); + evictionRate = in.readFloat(); + largeEntriesPagesPercentage = in.readFloat(); + pagesFillFactor = in.readFloat(); + dirtyPages = in.readLong(); + pageReplaceRate = in.readFloat(); + physicalMemoryPages = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorMemoryMetrics.class, this); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/29c26936/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index 02d3b65..f5ea567 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -18,8 +18,10 @@ package org.apache.ignite.internal.visor.node; import java.util.Collection; +import java.util.List; import java.util.concurrent.ConcurrentMap; import org.apache.ignite.IgniteFileSystem; +import org.apache.ignite.MemoryMetrics; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -33,6 +35,7 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.cache.VisorCache; +import org.apache.ignite.internal.visor.cache.VisorMemoryMetrics; import org.apache.ignite.internal.visor.compute.VisorComputeMonitoringHolder; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; @@ -115,8 +118,8 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa events0(res, arg.getEventsOrderKey(), arg.getEventsThrottleCounterKey(), arg.isTaskMonitoringEnabled()); } - catch (Exception eventsEx) { - res.setEventsEx(eventsEx); + catch (Exception e) { + res.setEventsEx(e); } } @@ -145,6 +148,23 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa } /** + * Collect memory metrics. + * + * @param res Job result. + */ + protected void memoryMetrics(VisorNodeDataCollectorJobResult res) { + try { + List<VisorMemoryMetrics> memoryMetrics = res.getMemoryMetrics(); + + for (MemoryMetrics m : ignite.memoryMetrics()) + memoryMetrics.add(new VisorMemoryMetrics(m)); + } + catch (Exception e) { + res.setMemoryMetricsEx(e); + } +} + + /** * Collect caches. * * @param res Job result. @@ -156,6 +176,8 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa GridCacheProcessor cacheProc = ignite.context().cache(); + List<VisorCache> resCaches = res.getCaches(); + for (String cacheName : cacheProc.cacheNames()) { if (proxyCache(cacheName)) continue; @@ -169,7 +191,7 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa if (ca == null || !ca.context().started()) continue; - res.getCaches().add(new VisorCache(ignite, ca)); + resCaches.add(new VisorCache(ignite, ca)); } catch(IllegalStateException | IllegalArgumentException e) { if (debug && ignite.log() != null) @@ -182,8 +204,8 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa } } } - catch (Exception cachesEx) { - res.setCachesEx(cachesEx); + catch (Exception e) { + res.setCachesEx(e); } } @@ -222,8 +244,8 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa } } } - catch (Exception igfssEx) { - res.setIgfssEx(igfssEx); + catch (Exception e) { + res.setIgfssEx(e); } } @@ -257,6 +279,11 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa if (debug) start0 = log(ignite.log(), "Collected events", getClass(), start0); + memoryMetrics(res); + + if (debug) + start0 = log(ignite.log(), "Collected memory metrics", getClass(), start0); + caches(res, arg); if (debug) http://git-wip-us.apache.org/repos/asf/ignite/blob/29c26936/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java index 761d0ff..ce4f9fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java @@ -26,6 +26,7 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; import org.apache.ignite.internal.visor.cache.VisorCache; +import org.apache.ignite.internal.visor.cache.VisorMemoryMetrics; import org.apache.ignite.internal.visor.event.VisorGridEvent; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; @@ -52,6 +53,12 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { /** Exception while collecting node events. */ private Throwable evtsEx; + /** Node memory metrics. */ + private List<VisorMemoryMetrics> memoryMetrics = new ArrayList<>(); + + /** Exception while collecting memory metrics. */ + private Throwable memoryMetricsEx; + /** Node caches. */ private List<VisorCache> caches = new ArrayList<>(); @@ -147,6 +154,27 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { } /** + * @return Collected memory metrics. + */ + public List<VisorMemoryMetrics> getMemoryMetrics() { + return memoryMetrics; + } + + /** + * @return Exception caught during collecting memory metrics. + */ + public Throwable getMemoryMetricsEx() { + return memoryMetricsEx; + } + + /** + * @param memoryMetricsEx Exception caught during collecting memory metrics. + */ + public void setMemoryMetricsEx(Throwable memoryMetricsEx) { + this.memoryMetricsEx = memoryMetricsEx; + } + + /** * @return Collected cache metrics. */ public List<VisorCache> getCaches() { @@ -244,6 +272,8 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { out.writeBoolean(taskMonitoringEnabled); U.writeCollection(out, evts); out.writeObject(evtsEx); + U.writeCollection(out, memoryMetrics); + out.writeObject(memoryMetricsEx); U.writeCollection(out, caches); out.writeObject(cachesEx); U.writeCollection(out, igfss); @@ -261,6 +291,8 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { taskMonitoringEnabled = in.readBoolean(); evts = U.readList(in); evtsEx = (Throwable)in.readObject(); + memoryMetrics = U.readList(in); + memoryMetricsEx = (Throwable)in.readObject(); caches = U.readList(in); cachesEx = (Throwable)in.readObject(); igfss = U.readList(in); http://git-wip-us.apache.org/repos/asf/ignite/blob/29c26936/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java index 2790dec..56b3718 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java @@ -99,6 +99,12 @@ public class VisorNodeDataCollectorTask extends VisorMultiNodeTask<VisorNodeData if (jobRes.getEventsEx() != null) taskRes.getEventsEx().put(nid, new VisorExceptionWrapper(jobRes.getEventsEx())); + if (!jobRes.getMemoryMetrics().isEmpty()) + taskRes.getMemoryMetrics().put(nid, jobRes.getMemoryMetrics()); + + if (jobRes.getMemoryMetricsEx() != null) + taskRes.getMemoryMetricsEx().put(nid, new VisorExceptionWrapper(jobRes.getMemoryMetricsEx())); + if (!jobRes.getCaches().isEmpty()) taskRes.getCaches().put(nid, jobRes.getCaches()); http://git-wip-us.apache.org/repos/asf/ignite/blob/29c26936/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java index 8d1fe8e..cef3a29 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; import org.apache.ignite.internal.visor.cache.VisorCache; +import org.apache.ignite.internal.visor.cache.VisorMemoryMetrics; import org.apache.ignite.internal.visor.event.VisorGridEvent; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; @@ -66,6 +67,12 @@ public class VisorNodeDataCollectorTaskResult extends VisorDataTransferObject { /** Exceptions caught during collecting events from nodes. */ private Map<UUID, VisorExceptionWrapper> evtsEx = new HashMap<>(); + /** All memory metrics collected from nodes. */ + private Map<UUID, Collection<VisorMemoryMetrics>> memoryMetrics = new HashMap<>(); + + /** Exceptions caught during collecting memory metrics from nodes. */ + private Map<UUID, VisorExceptionWrapper> memoryMetricsEx = new HashMap<>(); + /** All caches collected from nodes. */ private Map<UUID, Collection<VisorCache>> caches = new HashMap<>(); @@ -105,6 +112,8 @@ public class VisorNodeDataCollectorTaskResult extends VisorDataTransferObject { taskMonitoringEnabled.isEmpty() && evts.isEmpty() && evtsEx.isEmpty() && + memoryMetrics.isEmpty() && + memoryMetricsEx.isEmpty() && caches.isEmpty() && cachesEx.isEmpty() && igfss.isEmpty() && @@ -171,6 +180,20 @@ public class VisorNodeDataCollectorTaskResult extends VisorDataTransferObject { } /** + * @return All memory metrics collected from nodes. + */ + public Map<UUID, Collection<VisorMemoryMetrics>> getMemoryMetrics() { + return memoryMetrics; + } + + /** + * @return Exceptions caught during collecting memory metrics from nodes. + */ + public Map<UUID, VisorExceptionWrapper> getMemoryMetricsEx() { + return memoryMetricsEx; + } + + /** * @return All caches collected from nodes. */ public Map<UUID, Collection<VisorCache>> getCaches() { @@ -236,6 +259,8 @@ public class VisorNodeDataCollectorTaskResult extends VisorDataTransferObject { U.writeMap(out, errCnts); U.writeCollection(out, evts); U.writeMap(out, evtsEx); + U.writeMap(out, memoryMetrics); + U.writeMap(out, memoryMetricsEx); U.writeMap(out, caches); U.writeMap(out, cachesEx); U.writeMap(out, igfss); @@ -255,6 +280,8 @@ public class VisorNodeDataCollectorTaskResult extends VisorDataTransferObject { errCnts = U.readMap(in); evts = U.readList(in); evtsEx = U.readMap(in); + memoryMetrics = U.readMap(in); + memoryMetricsEx = U.readMap(in); caches = U.readMap(in); cachesEx = U.readMap(in); igfss = U.readMap(in);
