This is an automated email from the ASF dual-hosted git repository.
arshad pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-metrics.git
The following commit(s) were added to refs/heads/master by this push:
new a6324d8 [AMBARI-26135] Upgrade net.sf.ehcache to 3.10.0 (#132)
a6324d8 is described below
commit a6324d84c4464bb808244795c15fa998028a09fd
Author: Sandeep Kumar <[email protected]>
AuthorDate: Thu Oct 3 14:57:25 2024 +0530
[AMBARI-26135] Upgrade net.sf.ehcache to 3.10.0 (#132)
Signed-off-by: Mohammad Arshad <[email protected]>
---
ambari-metrics-timelineservice/pom.xml | 34 ++++-
.../core/timeline/TimelineMetricConfiguration.java | 13 +-
.../source/cache/InternalMetricsCache.java | 155 +++++++--------------
.../source/cache/InternalMetricsCacheProvider.java | 2 +-
.../cache/InternalMetricsCacheSizeOfEngine.java | 148 --------------------
5 files changed, 84 insertions(+), 268 deletions(-)
diff --git a/ambari-metrics-timelineservice/pom.xml
b/ambari-metrics-timelineservice/pom.xml
index 426afd3..1393492 100644
--- a/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics-timelineservice/pom.xml
@@ -511,11 +511,6 @@
<artifactId>hadoop-yarn-api</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- <version>2.2.2</version>
- </dependency>
<dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
@@ -835,9 +830,34 @@
<version>2.1.0</version>
</dependency>
<dependency>
- <groupId>net.sf.ehcache</groupId>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>2.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.istack</groupId>
+ <artifactId>istack-commons-runtime</artifactId>
+ <version>4.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.fastinfoset</groupId>
+ <artifactId>FastInfoset</artifactId>
+ <version>1.2.16</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ehcache</groupId>
<artifactId>ehcache</artifactId>
- <version>2.10.0</version>
+ <version>3.10.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.glassfish.jaxb</groupId>
+ <artifactId>jaxb-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jaxb</groupId>
+ <artifactId>jaxb-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
index f8f4bb0..05f7b03 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
@@ -317,8 +317,7 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS =
"timeline.metrics.collector.ignite.nodes.backups";
- public static final String INTERNAL_CACHE_HEAP_PERCENT =
- "timeline.metrics.internal.cache.%s.heap.percent";
+ public static final String INTERNAL_CACHE_ENTRY_COUNT =
"timeline.metrics.internal.cache.%s.entry.count";
public static final String EXTERNAL_SINK_INTERVAL =
"timeline.metrics.external.sink.%s.%s.interval";
@@ -628,12 +627,12 @@ public class TimelineMetricConfiguration {
return providerList;
}
- public String getInternalCacheHeapPercent(String instanceName) {
- String heapPercent =
metricsConf.get(String.format(INTERNAL_CACHE_HEAP_PERCENT, instanceName));
- if (StringUtils.isEmpty(heapPercent)) {
- return "5%";
+ public Integer getInternalCacheSize(String instanceName) {
+ String cacheEntryCount =
metricsConf.get(String.format(INTERNAL_CACHE_ENTRY_COUNT, instanceName));
+ if(StringUtils.isEmpty(cacheEntryCount)) {
+ return 500;
} else {
- return heapPercent.endsWith("%") ? heapPercent : heapPercent + "%";
+ return Integer.parseInt(cacheEntryCount);
}
}
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java
index b26921f..afef409 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java
@@ -24,36 +24,39 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.EnumSet;
+import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheException;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Ehcache;
-import net.sf.ehcache.Element;
-import net.sf.ehcache.config.CacheConfiguration;
-import net.sf.ehcache.config.PersistenceConfiguration;
-import net.sf.ehcache.config.SizeOfPolicyConfiguration;
-import net.sf.ehcache.event.CacheEventListener;
-import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
+import org.ehcache.Cache;
+import org.ehcache.CacheManager;
+import org.ehcache.config.CacheConfiguration;
+import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.ResourcePoolsBuilder;
+import org.ehcache.config.units.EntryUnit;
+import org.ehcache.event.CacheEvent;
+import org.ehcache.event.CacheEventListener;
+import org.ehcache.event.EventType;
+import org.ehcache.event.EventFiring;
+import org.ehcache.event.EventOrdering;
+import org.ehcache.expiry.Expirations;
public class InternalMetricsCache {
private static final Log LOG = LogFactory.getLog(InternalMetricsCache.class);
private final String instanceName;
- private final String maxHeapPercent;
+ private final Integer internalCacheEntryCount;
private volatile boolean isCacheInitialized = false;
- private Cache cache;
- static final String TIMELINE_METRIC_CACHE_MANAGER_NAME =
"internalMetricsCacheManager";
+ private Cache<InternalMetricCacheKey, InternalMetricCacheValue> cache;
private final Lock lock = new ReentrantLock();
private static final int LOCK_TIMEOUT_SECONDS = 2;
- public InternalMetricsCache(String instanceName, String maxHeapPercent) {
+ public InternalMetricsCache(String instanceName, Integer
internalCacheEntryCount) {
this.instanceName = instanceName;
- this.maxHeapPercent = maxHeapPercent;
+ this.internalCacheEntryCount = internalCacheEntryCount;
initialize();
}
@@ -63,71 +66,49 @@ public class InternalMetricsCache {
throw new RuntimeException("Cannot initialize internal cache twice");
}
- System.setProperty("net.sf.ehcache.skipUpdateCheck", "true");
- System.setProperty("net.sf.ehcache.sizeofengine." +
TIMELINE_METRIC_CACHE_MANAGER_NAME,
-
"org.apache.ambari.metrics.core.timeline.source.cache.InternalMetricsCacheSizeOfEngine");
-
- net.sf.ehcache.config.Configuration managerConfig =
- new net.sf.ehcache.config.Configuration();
- managerConfig.setName(TIMELINE_METRIC_CACHE_MANAGER_NAME);
-
- // Set max heap available to the cache manager
- managerConfig.setMaxBytesLocalHeap(maxHeapPercent);
-
- //Create a singleton CacheManager using defaults
- CacheManager manager = CacheManager.create(managerConfig);
-
- LOG.info("Creating Metrics Cache with maxHeapPercent => " +
maxHeapPercent);
+ CacheManager manager = CacheManagerBuilder.newCacheManagerBuilder()
+ .build(true);
// Create a Cache specifying its configuration.
- CacheConfiguration cacheConfiguration = new CacheConfiguration()
- .name(instanceName)
- .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
- .sizeOfPolicy(new SizeOfPolicyConfiguration() // Set sizeOf policy to
continue on max depth reached - avoid OOM
- .maxDepth(10000)
-
.maxDepthExceededBehavior(SizeOfPolicyConfiguration.MaxDepthExceededBehavior.CONTINUE))
- .eternal(true) // infinite time until eviction
- .persistence(new PersistenceConfiguration()
- .strategy(PersistenceConfiguration.Strategy.NONE.name()));
-
- cache = new Cache(cacheConfiguration);
- cache.getCacheEventNotificationService().registerListener(new
InternalCacheEvictionListener());
+ CacheConfiguration<InternalMetricCacheKey, InternalMetricCacheValue>
cacheConfig =
+ CacheConfigurationBuilder.newCacheConfigurationBuilder(
+ InternalMetricCacheKey.class,
InternalMetricCacheValue.class,
+
ResourcePoolsBuilder.newResourcePoolsBuilder().heap(internalCacheEntryCount,
EntryUnit.ENTRIES)
+ ).withExpiry(Expirations.noExpiration()).build();
- LOG.info("Registering internal metrics cache with provider: name = " +
- cache.getName() + ", guid: " + cache.getGuid());
+ cache = manager.createCache(instanceName, cacheConfig);
+ cache.getRuntimeConfiguration().registerCacheEventListener(new
InternalCacheEvictionListener(), EventOrdering.ORDERED,
EventFiring.SYNCHRONOUS, EnumSet.of(EventType.EVICTED));
- manager.addCache(cache);
+ LOG.info("Registering internal metrics cache with provider: name = " +
+ instanceName);
isCacheInitialized = true;
}
public InternalMetricCacheValue
getInternalMetricCacheValue(InternalMetricCacheKey key) {
- Element ele = cache.get(key);
- if (ele != null) {
- return (InternalMetricCacheValue) ele.getObjectValue();
- }
- return null;
+ return cache.get(key);
}
public Collection<TimelineMetrics> evictAll() {
TimelineMetrics metrics = new TimelineMetrics();
try {
if (lock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
- try{
- List keys = cache.getKeys();
- for (Object obj : keys) {
+ try {
+ Iterator<Cache.Entry<InternalMetricCacheKey,
InternalMetricCacheValue>> iterator = cache.iterator();
+ while (iterator.hasNext()) {
+ Cache.Entry<InternalMetricCacheKey, InternalMetricCacheValue>
entry = iterator.next();
TimelineMetric metric = new TimelineMetric();
- InternalMetricCacheKey key = (InternalMetricCacheKey) obj;
+ InternalMetricCacheKey key = entry.getKey();
metric.setMetricName(key.getMetricName());
metric.setAppId(key.getAppId());
metric.setInstanceId(key.getInstanceId());
metric.setHostName(key.getHostname());
metric.setStartTime(key.getStartTime());
- Element ele = cache.get(key);
- metric.setMetricValues(((InternalMetricCacheValue)
ele.getObjectValue()).getMetricValues());
+ InternalMetricCacheValue value = cache.get(key);
+ metric.setMetricValues(value.getMetricValues());
metrics.getMetrics().add(metric);
+ iterator.remove();
}
- cache.removeAll();
} finally {
lock.unlock();
}
@@ -157,14 +138,13 @@ public class InternalMetricsCache {
timelineMetric.getStartTime()
);
- Element ele = cache.get(key);
- if (ele != null) {
- InternalMetricCacheValue value = (InternalMetricCacheValue)
ele.getObjectValue();
+ InternalMetricCacheValue value = cache.get(key);
+ if (value != null) {
value.addMetricValues(timelineMetric.getMetricValues());
} else {
- InternalMetricCacheValue value = new
InternalMetricCacheValue();
+ value = new InternalMetricCacheValue();
value.setMetricValues(timelineMetric.getMetricValues());
- cache.put(new Element(key, value));
+ cache.put(key, value);
}
}
}
@@ -181,49 +161,14 @@ public class InternalMetricsCache {
}
}
- class InternalCacheEvictionListener implements CacheEventListener {
-
- @Override
- public void notifyElementRemoved(Ehcache cache, Element element) throws
CacheException {
- // expected
- }
-
- @Override
- public void notifyElementPut(Ehcache cache, Element element) throws
CacheException {
- // do nothing
- }
-
+ class InternalCacheEvictionListener implements
CacheEventListener<InternalMetricCacheKey, InternalMetricCacheValue> {
@Override
- public void notifyElementUpdated(Ehcache cache, Element element) throws
CacheException {
- // do nothing
- }
-
- @Override
- public void notifyElementExpired(Ehcache cache, Element element) {
- // do nothing
- }
-
- @Override
- public void notifyElementEvicted(Ehcache cache, Element element) {
- // Bad - Remote endpoint cannot keep up resulting in flooding
- InternalMetricCacheKey key = (InternalMetricCacheKey)
element.getObjectKey();
- LOG.warn("Evicting element from internal metrics cache, metric => " + key
- .getMetricName() + ", startTime = " + new Date(key.getStartTime()));
- }
-
- @Override
- public void notifyRemoveAll(Ehcache cache) {
- // expected
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return null;
- }
-
- @Override
- public void dispose() {
- // do nothing
+ public void onEvent(CacheEvent<? extends InternalMetricCacheKey, ? extends
InternalMetricCacheValue> event) {
+ if (event.getType() == EventType.EVICTED) {
+ InternalMetricCacheKey key = event.getKey();
+ LOG.warn("Evicting element from internal metrics cache, metric => " +
key
+ .getMetricName() + ", startTime = " + new
Date(key.getStartTime()));
+ }
}
}
}
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java
index 8ec9acd..1fc954c 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java
@@ -39,7 +39,7 @@ public class InternalMetricsCacheProvider {
} else {
TimelineMetricConfiguration conf =
TimelineMetricConfiguration.getInstance();
InternalMetricsCache cache = new InternalMetricsCache(instanceName,
- conf.getInternalCacheHeapPercent(instanceName));
+ conf.getInternalCacheSize(instanceName));
metricsCacheMap.put(instanceName, cache);
return cache;
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
deleted file mode 100644
index cfa4f97..0000000
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.core.timeline.source.cache;
-
-import net.sf.ehcache.Element;
-import net.sf.ehcache.pool.Size;
-import net.sf.ehcache.pool.SizeOfEngine;
-import net.sf.ehcache.pool.impl.DefaultSizeOfEngine;
-import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
-import net.sf.ehcache.pool.sizeof.SizeOf;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-public class InternalMetricsCacheSizeOfEngine implements SizeOfEngine {
- private final static Logger LOG =
LoggerFactory.getLogger(InternalMetricsCacheSizeOfEngine.class);
-
- private static int DEFAULT_MAX_DEPTH = 1000;
- private static boolean DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED = false;
-
- // Base Engine
- protected SizeOfEngine underlying = null;
-
- // Counter
- protected SizeOf reflectionSizeOf = new ReflectionSizeOf();
-
- // Optimizations
- private volatile long timelineMetricPrimitivesApproximation = 0;
-
- // Map entry sizing
- private long sizeOfMapEntry;
- private long sizeOfMapEntryOverhead;
- private long sizeOfElement;
-
- protected InternalMetricsCacheSizeOfEngine(SizeOfEngine underlying) {
- this.underlying = underlying;
- }
-
- public InternalMetricsCacheSizeOfEngine() {
- this(new DefaultSizeOfEngine(DEFAULT_MAX_DEPTH,
DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED));
-
- this.sizeOfMapEntry = reflectionSizeOf.sizeOf(new Long(1)) +
- reflectionSizeOf.sizeOf(new Double(2.0));
-
- this.sizeOfElement = reflectionSizeOf.sizeOf(new Element(new Object(), new
Object()));
-
- //SizeOfMapEntryOverhead = SizeOfMapWithOneEntry - (SizeOfEmptyMap +
SizeOfOneEntry)
- TreeMap<Long, Double> map = new TreeMap<>();
- long emptyMapSize = reflectionSizeOf.sizeOf(map);
- map.put(new Long(1), new Double(2.0));
- long sizeOfMapOneEntry = reflectionSizeOf.deepSizeOf(DEFAULT_MAX_DEPTH,
DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED, map).getCalculated();
- this.sizeOfMapEntryOverhead = sizeOfMapOneEntry - (emptyMapSize +
this.sizeOfMapEntry);
-
- LOG.info("Creating custom sizeof engine for TimelineMetrics.");
- }
-
- /**
- * Return size of the metrics TreeMap in an optimized way.
- *
- */
- protected long getTimelineMetricsSize(TimelineMetrics metrics) {
- long size = 8; // Object reference
-
- if (metrics != null) {
- for (TimelineMetric metric : metrics.getMetrics()) {
-
- if (timelineMetricPrimitivesApproximation == 0) {
- timelineMetricPrimitivesApproximation +=
reflectionSizeOf.sizeOf(metric.getMetricName());
- timelineMetricPrimitivesApproximation +=
reflectionSizeOf.sizeOf(metric.getAppId());
- timelineMetricPrimitivesApproximation +=
reflectionSizeOf.sizeOf(metric.getHostName());
- timelineMetricPrimitivesApproximation +=
reflectionSizeOf.sizeOf(metric.getInstanceId());
- timelineMetricPrimitivesApproximation +=
reflectionSizeOf.sizeOf(metric.getStartTime());
- timelineMetricPrimitivesApproximation +=
reflectionSizeOf.sizeOf(metric.getType());
- timelineMetricPrimitivesApproximation += 8; // Object overhead
-
- LOG.debug("timelineMetricPrimitivesApproximation bytes = " +
timelineMetricPrimitivesApproximation);
- }
- size += timelineMetricPrimitivesApproximation;
- size += getValueMapSize(metric.getMetricValues());
- }
- LOG.debug("Total Size of metric values in cache: " + size);
- }
- return size;
- }
-
- protected long getValueMapSize(Map<Long, Double> metricValues) {
- long size = 0;
- if (metricValues != null && !metricValues.isEmpty()) {
- // Numeric wrapper: 12 bytes + 8 bytes Data type + 4 bytes alignment =
48 (Long, Double)
- // Tree Map: 12 bytes for header + 20 bytes for 5 object fields :
pointers + 1 byte for flag = 40
- LOG.debug("Size of metric value: " + (sizeOfMapEntry +
sizeOfMapEntryOverhead) * metricValues.size());
- size += (sizeOfMapEntry + sizeOfMapEntryOverhead) * metricValues.size();
// Treemap size is O(1)
- }
- return size;
- }
-
- @Override
- public Size sizeOf(Object key, Object value, Object container) {
- return new Size(sizeOfElement + getSizeOfEntry(key, value), false);
- }
-
- @Override
- public SizeOfEngine copyWith(int maxDepth, boolean
abortWhenMaxDepthExceeded) {
- LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}",
maxDepth, abortWhenMaxDepthExceeded);
-
- return underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded);
- }
-
- protected long getSizeOfEntry(Object key, Object value) {
- try {
- LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value);
- long size = 0;
- if (key instanceof InternalMetricCacheKey) {
- InternalMetricCacheKey metricCacheKey = (InternalMetricCacheKey) key;
- size += reflectionSizeOf.sizeOf(metricCacheKey.getMetricName());
- size += reflectionSizeOf.sizeOf(metricCacheKey.getAppId());
- size += reflectionSizeOf.sizeOf(metricCacheKey.getInstanceId()); //
null safe
- size += reflectionSizeOf.sizeOf(metricCacheKey.getHostname());
- }
- if (value instanceof InternalMetricCacheValue) {
- size += getValueMapSize(((InternalMetricCacheValue)
value).getMetricValues());
- }
- // Mark size as not being exact
- return size;
- } finally {
- LOG.debug("END - Sizeof, key: {}", key);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
