This is an automated email from the ASF dual-hosted git repository.
ddekany pushed a commit to branch FREEMARKER-35
in repository https://gitbox.apache.org/repos/asf/freemarker.git
The following commit(s) were added to refs/heads/FREEMARKER-35 by this push:
new fdda2ffc FREEMARKER-35: Added memory leak (forever growing cache)
protection to Environment-level by-pattern Temporal format cache. Added tests
for format cache flushing, both for this cache, and for the global caches in
format factories.
fdda2ffc is described below
commit fdda2ffcb84f2cc0168677c9346bdaa7c3d35891
Author: ddekany <[email protected]>
AuthorDate: Sun Aug 28 18:12:58 2022 +0200
FREEMARKER-35: Added memory leak (forever growing cache) protection to
Environment-level by-pattern Temporal format cache. Added tests for format
cache flushing, both for this cache, and for the global caches in format
factories.
---
src/main/java/freemarker/core/Environment.java | 54 +++++++++-
.../java/freemarker/core/FastLRUKeyValueStore.java | 21 +++-
.../core/JavaTemplateDateFormatFactory.java | 36 +++++--
.../core/JavaTemplateNumberFormatFactory.java | 31 +++++-
.../core/JavaTemplateTemporalFormatFactory.java | 17 ++-
...ormatCacheMemoryLeakPreventionFlushingTest.java | 114 +++++++++++++++++++++
6 files changed, 253 insertions(+), 20 deletions(-)
diff --git a/src/main/java/freemarker/core/Environment.java
b/src/main/java/freemarker/core/Environment.java
index a5ef0b1a..93a41166 100644
--- a/src/main/java/freemarker/core/Environment.java
+++ b/src/main/java/freemarker/core/Environment.java
@@ -143,6 +143,12 @@ public final class Environment extends Configurable {
C_NUMBER_FORMAT_ICI_2_3_21.setDecimalFormatSymbols(symbols);
}
+ /**
+ * Maximum number of patterns per class in {@link
#cachedTemporalFormatsByFormatString}, after which cache size
+ * must be reduced. Not private to help unit testing
+ */
+ static final int
CACHED_TEMPORAL_FORMATS_BY_FORMAT_STRING_MAX_SIZE_PER_CLASS = 128;
+
private final Configuration configuration;
private final boolean incompatibleImprovementsGE2328;
private final TemplateHashModel rootDataModel;
@@ -2901,15 +2907,57 @@ public final class Environment extends Configurable {
throw new BugException("Unhandled temporal class: " +
temporalClass.getName());
}
- return classSpecificMap.computeIfAbsent(
- formatString,
- k -> new
TemplateTemporalFormat[CACHED_TEMPORAL_FORMATS_BY_FORMAT_STRING_VALUE_ARRAY_LENGTH]);
+ TemplateTemporalFormat[] cachedFormats =
classSpecificMap.get(formatString);
+ if (cachedFormats != null) {
+ return cachedFormats;
+ }
+
+ // To defend against memory leak if some misbehaving template
generates lots of unique formats:
+ if (classSpecificMap.size() >=
CACHED_TEMPORAL_FORMATS_BY_FORMAT_STRING_MAX_SIZE_PER_CLASS) {
+ classSpecificMap.clear();
+ }
+
+ cachedFormats = new
TemplateTemporalFormat[CACHED_TEMPORAL_FORMATS_BY_FORMAT_STRING_VALUE_ARRAY_LENGTH];
+ classSpecificMap.put(formatString, cachedFormats);
+ return cachedFormats;
}
+ /** Exposed for unit testing */
void clearCachedTemplateTemporalFormatsByFormatString() {
cachedTemporalFormatsByFormatString = null;
}
+ /** This is used for unit testing */
+ int getTemplateTemporalFormatsByFormatStringMapSize(Class<? extends
Temporal> temporalClass) {
+ if (cachedTemporalFormatsByFormatString == null) {
+ return 0;
+ }
+
+ Map<String, TemplateTemporalFormat[]> classSpecificMap;
+ if (temporalClass == Instant.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.instantFormats;
+ } else if (temporalClass == LocalDate.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.localDateFormats;
+ } else if (temporalClass == LocalDateTime.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.localDateTimeFormats;
+ } else if (temporalClass == LocalTime.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.localTimeFormats;
+ } else if (temporalClass == OffsetDateTime.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.offsetDateTimeFormats;
+ } else if (temporalClass == OffsetTime.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.offsetTimeFormats;
+ } else if (temporalClass == ZonedDateTime.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.zonedDateTimeFormats;
+ } else if (temporalClass == YearMonth.class) {
+ classSpecificMap =
cachedTemporalFormatsByFormatString.yearMonthFormats;
+ } else if (temporalClass == Year.class) {
+ classSpecificMap = cachedTemporalFormatsByFormatString.yearFormats;
+ } else {
+ throw new BugException("Unhandled temporal class: " +
temporalClass.getName());
+ }
+ return classSpecificMap != null ? classSpecificMap.size() : 0;
+ }
+
/**
* Returns the {@link TemplateTemporalFormat} for the given parameters
without using any
* {@link Environment}-level cacheing. The {@link
TemplateTemporalFormatFactory} involved might still uses its
diff --git a/src/main/java/freemarker/core/FastLRUKeyValueStore.java
b/src/main/java/freemarker/core/FastLRUKeyValueStore.java
index 9cbf465a..20508258 100644
--- a/src/main/java/freemarker/core/FastLRUKeyValueStore.java
+++ b/src/main/java/freemarker/core/FastLRUKeyValueStore.java
@@ -67,6 +67,8 @@ class FastLRUKeyValueStore<K, V> {
}
/**
+ * Gets the entry with the given key, and ensures that it becomes a
"recent" entry.
+ *
* @return {@code null} if there's no entry with the given key.
*/
V get(K cacheKey) {
@@ -83,8 +85,23 @@ class FastLRUKeyValueStore<K, V> {
return putIfAbsentThenReturnStored(cacheKey, value);
}
+ /**
+ * Drops all entries.
+ */
void clear() {
- olderEntries.clear();
- recentEntries.clear();
+ synchronized (this) {
+ olderEntries.clear();
+ recentEntries.clear();
+ }
+ }
+
+ /**
+ * Total number of entries stored at the moment. Can be inaccurate if the
store concurrent modified, but the main
+ * application is unit testing, where we can avoid that.
+ */
+ int size() {
+ synchronized (this) {
+ return recentEntries.size() + olderEntries.size();
+ }
}
}
diff --git a/src/main/java/freemarker/core/JavaTemplateDateFormatFactory.java
b/src/main/java/freemarker/core/JavaTemplateDateFormatFactory.java
index f4773ac3..09a8ab8d 100644
--- a/src/main/java/freemarker/core/JavaTemplateDateFormatFactory.java
+++ b/src/main/java/freemarker/core/JavaTemplateDateFormatFactory.java
@@ -25,24 +25,27 @@ import java.util.Locale;
import java.util.StringTokenizer;
import java.util.TimeZone;
-import freemarker.log.Logger;
import freemarker.template.TemplateDateModel;
class JavaTemplateDateFormatFactory extends TemplateDateFormatFactory {
-
- static final JavaTemplateDateFormatFactory INSTANCE = new
JavaTemplateDateFormatFactory();
-
- private static final Logger LOG = Logger.getLogger("freemarker.runtime");
- private final FastLRUKeyValueStore<CacheKey, DateFormat> dateFormatCache =
new FastLRUKeyValueStore<>(512);
+ static final JavaTemplateDateFormatFactory INSTANCE = new
JavaTemplateDateFormatFactory();
+
+ /**
+ * Exposed to unit testing.
+ */
+ static final int GUARANTEED_RECENT_ENTRIES = 512;
+
+ private final FastLRUKeyValueStore<CacheKey, DateFormat> dateFormatCache =
+ new FastLRUKeyValueStore<>(GUARANTEED_RECENT_ENTRIES);
private JavaTemplateDateFormatFactory() {
// Can't be instantiated
}
-
+
/**
* @param zonelessInput
- * Has no effect in this implementation.
+ * Has no effect in this implementation.
*/
@Override
public TemplateDateFormat get(String params, int dateType, Locale locale,
TimeZone timeZone, boolean zonelessInput,
@@ -101,6 +104,21 @@ class JavaTemplateDateFormatFactory extends
TemplateDateFormatFactory {
}
}
+
+ /**
+ * Used for unit testing.
+ */
+ void clear() {
+ dateFormatCache.clear();
+ }
+
+ /**
+ * Used for unit testing.
+ */
+ int getSize() {
+ return dateFormatCache.size();
+ }
+
private static final class CacheKey {
private final int dateType;
private final String pattern;
@@ -145,5 +163,5 @@ class JavaTemplateDateFormatFactory extends
TemplateDateFormatFactory {
}
return -1;
}
-
+
}
diff --git a/src/main/java/freemarker/core/JavaTemplateNumberFormatFactory.java
b/src/main/java/freemarker/core/JavaTemplateNumberFormatFactory.java
index a3de36a3..5ad814d3 100644
--- a/src/main/java/freemarker/core/JavaTemplateNumberFormatFactory.java
+++ b/src/main/java/freemarker/core/JavaTemplateNumberFormatFactory.java
@@ -26,17 +26,23 @@ import java.util.Locale;
* Deals with {@link TemplateNumberFormat}-s that just wrap a Java {@link
NumberFormat}.
*/
class JavaTemplateNumberFormatFactory extends TemplateNumberFormatFactory {
-
+
static final JavaTemplateNumberFormatFactory INSTANCE = new
JavaTemplateNumberFormatFactory();
static final String COMPUTER = "computer";
- private final FastLRUKeyValueStore<CacheKey, NumberFormat>
numberFormatCache = new FastLRUKeyValueStore<>(512);
+ /**
+ * Exposed to unit testing.
+ */
+ static final int GUARANTEED_RECENT_ENTRIES = 512;
+
+ private final FastLRUKeyValueStore<CacheKey, NumberFormat>
numberFormatCache = new FastLRUKeyValueStore<>(
+ GUARANTEED_RECENT_ENTRIES);
private JavaTemplateNumberFormatFactory() {
// Not meant to be instantiated
}
-
+
@Override
public TemplateNumberFormat get(String params, Locale locale, Environment
env)
throws InvalidFormatParametersException {
@@ -49,7 +55,7 @@ class JavaTemplateNumberFormatFactory extends
TemplateNumberFormatFactory {
numberFormat = getNumberFormatNoCache(params, locale, env);
numberFormat =
numberFormatCache.putIfAbsentThenReturnStored(cacheKey, numberFormat);
}
-
+
return new JavaTemplateNumberFormat((NumberFormat)
numberFormat.clone(), params);
}
@@ -74,6 +80,21 @@ class JavaTemplateNumberFormatFactory extends
TemplateNumberFormatFactory {
}
}
+
+ /**
+ * Used for unit testing.
+ */
+ void clear() {
+ numberFormatCache.clear();
+ }
+
+ /**
+ * Used for unit testing.
+ */
+ int getSize() {
+ return numberFormatCache.size();
+ }
+
private static final class CacheKey {
private final String pattern;
private final Locale locale;
@@ -97,5 +118,5 @@ class JavaTemplateNumberFormatFactory extends
TemplateNumberFormatFactory {
return pattern.hashCode() ^ locale.hashCode();
}
}
-
+
}
diff --git
a/src/main/java/freemarker/core/JavaTemplateTemporalFormatFactory.java
b/src/main/java/freemarker/core/JavaTemplateTemporalFormatFactory.java
index 2ad345c3..b88c0dc8 100644
--- a/src/main/java/freemarker/core/JavaTemplateTemporalFormatFactory.java
+++ b/src/main/java/freemarker/core/JavaTemplateTemporalFormatFactory.java
@@ -27,8 +27,13 @@ import java.util.TimeZone;
class JavaTemplateTemporalFormatFactory extends TemplateTemporalFormatFactory {
public static final JavaTemplateTemporalFormatFactory INSTANCE = new
JavaTemplateTemporalFormatFactory();
+ /**
+ * Exposed to unit testing.
+ */
+ static final int GUARANTEED_RECENT_ENTRIES = 512;
+
private final FastLRUKeyValueStore<CacheKey, JavaTemplateTemporalFormat>
formatCache =
- new FastLRUKeyValueStore<>(512);
+ new FastLRUKeyValueStore<>(GUARANTEED_RECENT_ENTRIES);
private JavaTemplateTemporalFormatFactory() {
// Not instantiated from outside
@@ -49,10 +54,20 @@ class JavaTemplateTemporalFormatFactory extends
TemplateTemporalFormatFactory {
return format;
}
+ /**
+ * Used for unit testing.
+ */
void clear() {
formatCache.clear();
}
+ /**
+ * Used for unit testing.
+ */
+ int getSize() {
+ return formatCache.size();
+ }
+
private static class CacheKey {
private final String params;
private final Class<? extends Temporal> temporalClass;
diff --git
a/src/test/java/freemarker/core/FormatCacheMemoryLeakPreventionFlushingTest.java
b/src/test/java/freemarker/core/FormatCacheMemoryLeakPreventionFlushingTest.java
new file mode 100644
index 00000000..b0c1e6d9
--- /dev/null
+++
b/src/test/java/freemarker/core/FormatCacheMemoryLeakPreventionFlushingTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package freemarker.core;
+
+import static org.junit.Assert.*;
+
+import java.time.Instant;
+
+import org.junit.Test;
+
+import freemarker.template.TemplateDateModel;
+
+/**
+ * This is to cover the very rare situation where a format cache grows too
big, and we fo some sort of flushing to
+ * prevent a memory leak. The main goals is to catch any exception causing bug
in that rarely ran code branch. But we
+ * also assert when exactly the flushing happens; that can change in the
future, in which case adjust the tests.
+ */
+public class FormatCacheMemoryLeakPreventionFlushingTest
+ extends TemplateTemporalFormatAbstractCachingInEnvironmentTest {
+
+ @Test
+ public void testGlobalJavaTemporalFormatCacheLeakProtection() throws
Exception {
+ JavaTemplateTemporalFormatFactory formatFactory =
JavaTemplateTemporalFormatFactory.INSTANCE;
+ formatFactory.clear();
+ int expectedMapSize = 0;
+ for (int i = 0; i <
JavaTemplateTemporalFormatFactory.GUARANTEED_RECENT_ENTRIES * 3 * 2 + 1; i++) {
+ assertEquals(expectedMapSize, formatFactory.getSize());
+ formatFactory.get(
+ "yyyyMMddHHmm '" + i + "'",
+ Instant.class, env.getLocale(), env.getTimeZone(), env);
+ expectedMapSize++;
+ // Following builds on implementation details (not on API
contract), so adjust this if that changes.
+ if (expectedMapSize >
JavaTemplateTemporalFormatFactory.GUARANTEED_RECENT_ENTRIES * 2) {
+ // The flushed entries are kept, and just become "old"
entries, therefore after flushing we have
+ // GUARANTEED_RECENT_ENTRIES entries. Then the new entry that
caused the flushing is added to the
+ // "recent" entries, hence the +1.
+ expectedMapSize =
JavaTemplateTemporalFormatFactory.GUARANTEED_RECENT_ENTRIES + 1;
+ }
+ }
+ }
+
+ @Test
+ public void testGlobalJavaDateFormatCacheLeakProtection() throws Exception
{
+ JavaTemplateDateFormatFactory formatFactory =
JavaTemplateDateFormatFactory.INSTANCE;
+ formatFactory.clear();
+ int expectedMapSize = 0;
+ for (int i = 0; i <
JavaTemplateDateFormatFactory.GUARANTEED_RECENT_ENTRIES * 3 * 2 + 1; i++) {
+ assertEquals(expectedMapSize, formatFactory.getSize());
+ formatFactory.get(
+ "yyyyMMddHHmm '" + i + "'",
+ TemplateDateModel.DATETIME, env.getLocale(),
env.getTimeZone(), false, env);
+ expectedMapSize++;
+ // Following builds on implementation details (not on API
contract), so adjust this if that changes.
+ if (expectedMapSize >
JavaTemplateDateFormatFactory.GUARANTEED_RECENT_ENTRIES * 2) {
+ // The flushed entries are kept, and just become "old"
entries, therefore after flushing we have
+ // GUARANTEED_RECENT_ENTRIES entries. Then the new entry that
caused the flushing is added to the
+ // "recent" entries, hence the +1.
+ expectedMapSize =
JavaTemplateDateFormatFactory.GUARANTEED_RECENT_ENTRIES + 1;
+ }
+ }
+ }
+
+ @Test
+ public void testGlobalNumberFormatCacheLeakProtection() throws Exception {
+ JavaTemplateNumberFormatFactory formatFactory =
JavaTemplateNumberFormatFactory.INSTANCE;
+ formatFactory.clear();
+ int expectedMapSize = 0;
+ for (int i = 0; i <
JavaTemplateNumberFormatFactory.GUARANTEED_RECENT_ENTRIES * 3 * 2 + 1; i++) {
+ assertEquals(expectedMapSize, formatFactory.getSize());
+ formatFactory.get(
+ "#.# '" + i + "'",
+ env.getLocale(), env);
+ expectedMapSize++;
+ // Following builds on implementation details (not on API
contract), so adjust this if that changes.
+ if (expectedMapSize >
JavaTemplateNumberFormatFactory.GUARANTEED_RECENT_ENTRIES * 2) {
+ // The flushed entries are kept, and just become "old"
entries, therefore after flushing we have
+ // GUARANTEED_RECENT_ENTRIES entries. Then the new entry that
caused the flushing is added to the
+ // "recent" entries, hence the +1.
+ expectedMapSize =
JavaTemplateNumberFormatFactory.GUARANTEED_RECENT_ENTRIES + 1;
+ }
+ }
+ }
+
+ @Test
+ public void testEnvironmentLevelByFormatCacheMemoryLeakProtection() throws
Exception {
+ env.clearCachedTemplateTemporalFormatsByFormatString();
+ int expectedMapSize = 0;
+ for (int i = 0; i <
Environment.CACHED_TEMPORAL_FORMATS_BY_FORMAT_STRING_MAX_SIZE_PER_CLASS * 3 +
1; i++) {
+ assertEquals(expectedMapSize,
env.getTemplateTemporalFormatsByFormatStringMapSize(Instant.class));
+ env.getTemplateTemporalFormat("yyyyMMddHHmm '" + i + "'",
Instant.class);
+ expectedMapSize++;
+ if (expectedMapSize >
Environment.CACHED_TEMPORAL_FORMATS_BY_FORMAT_STRING_MAX_SIZE_PER_CLASS) {
+ expectedMapSize = 1; // Because we should store the new entry
that caused the flushing
+ }
+ }
+ }
+}
