Updated Branches: refs/heads/develop 6346d314a -> c3d897a95
- clean up caching (MARMOTTA-369), could also fix some of the concurrency issues - use string interning in critical parts Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/c8616cf6 Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/c8616cf6 Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/c8616cf6 Branch: refs/heads/develop Commit: c8616cf66690cbd1c5b780709c475f3f2b5c7c88 Parents: 1cb7624 Author: Sebastian Schaffert <[email protected]> Authored: Fri Nov 22 12:56:57 2013 +0100 Committer: Sebastian Schaffert <[email protected]> Committed: Fri Nov 22 12:56:57 2013 +0100 ---------------------------------------------------------------------- .../commons/sesame/model/LiteralKey.java | 4 +- .../kiwi/loader/generic/KiWiHandler.java | 2 +- .../marmotta/kiwi/caching/KiWiCacheManager.java | 13 -- .../marmotta/kiwi/sail/KiWiValueFactory.java | 128 +++++++------------ .../src/main/resources/ehcache-kiwi.xml | 9 -- .../src/test/resources/logback.xml | 3 + .../src/test/resources/logback.xml | 1 + 7 files changed, 54 insertions(+), 106 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/c8616cf6/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/LiteralKey.java ---------------------------------------------------------------------- diff --git a/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/LiteralKey.java b/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/LiteralKey.java index 3217c7e..5170e0d 100644 --- a/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/LiteralKey.java +++ b/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/LiteralKey.java @@ -32,8 +32,8 @@ public class LiteralKey { public LiteralKey(Object value, String type, String lang) { this.value = value; - this.type = type; - this.lang = lang; + this.type = type != null ? type.intern() : null; + this.lang = lang != null ? lang.intern() : null; } public String getLang() { http://git-wip-us.apache.org/repos/asf/marmotta/blob/c8616cf6/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java index f100791..46518fc 100644 --- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java +++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java @@ -254,7 +254,7 @@ public class KiWiHandler implements RDFHandler { protected KiWiLiteral createLiteral(Literal l) throws ExecutionException { String value = l.getLabel(); - String lang = l.getLanguage(); + String lang = l.getLanguage() != null ? l.getLanguage().intern() : null; URI type = l.getDatatype(); http://git-wip-us.apache.org/repos/asf/marmotta/blob/c8616cf6/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java index 38be64c..3ea69f8 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java @@ -137,19 +137,6 @@ public class KiWiCacheManager { /** - * Return the triple query cache, which caches (smaller) listTriples results. This cache is used fairly - * often, but since the cache elements will often be bigger collections of results, it should not be too - * big (default 10.000 elements). Since the underlying database might change, cache entries should expire - * after a certain time (default 1 hour). - * - * @return an EHCache Cache instance containing the query pattern -> query result mappings - */ - public KiWiQueryCache getQueryCache() { - return queryCache; - } - - - /** * Clear all caches managed by this cache manager. */ public void clear() { http://git-wip-us.apache.org/repos/asf/marmotta/blob/c8616cf6/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java index 33acd4c..c615f69 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java @@ -17,10 +17,9 @@ */ package org.apache.marmotta.kiwi.sail; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import org.apache.marmotta.commons.locking.ObjectLocks; +import net.sf.ehcache.Element; +import net.sf.ehcache.constructs.blocking.CacheEntryFactory; +import net.sf.ehcache.constructs.blocking.SelfPopulatingCache; import org.apache.marmotta.commons.sesame.model.LiteralCommons; import org.apache.marmotta.commons.sesame.model.LiteralKey; import org.apache.marmotta.commons.sesame.model.Namespaces; @@ -46,8 +45,6 @@ import java.util.IllformedLocaleException; import java.util.Locale; import java.util.Random; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; /** * Add file description here! @@ -60,10 +57,6 @@ public class KiWiValueFactory implements ValueFactory { private Random anonIdGenerator; - private ObjectLocks uriLocks; - private ObjectLocks literalLocks; - private ObjectLocks bnodeLocks; - /** * This is a hash map for storing references to resources that have not yet been persisted. It is used e.g. when * one or more transactions are currently active and request the creation of same resource several times @@ -85,9 +78,9 @@ public class KiWiValueFactory implements ValueFactory { private String defaultContext; - private LoadingCache<String,KiWiUriResource> uriCache; - private LoadingCache<String,KiWiAnonResource> bnodeCache; - private LoadingCache<LiteralKey, KiWiLiteral> literalCache; + protected SelfPopulatingCache literalCache; + protected SelfPopulatingCache uriCache; + protected SelfPopulatingCache bnodeCache; public KiWiValueFactory(KiWiStore store, String defaultContext) { anonIdGenerator = new Random(); @@ -96,58 +89,27 @@ public class KiWiValueFactory implements ValueFactory { this.store = store; this.defaultContext = defaultContext; - this.uriLocks = new ObjectLocks(); - this.bnodeLocks = new ObjectLocks(); - this.literalLocks = new ObjectLocks(); - - this.uriCache = CacheBuilder.newBuilder() - .maximumSize(store.getPersistence().getConfiguration().getUriCacheSize()) - .expireAfterAccess(60, TimeUnit.MINUTES) - .concurrencyLevel(10) - .build(new CacheLoader<String, KiWiUriResource>() { - @Override - public KiWiUriResource load(String key) throws Exception { - uriLocks.lock(key); - try { - return createURIInternal(key); - } finally { - uriLocks.unlock(key); - } - } - }); - - - this.bnodeCache = CacheBuilder.newBuilder() - .maximumSize(store.getPersistence().getConfiguration().getBNodeCacheSize()) - .expireAfterAccess(60, TimeUnit.MINUTES) - .concurrencyLevel(10) - .build(new CacheLoader<String, KiWiAnonResource>() { - @Override - public KiWiAnonResource load(String key) throws Exception { - bnodeLocks.lock(key); - try { - return createBNodeInternal(key); - } finally { - bnodeLocks.unlock(key); - } - } - }); - - this.literalCache = CacheBuilder.newBuilder() - .maximumSize(store.getPersistence().getConfiguration().getLiteralCacheSize()) - .expireAfterAccess(60, TimeUnit.MINUTES) - .concurrencyLevel(10) - .build(new CacheLoader<LiteralKey, KiWiLiteral>() { - @Override - public KiWiLiteral load(LiteralKey key) throws Exception { - literalLocks.lock(key); - try { - return createLiteralInternal(key); - } finally { - literalLocks.unlock(key); - } - } - }); + this.literalCache = new SelfPopulatingCache(store.getPersistence().getCacheManager().getLoaderCache(), new CacheEntryFactory() { + @Override + public Object createEntry(Object key) throws Exception { + return createLiteralInternal((LiteralKey) key); + } + }); + + this.uriCache = new SelfPopulatingCache(store.getPersistence().getCacheManager().getLoaderCache(), new CacheEntryFactory() { + @Override + public Object createEntry(Object key) throws Exception { + return createURIInternal(key.toString().intern()); + } + }); + + this.bnodeCache = new SelfPopulatingCache(store.getPersistence().getCacheManager().getLoaderCache(), new CacheEntryFactory() { + @Override + public Object createEntry(Object key) throws Exception { + return createBNodeInternal(key.toString().intern()); + } + }); + } protected KiWiConnection aqcuireConnection() { @@ -187,11 +149,12 @@ public class KiWiValueFactory implements ValueFactory { */ @Override public URI createURI(String uri) { - try { - return uriCache.get(uri); - } catch (ExecutionException e) { - log.error("could not load URI resource",e.getCause()); - throw new IllegalStateException("database error, could not load URI resource",e.getCause()); + Element e = uriCache.get(uri); + if(e != null) { + return (KiWiUriResource) e.getObjectValue(); + } else { + log.error("could not load URI resource"); + throw new IllegalStateException("database error, could not load URI resource"); } } @@ -249,11 +212,12 @@ public class KiWiValueFactory implements ValueFactory { */ @Override public BNode createBNode(String nodeID) { - try { - return bnodeCache.get(nodeID); - } catch (ExecutionException e) { - log.error("could not load BNode resource",e.getCause()); - throw new IllegalStateException("database error, could not load anonymous resource",e.getCause()); + Element e = bnodeCache.get(nodeID); + if(e != null) { + return (KiWiAnonResource) e.getObjectValue(); + } else { + log.error("could not load BNode resource"); + throw new IllegalStateException("database error, could not load anonymous resource"); } } @@ -374,13 +338,15 @@ public class KiWiValueFactory implements ValueFactory { // FIXME: MARMOTTA-39 (no default datatype before RDF-1.1) // type = LiteralCommons.getXSDType(value.getClass()); } - LiteralKey lkey = new LiteralKey(value,type,lang); + LiteralKey lkey = new LiteralKey(value,type, lang != null ? lang.intern() : null); - try { - return literalCache.get(lkey); - } catch (ExecutionException e) { - log.error("could not load Literal value",e.getCause()); - throw new IllegalStateException("database error, could not load literal value",e.getCause()); + + Element e = literalCache.get(lkey); + if(e != null) { + return (KiWiLiteral) e.getObjectValue(); + } else { + log.error("could not load Literal value"); + throw new IllegalStateException("database error, could not load literal value"); } } http://git-wip-us.apache.org/repos/asf/marmotta/blob/c8616cf6/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml b/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml index 44b586b..ccbd3fe 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml @@ -715,15 +715,6 @@ Here is an example of CacheManager level resource tuning, which will use up to 4 timeToLiveSeconds="3600" memoryStoreEvictionPolicy="LFU"/> - <!-- - the cache used for triple queries (getStatements) by the KiWi triplestore; this can be helpful if the same - query is repeated very often, but needs not be very big - --> - <cache name="query-cache" - statistics="true" - maxBytesLocalHeap="10%" - timeToLiveSeconds="3600" - overflowToDisk="false"/> <!-- a cache from URI to KiWiUriResource --> <cache name="uri-cache" http://git-wip-us.apache.org/repos/asf/marmotta/blob/c8616cf6/libraries/kiwi/kiwi-triplestore/src/test/resources/logback.xml ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/test/resources/logback.xml b/libraries/kiwi/kiwi-triplestore/src/test/resources/logback.xml index 1bfecff..9678d31 100644 --- a/libraries/kiwi/kiwi-triplestore/src/test/resources/logback.xml +++ b/libraries/kiwi/kiwi-triplestore/src/test/resources/logback.xml @@ -21,6 +21,9 @@ <pattern>%d{HH:mm:ss.SSS} %highlight(%level) %cyan(%logger{15}) - %m%n</pattern> </encoder> </appender> + + <logger name="net.sf.ehcache.pool.impl" level="WARN" /> + <root level="${root-level:-INFO}"> <appender-ref ref="CONSOLE"/> </root> http://git-wip-us.apache.org/repos/asf/marmotta/blob/c8616cf6/platform/marmotta-core/src/test/resources/logback.xml ---------------------------------------------------------------------- diff --git a/platform/marmotta-core/src/test/resources/logback.xml b/platform/marmotta-core/src/test/resources/logback.xml index 4192b44..455ca58 100644 --- a/platform/marmotta-core/src/test/resources/logback.xml +++ b/platform/marmotta-core/src/test/resources/logback.xml @@ -30,6 +30,7 @@ <logger name="org.apache.marmotta.platform.core" level="INFO" /> <logger name="org.apache.marmotta.platform.reasoner" level="INFO" /> + <logger name="net.sf.ehcache.pool.impl" level="WARN" /> <root level="info"> <appender-ref ref="STDOUT" />
