Repository: incubator-atlas Updated Branches: refs/heads/master d204df78e -> 37c8a4d1a
Cache of compiled DSL queries Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/37c8a4d1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/37c8a4d1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/37c8a4d1 Branch: refs/heads/master Commit: 37c8a4d1a57aabec88d5ecee60df6f562caaf8b8 Parents: d204df7 Author: Jeff Hagelberg <[email protected]> Authored: Mon Jan 30 14:31:53 2017 +0530 Committer: Vimal Sharma <[email protected]> Committed: Mon Jan 30 14:33:28 2017 +0530 ---------------------------------------------------------------------- common/pom.xml | 1 - .../java/org/apache/atlas/utils/LruCache.java | 97 ++++++++ .../org/apache/atlas/utils/LruCacheTest.java | 233 +++++++++++++++++++ distro/src/conf/atlas-application.properties | 14 ++ release-log.txt | 1 + .../graph/GraphBackedDiscoveryService.java | 93 +++++--- .../util/AtlasRepositoryConfiguration.java | 48 +++- .../atlas/util/CompiledQueryCacheKey.java | 87 +++++++ .../org/apache/atlas/util/NoopGremlinQuery.java | 39 ++++ .../org/apache/atlas/query/QueryProcessor.scala | 35 ++- .../atlas/util/CompiledQueryCacheKeyTest.java | 86 +++++++ 11 files changed, 686 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/common/pom.xml ---------------------------------------------------------------------- diff --git a/common/pom.xml b/common/pom.xml index 0226541..5fe5d57 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -70,7 +70,6 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${guava.version}</version> - <scope>test</scope> </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/common/src/main/java/org/apache/atlas/utils/LruCache.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/utils/LruCache.java b/common/src/main/java/org/apache/atlas/utils/LruCache.java new file mode 100644 index 0000000..dcaffef --- /dev/null +++ b/common/src/main/java/org/apache/atlas/utils/LruCache.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.utils; + +import java.text.DateFormat; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Fixed size LRU Cache. + * + */ +public class LruCache<K, V> extends LinkedHashMap<K, V>{ + + private static final long serialVersionUID = 8715233786643882558L; + + private static final Logger LOGGER = LoggerFactory.getLogger(LruCache.class.getName()); + + /** + * Specifies the number evictions that pass before a warning is logged. + */ + private final int evictionWarningThrottle; + + // The number of evictions since the last warning was logged. + private long evictionsSinceWarning = 0; + + // When the last eviction warning was issued. + private Date lastEvictionWarning = new Date(); + + // The maximum number of entries the cache holds. + private final int capacity; + + + /** + * + * @param cacheSize The size of the cache. + * @param evictionWarningThrottle The number evictions that pass before a warning is logged. + */ + public LruCache(int cacheSize, int evictionWarningThrottle) { + super(cacheSize, 0.75f, true); + this.evictionWarningThrottle = evictionWarningThrottle; + this.capacity = cacheSize; + } + + @Override + protected boolean removeEldestEntry(Map.Entry<K,V> eldest) { + if( size() > capacity) { + evictionWarningIfNeeded(); + return true; + } + return false; + } + + /** + * Logs a warning if a threshold number of evictions has occurred since the + * last warning. + */ + private void evictionWarningIfNeeded() { + // If not logging eviction warnings, just return. + if (evictionWarningThrottle <= 0) { + return; + } + + evictionsSinceWarning++; + + if (evictionsSinceWarning >= evictionWarningThrottle) { + DateFormat dateFormat = DateFormat.getDateTimeInstance(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("There have been " + evictionsSinceWarning + + " evictions from the cache since " + + dateFormat.format(lastEvictionWarning)); + } + evictionsSinceWarning = 0; + lastEvictionWarning = new Date(); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java b/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java new file mode 100644 index 0000000..24d62f5 --- /dev/null +++ b/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.utils; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.RandomStringUtils; +import org.testng.annotations.Test; + +/** + * Tests the LruCache. + */ +@Test +public class LruCacheTest { + + /** + * Tests the basic operations on the cache. + */ + @Test + public void testBasicOps() throws Exception { + + LruCache<String, String> cache = new LruCache<>(1000, 0); + // Get the static cache and populate it. Its size and other + // characteristics depend on the bootstrap properties that are hard to + // control in a test. So it is hard to see that if we add more entries + // than the size of the cache one is evicted, or that it gets reaped at + // the right time. However, a lot of this type of functionality is + // tested by the underlying LruCache's test. + + // Note that query handle IDs are of the form sessionID::queryID + String h1 = createHandle("s1::", "1::"); + String q1 = createQuery(); + + String h2 = createHandle("s1::", "2::"); + String q2 = createQuery(); + + String h3 = createHandle("s2::", "1::"); + String q3 = createQuery(); + + String h4 = createHandle("s1::", "3::"); + String q4 = createQuery(); + + String h5 = createHandle("s3::", null); + String q5 = createQuery(); + + String h5b = createHandle("s3::", null); + String q5b = createQuery(); + + String h6 = createHandle(null, "3::"); + String q6 = createQuery(); + + String h6b = createHandle(null, "3::"); + String q6b = createQuery(); + + // Test put and get. + cache.put(h1, q1); + cache.put(h2, q2); + cache.put(h3, q3); + cache.put(h4, q4); + cache.put(h5, q5); + cache.put(h6, q6); + + assertEquals(cache.get(h1), q1); + assertEquals(cache.get(h2), q2); + assertEquals(cache.get(h3), q3); + assertEquals(cache.get(h4), q4); + assertEquals(cache.get(h5), q5); + + assertEquals(cache.remove(h1), q1); + assertEquals(cache.remove(h2), q2); + assertEquals(cache.remove(h3), q3); + assertEquals(cache.remove(h4), q4); + assertEquals(cache.remove(h5), q5); + assertNull(cache.remove(h5b)); + assertEquals(q6, cache.remove(h6)); + assertNull(cache.remove(h6b)); + + cache.put(h5b, q5b); + cache.put(h6b, q6b); + + assertEquals(q5b, cache.remove(h5)); + assertNull(cache.remove(h5b)); + assertEquals(q6b, cache.remove(h6)); + assertNull(cache.remove(h6b)); + } + + @Test + public void testMapOperations() { + + Map<String, String> reference = new HashMap<>(); + reference.put("name", "Fred"); + reference.put("occupation", "student"); + reference.put("height", "5'11"); + reference.put("City", "Littleton"); + reference.put("State", "MA"); + + LruCache<String, String> map = new LruCache<>(10, 10); + map.putAll(reference); + + assertEquals(map.size(), reference.size()); + assertEquals(map.keySet().size(), reference.keySet().size()); + assertTrue(map.keySet().containsAll(reference.keySet())); + assertTrue(reference.keySet().containsAll(map.keySet())); + + assertEquals(reference.entrySet().size(), map.entrySet().size()); + for(Map.Entry<String, String> entry : map.entrySet()) { + assertTrue(reference.containsKey(entry.getKey())); + assertEquals(entry.getValue(), reference.get(entry.getKey())); + assertTrue(map.containsKey(entry.getKey())); + assertTrue(map.containsValue(entry.getValue())); + assertTrue(map.values().contains(entry.getValue())); + } + assertTrue(reference.equals(map)); + assertTrue(map.equals(reference)); + + } + + @Test + public void testReplaceValueInMap() { + LruCache<String, String> map = new LruCache<>(10, 10); + map.put("name", "Fred"); + map.put("name", "George"); + + assertEquals(map.get("name"), "George"); + assertEquals(map.size(), 1); + } + + + + @Test + public void testOrderUpdatedWhenAddExisting() { + LruCache<String, String> map = new LruCache<>(2, 10); + map.put("name", "Fred"); + map.put("age", "15"); + map.put("name", "George"); + + //age should be evicted + map.put("height", "5'3\""); + //age is now least recently used + assertFalse(map.containsKey("age")); + } + + @Test + public void testMapRemove() { + LruCache<String, String> map = new LruCache<>(10, 10); + map.put("name", "Fred"); + map.put("occupation", "student"); + map.put("height", "5'11"); + map.put("City", "Littleton"); + map.put("State", "MA"); + assertMapHasSize(map, 5); + assertTrue(map.containsKey("State")); + map.remove("State"); + assertMapHasSize(map, 4); + assertFalse(map.containsKey("State")); + + } + + private void assertMapHasSize(LruCache<String, String> map, int size) { + assertEquals(map.size(), size); + assertEquals(map.keySet().size(), size); + assertEquals(map.values().size(), size); + assertEquals(map.entrySet().size(), size); + } + + @Test + public void testEvict() { + LruCache<String, String> map = new LruCache<>(5, 10); + map.put("name", "Fred"); + map.put("occupation", "student"); + map.put("height", "5'11"); + map.put("City", "Littleton"); + map.put("State", "MA"); + assertMapHasSize(map, 5); + + //name should be evicted next + assertTrue(map.containsKey("name")); + map.put("zip", "01460"); + assertFalse(map.containsKey("name")); + assertMapHasSize(map, 5); + + map.get("occupation"); + //height should be evicted next + assertTrue(map.containsKey("height")); + map.put("country", "USA"); + assertFalse(map.containsKey("height")); + assertMapHasSize(map, 5); + } + + /** + * Create a fake query handle for testing. + * + * @param queryPrefix + * @param pkgPrefix + * @return a new query handle. + */ + private String createHandle(String s1, String s2) { + return s1 + ": " + s2 + ":select x from x in y"; + } + + /** + * Create a mock IInternalQuery. + * + * @return a mock IInternalQuery. + * @throws QueryException + */ + private String createQuery() { + return RandomStringUtils.randomAlphabetic(10); + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/distro/src/conf/atlas-application.properties ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index 303ce7b..d9e2f6e 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -218,3 +218,17 @@ atlas.metric.query.cache.ttlInSecs=900 #atlas.metric.query.entity.entityTagged= # #atlas.metric.query.tags.entityTags= + +######### Compiled Query Cache Configuration ######### + +# The size of the compiled query cache. Older queries will be evicted from the cache +# when we reach the capacity. + +#atlas.CompiledQueryCache.capacity=1000 + +# Allows notifications when items are evicted from the compiled query +# cache because it has become full. A warning will be issued when +# the specified number of evictions have occurred. If the eviction +# warning threshold <= 0, no eviction warnings will be issued. + +#atlas.CompiledQueryCache.evictionWarningThrottle=0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b04dbb8..1d9b61e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1387 Compiled Query Cache ([email protected] via svimal2106) ATLAS-1312 Update QuickStart to use the v2 APIs for types and entities creation ([email protected] via mneethiraj) ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj) ATLAS-1114 Performance improvements for create/update entity (jnhagelb) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java index fb488cd..f84f405 100755 --- a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java @@ -18,6 +18,17 @@ package org.apache.atlas.discovery.graph; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.script.ScriptException; + import org.apache.atlas.AtlasClient; import org.apache.atlas.GraphTransaction; import org.apache.atlas.discovery.DiscoveryException; @@ -38,24 +49,17 @@ import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.util.CompiledQueryCacheKey; +import org.apache.atlas.util.NoopGremlinQuery; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.util.Either; import scala.util.parsing.combinator.Parsers; -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.script.ScriptException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - /** * Graph backed implementation of Search. */ @@ -124,42 +128,57 @@ public class GraphBackedDiscoveryService implements DiscoveryService { } public GremlinQueryResult evaluate(String dslQuery, QueryParams queryParams) throws DiscoveryException { - - if (LOG.isDebugEnabled()) { + if(LOG.isDebugEnabled()) { LOG.debug("Executing dsl query={}", dslQuery); } - try { - Either<Parsers.NoSuccess, Expressions.Expression> either = QueryParser.apply(dslQuery, queryParams); - if (either.isRight()) { - Expressions.Expression expression = either.right().get(); - return evaluate(dslQuery, expression); - } else { - throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left()); + GremlinQuery gremlinQuery = parseAndTranslateDsl(dslQuery, queryParams); + if(gremlinQuery instanceof NoopGremlinQuery) { + return new GremlinQueryResult(dslQuery, ((NoopGremlinQuery)gremlinQuery).getDataType(), Collections.emptyList()); } + + return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate(); + } catch (Exception e) { // unable to catch ExpressionException throw new DiscoveryException("Invalid expression : " + dslQuery, e); } } - private GremlinQueryResult evaluate(String dslQuery, Expressions.Expression expression) { - Expressions.Expression validatedExpression = QueryProcessor.validate(expression); + private GremlinQuery parseAndTranslateDsl(String dslQuery, QueryParams queryParams) throws DiscoveryException { - //If the final limit is 0, don't launch the query, return with 0 rows - if (validatedExpression instanceof Expressions.LimitExpression - && ((Integer)((Expressions.LimitExpression) validatedExpression).limit().rawValue()) == 0) { - return new GremlinQueryResult(dslQuery, validatedExpression.dataType(), Collections.emptyList()); - } + CompiledQueryCacheKey entry = new CompiledQueryCacheKey(dslQuery, queryParams); + GremlinQuery gremlinQuery = QueryProcessor.compiledQueryCache().get(entry); + if(gremlinQuery == null) { + Expressions.Expression validatedExpression = parseQuery(dslQuery, queryParams); - GremlinQuery gremlinQuery = new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate(); + //If the final limit is 0, don't launch the query, return with 0 rows + if (validatedExpression instanceof Expressions.LimitExpression + && ((Integer)((Expressions.LimitExpression) validatedExpression).limit().rawValue()) == 0) { + gremlinQuery = new NoopGremlinQuery(validatedExpression.dataType()); + } + else { + gremlinQuery = new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate(); + if (LOG.isDebugEnabled()) { + LOG.debug("Query = {}", validatedExpression); + LOG.debug("Expression Tree = {}", validatedExpression.treeString()); + LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr()); + } + } + QueryProcessor.compiledQueryCache().put(entry, gremlinQuery); + } + return gremlinQuery; + } - if (LOG.isDebugEnabled()) { - LOG.debug("Query = {}", validatedExpression); - LOG.debug("Expression Tree = {}", validatedExpression.treeString()); - LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr()); + private Expressions.Expression parseQuery(String dslQuery, QueryParams queryParams) throws DiscoveryException { + Either<Parsers.NoSuccess, Expressions.Expression> either = QueryParser.apply(dslQuery, queryParams); + if (either.isRight()) { + Expressions.Expression expression = either.right().get(); + Expressions.Expression validatedExpression = QueryProcessor.validate(expression); + return validatedExpression; + } else { + throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left()); } - return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate(); } /** @@ -182,12 +201,12 @@ public class GraphBackedDiscoveryService implements DiscoveryService { throw new DiscoveryException(se); } } - + private List<Map<String, String>> extractResult(final Object o) throws DiscoveryException { List<Map<String, String>> result = new ArrayList<>(); if (o instanceof List) { List l = (List) o; - + for (Object value : l) { Map<String, String> oRow = new HashMap<>(); if (value instanceof Map) { @@ -205,7 +224,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService { oRow.put(key, propertyValue.toString()); } } - + } else if (value instanceof String) { oRow.put("", value.toString()); } else if(value instanceof AtlasEdge) { @@ -220,14 +239,14 @@ public class GraphBackedDiscoveryService implements DiscoveryService { } else { throw new DiscoveryException(String.format("Cannot process result %s", String.valueOf(value))); } - + result.add(oRow); } } else { result.add(new HashMap<String, String>() {{ put("result", o.toString()); - }}); + }}); } return result; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java index 71c7ff8..a04dd95 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java @@ -43,6 +43,9 @@ public class AtlasRepositoryConfiguration { private static Logger LOG = LoggerFactory.getLogger(AtlasRepositoryConfiguration.class); + public static final int DEFAULT_COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE = 0; + public static final int DEFAULT_COMPILED_QUERY_CACHE_CAPACITY = 1000; + public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl"; public static final String AUDIT_EXCLUDED_OPERATIONS = "atlas.audit.excludes"; private static List<String> skippedOperations = null; @@ -70,7 +73,7 @@ public class AtlasRepositoryConfiguration { public static Class<? extends EntityAuditRepository> getAuditRepositoryImpl() { try { Configuration config = ApplicationProperties.get(); - return ApplicationProperties.getClass(config, + return ApplicationProperties.getClass(config, AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); } catch (AtlasException e) { throw new RuntimeException(e); @@ -83,7 +86,7 @@ public class AtlasRepositoryConfiguration { public static Class<? extends DeleteHandler> getDeleteHandlerImpl() { try { Configuration config = ApplicationProperties.get(); - return ApplicationProperties.getClass(config, + return ApplicationProperties.getClass(config, DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class); } catch (AtlasException e) { throw new RuntimeException(e); @@ -99,15 +102,50 @@ public class AtlasRepositoryConfiguration { throw new RuntimeException(e); } } - + + public static final String COMPILED_QUERY_CACHE_CAPACITY = "atlas.CompiledQueryCache.capacity"; + + /** + * Get the configuration property that specifies the size of the compiled query + * cache. This is an optional property. A default is used if it is not + * present. + * + * @return the size to be used when creating the compiled query cache. + */ + public static int getCompiledQueryCacheCapacity() { + try { + return ApplicationProperties.get().getInt(COMPILED_QUERY_CACHE_CAPACITY, DEFAULT_COMPILED_QUERY_CACHE_CAPACITY); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } + + public static final String COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE = "atlas.CompiledQueryCache.evictionWarningThrottle"; + + /** + * Get the configuration property that specifies the number evictions that pass + * before a warning is logged. This is an optional property. A default is + * used if it is not present. + * + * @return the number of evictions before a warning is logged. + */ + public static int getCompiledQueryCacheEvictionWarningThrottle() { + try { + return ApplicationProperties.get().getInt(COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE, DEFAULT_COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } + + private static final String GRAPH_DATABASE_IMPLEMENTATION_PROPERTY = "atlas.graphdb.backend"; private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase"; - + @SuppressWarnings("unchecked") public static Class<? extends GraphDatabase> getGraphDatabaseImpl() { try { Configuration config = ApplicationProperties.get(); - return ApplicationProperties.getClass(config, + return ApplicationProperties.getClass(config, GRAPH_DATABASE_IMPLEMENTATION_PROPERTY, DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS, GraphDatabase.class); } catch (AtlasException e) { throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/repository/src/main/java/org/apache/atlas/util/CompiledQueryCacheKey.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/CompiledQueryCacheKey.java b/repository/src/main/java/org/apache/atlas/util/CompiledQueryCacheKey.java new file mode 100644 index 0000000..56a5a2a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/CompiledQueryCacheKey.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +import org.apache.atlas.query.QueryParams; + +/** + * Represents a key for an entry in the compiled query cache. + * + */ +public class CompiledQueryCacheKey { + + private final String dslQuery; + private final QueryParams queryParams; + + public CompiledQueryCacheKey(String dslQuery, QueryParams queryParams) { + super(); + this.dslQuery = dslQuery; + this.queryParams = queryParams; + } + + public CompiledQueryCacheKey(String dslQuery) { + super(); + this.dslQuery = dslQuery; + this.queryParams = null; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((dslQuery == null) ? 0 : dslQuery.hashCode()); + result = prime * result + ((queryParams == null) ? 0 : queryParams.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + if (!(obj instanceof CompiledQueryCacheKey)) { + return false; + } + + CompiledQueryCacheKey other = (CompiledQueryCacheKey) obj; + if (! equals(dslQuery, other.dslQuery)) { + return false; + } + + if (! equals(queryParams, other.queryParams)) { + return false; + } + + return true; + } + + private static boolean equals(Object o1, Object o2) { + if(o1 == o2) { + return true; + } + if(o1 == null) { + return o2 == null; + } + return o1.equals(o2); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/repository/src/main/java/org/apache/atlas/util/NoopGremlinQuery.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/NoopGremlinQuery.java b/repository/src/main/java/org/apache/atlas/util/NoopGremlinQuery.java new file mode 100644 index 0000000..280570e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/NoopGremlinQuery.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +import org.apache.atlas.query.GremlinQuery; +import org.apache.atlas.typesystem.types.IDataType; + +/** + * Represents a query that we know will have no results. + * + */ +public class NoopGremlinQuery extends GremlinQuery { + + private final IDataType dataType; + + public NoopGremlinQuery(IDataType dataType) { + super(null, null, null); + this.dataType = dataType; + } + + public IDataType getDataType() { + return dataType; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala b/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala old mode 100644 new mode 100755 index 5693c9e..e1e8408 --- a/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala +++ b/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala @@ -21,10 +21,18 @@ package org.apache.atlas.query import org.apache.atlas.repository.graphdb.AtlasGraph import org.apache.atlas.query.Expressions._ import org.slf4j.{Logger, LoggerFactory} +import org.apache.atlas.util.AtlasRepositoryConfiguration +import org.apache.atlas.utils.LruCache +import org.apache.atlas.util.CompiledQueryCacheKey +import java.util.Collections object QueryProcessor { val LOG : Logger = LoggerFactory.getLogger("org.apache.atlas.query.QueryProcessor") + val compiledQueryCache = Collections.synchronizedMap(new LruCache[CompiledQueryCacheKey, GremlinQuery]( + AtlasRepositoryConfiguration.getCompiledQueryCacheCapacity(), + AtlasRepositoryConfiguration.getCompiledQueryCacheEvictionWarningThrottle())); + def evaluate(e: Expression, g: AtlasGraph[_,_], gP : GraphPersistenceStrategies = null): GremlinQueryResult = { @@ -33,11 +41,28 @@ object QueryProcessor { strategy = GraphPersistenceStrategy1(g); } - val e1 = validate(e) - val q = new GremlinTranslator(e1, strategy).translate() - LOG.debug("Query: " + e1) - LOG.debug("Expression Tree:\n" + e1.treeString) - LOG.debug("Gremlin Query: " + q.queryStr) + //convert the query expression to DSL so we can check whether or not it is in the compiled + //query cache and avoid validating/translating it again if it is. + val dsl = e.toString(); + val cacheKey = new CompiledQueryCacheKey(dsl); + var q = compiledQueryCache.get(cacheKey); + if(q == null) { + + //query was not found in the compiled query cache. Validate + //and translate it, then cache the result. + + val e1 = validate(e) + q = new GremlinTranslator(e1, strategy).translate() + compiledQueryCache.put(cacheKey, q); + if(LOG.isDebugEnabled()) { + LOG.debug("Validated Query: " + e1) + LOG.debug("Expression Tree:\n" + e1.treeString); + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("DSL Query: " + dsl); + LOG.debug("Gremlin Query: " + q.queryStr) + } new GremlinEvaluator(q, strategy, g).evaluate() } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/37c8a4d1/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java b/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java new file mode 100644 index 0000000..c926f4d --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java @@ -0,0 +1,86 @@ +package org.apache.atlas.util; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; + +import org.apache.atlas.query.QueryParams; +import org.testng.annotations.Test; + +/** + * Tests hashcode/equals behavior of CompiledQueryCacheKey + * + * + */ +public class CompiledQueryCacheKeyTest { + + @Test + public void testNoQueryParams() { + + + CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1"); + CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1"); + CompiledQueryCacheKey e3 = new CompiledQueryCacheKey("query 2"); + + assertKeysEqual(e1, e2); + assertKeysDifferent(e2, e3); + } + + + @Test + public void testWithQueryParams() { + + CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10)); + CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10)); + CompiledQueryCacheKey e3 = new CompiledQueryCacheKey("query 2", new QueryParams(10,10)); + + assertKeysEqual(e1, e2); + assertKeysDifferent(e2, e3); + } + + @Test + public void testOnlyQueryParamsDifferent() { + + + CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10)); + CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1", new QueryParams(20,10)); + + assertKeysDifferent(e1, e2); + } + + @Test + public void testOnlyDslDifferent() { + + + CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10)); + CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 2", new QueryParams(10,10)); + + assertKeysDifferent(e1, e2); + } + + + @Test + public void testMixOfQueryParamsAndNone() { + + + CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10)); + CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1"); + + assertKeysDifferent(e1, e2); + } + + + private void assertKeysEqual(CompiledQueryCacheKey e1, CompiledQueryCacheKey e2) { + + assertEquals(e1.hashCode(), e2.hashCode()); + assertEquals(e1, e2); + assertEquals(e2, e1); + } + + private void assertKeysDifferent(CompiledQueryCacheKey e1, CompiledQueryCacheKey e2) { + + assertNotSame(e1.hashCode(), e2.hashCode()); + assertNotSame(e1, e2); + assertNotSame(e2, e1); + } + +}
