Repository: incubator-atlas Updated Branches: refs/heads/master b1717f2e8 -> 48c10133e
ATLAS-1606: introduced query provider to handle Gremlin version specific queries Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/48c10133 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/48c10133 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/48c10133 Branch: refs/heads/master Commit: 48c10133e1b67ac727b0a858aa51801150ddb35e Parents: b1717f2 Author: apoorvnaik <[email protected]> Authored: Tue Feb 28 12:11:39 2017 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Feb 28 17:31:01 2017 -0800 ---------------------------------------------------------------------- .../atlas/discovery/EntityLineageService.java | 47 ++++------- .../apache/atlas/services/MetricsService.java | 89 +++++++++++--------- .../atlas/util/AtlasGremlin2QueryProvider.java | 68 +++++++++++++++ .../atlas/util/AtlasGremlin3QueryProvider.java | 27 ++++++ .../atlas/util/AtlasGremlinQueryProvider.java | 57 +++++++++++++ .../atlas/services/MetricsServiceTest.java | 2 +- .../atlas/web/resources/ExportService.java | 30 ++++--- 7 files changed, 236 insertions(+), 84 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/48c10133/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 6b7d7d3..4a98920 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -31,6 +31,8 @@ import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.util.AtlasGremlinQueryProvider; +import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; import org.apache.commons.collections.CollectionUtils; import javax.inject.Inject; @@ -47,29 +49,12 @@ public class EntityLineageService implements AtlasLineageService { private static final String OUTPUT_PROCESS_EDGE = "__Process.outputs"; private final AtlasGraph graph; - - /** - * Gremlin query to retrieve input/output lineage for specified depth on a DataSet entity. - * return list of Atlas vertices paths. - */ - private static final String PARTIAL_LINEAGE_QUERY = "g.V('__guid', '%s').as('src').in('%s').out('%s')." + - "loop('src', {it.loops <= %s}, {((it.object.'__superTypeNames') ? " + - "(it.object.'__superTypeNames'.contains('DataSet')) : false)})." + - "path().toList()"; - - /** - * Gremlin query to retrieve all (no fixed depth) input/output lineage for a DataSet entity. - * return list of Atlas vertices paths. - */ - private static final String FULL_LINEAGE_QUERY = "g.V('__guid', '%s').as('src').in('%s').out('%s')." + - "loop('src', {((it.path.contains(it.object)) ? false : true)}, " + - "{((it.object.'__superTypeNames') ? " + - "(it.object.'__superTypeNames'.contains('DataSet')) : false)})." + - "path().toList()"; + private final AtlasGremlinQueryProvider gremlinQueryProvider; @Inject EntityLineageService() throws DiscoveryException { this.graph = AtlasGraphProvider.getGraphInstance(); + this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; } @Override @@ -157,23 +142,27 @@ public class EntityLineageService implements AtlasLineageService { String lineageQuery = null; if (direction.equals(LineageDirection.INPUT)) { - if (depth < 1) { - lineageQuery = String.format(FULL_LINEAGE_QUERY, entityGuid, OUTPUT_PROCESS_EDGE, INPUT_PROCESS_EDGE); - } else { - lineageQuery = String.format(PARTIAL_LINEAGE_QUERY, entityGuid, OUTPUT_PROCESS_EDGE, INPUT_PROCESS_EDGE, depth); - } + lineageQuery = generateLineageQuery(entityGuid, depth, OUTPUT_PROCESS_EDGE, INPUT_PROCESS_EDGE); } else if (direction.equals(LineageDirection.OUTPUT)) { - if (depth < 1) { - lineageQuery = String.format(FULL_LINEAGE_QUERY, entityGuid, INPUT_PROCESS_EDGE, OUTPUT_PROCESS_EDGE); - } else { - lineageQuery = String.format(PARTIAL_LINEAGE_QUERY, entityGuid, INPUT_PROCESS_EDGE, OUTPUT_PROCESS_EDGE, depth); - } + lineageQuery = generateLineageQuery(entityGuid, depth, INPUT_PROCESS_EDGE, OUTPUT_PROCESS_EDGE); } return lineageQuery; } + private String generateLineageQuery(String entityGuid, int depth, String incomingFrom, String outgoingTo) { + String lineageQuery; + if (depth < 1) { + String query = gremlinQueryProvider.getQuery(AtlasGremlinQuery.FULL_LINEAGE); + lineageQuery = String.format(query, entityGuid, incomingFrom, outgoingTo); + } else { + String query = gremlinQueryProvider.getQuery(AtlasGremlinQuery.PARTIAL_LINEAGE); + lineageQuery = String.format(query, entityGuid, incomingFrom, outgoingTo, depth); + } + return lineageQuery; + } + private AtlasEntityHeader toAtlasEntityHeader(Object vertexObj) { AtlasEntityHeader ret = new AtlasEntityHeader(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/48c10133/repository/src/main/java/org/apache/atlas/services/MetricsService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/MetricsService.java b/repository/src/main/java/org/apache/atlas/services/MetricsService.java index 7de9bd0..3ef0138 100644 --- a/repository/src/main/java/org/apache/atlas/services/MetricsService.java +++ b/repository/src/main/java/org/apache/atlas/services/MetricsService.java @@ -24,14 +24,14 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.util.AtlasGremlinQueryProvider; +import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.script.ScriptException; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -39,51 +39,54 @@ import java.util.Map; public class MetricsService { private static final Logger LOG = LoggerFactory.getLogger(MetricsService.class); - public static final String METRIC_QUERY_PREFIX = "atlas.metric.query."; - public static final String METRIC_QUERY_CACHE_TTL = "atlas.metric.query.cache.ttlInSecs"; - public static final int DEFAULT_CACHE_TTL_IN_SECS = 900; - + // Query Category constants public static final String TYPE = "type"; public static final String ENTITY = "entity"; public static final String TAG = "tag"; public static final String GENERAL = "general"; - public static final String METRIC_TYPE_COUNT = TYPE + "Count"; - public static final String METRIC_TYPE_UNUSED_COUNT = TYPE + "UnusedCount"; - public static final String METRIC_TYPE_ENTITIES = TYPE + "Entities"; + // Query names + protected static final String METRIC_TYPE_COUNT = TYPE + "Count"; + protected static final String METRIC_TYPE_UNUSED_COUNT = TYPE + "UnusedCount"; + protected static final String METRIC_TYPE_ENTITIES = TYPE + "Entities"; - public static final String METRIC_ENTITY_COUNT = ENTITY + "Count"; - public static final String METRIC_ENTITY_DELETED = ENTITY + "Deleted"; - public static final String METRIC_TAGGED_ENTITIES = ENTITY + "Tagged"; - public static final String METRIC_TAGS_PER_ENTITY = ENTITY + "Tags"; + protected static final String METRIC_ENTITY_COUNT = ENTITY + "Count"; + protected static final String METRIC_ENTITY_DELETED = ENTITY + "Deleted"; + protected static final String METRIC_TAGGED_ENTITIES = ENTITY + "Tagged"; + protected static final String METRIC_TAGS_PER_ENTITY = ENTITY + "Tags"; - public static final String METRIC_TAG_COUNT = TAG + "Count"; - public static final String METRIC_ENTITIES_PER_TAG = TAG + "Entities"; + protected static final String METRIC_TAG_COUNT = TAG + "Count"; + protected static final String METRIC_ENTITIES_PER_TAG = TAG + "Entities"; + + public static final String METRIC_QUERY_PREFIX = "atlas.metric.query."; + public static final String METRIC_QUERY_CACHE_TTL = "atlas.metric.query.cache.ttlInSecs"; + public static final int DEFAULT_CACHE_TTL_IN_SECS = 900; public static final String METRIC_COLLECTION_TIME = "collectionTime"; - private static Configuration configuration = null; - private final AtlasGraph atlasGraph; - private final AtlasTypeRegistry atlasTypeRegistry; - private final int cacheTTLInSecs; + private static Configuration configuration = null; + private static AtlasGremlinQueryProvider gremlinQueryProvider = null; + + private final AtlasGraph atlasGraph; + private final int cacheTTLInSecs; private AtlasMetrics cachedMetrics = null; private long cacheExpirationTime = 0; @Inject - public MetricsService(AtlasTypeRegistry typeRegistry) throws AtlasException { - this(ApplicationProperties.get(), AtlasGraphProvider.getGraphInstance(), typeRegistry); + public MetricsService() throws AtlasException { + this(ApplicationProperties.get(), AtlasGraphProvider.getGraphInstance()); } @VisibleForTesting - MetricsService(Configuration configuration, AtlasGraph graph, AtlasTypeRegistry typeRegistry) { + MetricsService(Configuration configuration, AtlasGraph graph) { MetricsService.configuration = configuration; - atlasTypeRegistry = typeRegistry; atlasGraph = graph; cacheTTLInSecs = configuration != null ? configuration.getInt(METRIC_QUERY_CACHE_TTL, DEFAULT_CACHE_TTL_IN_SECS) - : DEFAULT_CACHE_TTL_IN_SECS; + : DEFAULT_CACHE_TTL_IN_SECS; + gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; } @SuppressWarnings("unchecked") @@ -96,9 +99,13 @@ public class MetricsService { if (LOG.isDebugEnabled()) { LOG.debug("Executing query: {}", metricQuery); } - executeGremlinQuery(metrics, metricQuery.type, metricQuery.name, metricQuery.query); + executeGremlinQuery(metrics, metricQuery.group, metricQuery.name, metricQuery.query); } catch (ScriptException e) { - LOG.error("Gremlin execution failed for metric {}", metricQuery, e); + if (LOG.isDebugEnabled()) { + LOG.debug("Gremlin execution failed for metric {}", metricQuery, e); + } else { + LOG.warn("Gremlin execution failed for metric {}", metricQuery); + } } } @@ -145,7 +152,7 @@ public class MetricsService { private static String getQuery(String type, String name, String defaultQuery) { String ret = configuration != null ? configuration.getString(METRIC_QUERY_PREFIX + type + "." + name, defaultQuery) - : defaultQuery; + : defaultQuery; if (LOG.isDebugEnabled()) { LOG.debug("query for {}.{}: {}", type, name, ret); @@ -161,31 +168,31 @@ public class MetricsService { * query is not blank/empty. */ private enum MetricQuery { - TYPE_COUNT(GENERAL, METRIC_TYPE_COUNT, "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).count()"), - UNUSED_TYPE_COUNT(GENERAL, METRIC_TYPE_UNUSED_COUNT, "g.V('__type', 'typeSystem').filter({ it.'__type.category'.name() != 'TRAIT' && it.inE.count() == 0}).count()"), - ENTITY_COUNT(GENERAL, METRIC_ENTITY_COUNT, "g.V().has('__superTypeNames', T.in, ['Referenceable']).count()"), - TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"), - DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"), + TYPE_COUNT(GENERAL, METRIC_TYPE_COUNT, AtlasGremlinQuery.TYPE_COUNT_METRIC), + UNUSED_TYPE_COUNT(GENERAL, METRIC_TYPE_UNUSED_COUNT, AtlasGremlinQuery.TYPE_UNUSED_COUNT_METRIC), + ENTITY_COUNT(GENERAL, METRIC_ENTITY_COUNT, AtlasGremlinQuery.ENTITY_COUNT_METRIC), + TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, AtlasGremlinQuery.TAG_COUNT_METRIC), + DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, AtlasGremlinQuery.ENTITY_DELETED_METRIC), - ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"), - TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"), + ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, AtlasGremlinQuery.ENTITIES_PER_TYPE_METRIC), + TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, AtlasGremlinQuery.TAGGED_ENTITIES_METRIC), - ENTITIES_WITH_SPECIFIC_TAG(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.'__type.category'.name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"), + ENTITIES_WITH_SPECIFIC_TAG(TAG, METRIC_ENTITIES_PER_TAG, AtlasGremlinQuery.ENTITIES_FOR_TAG_METRIC), ; - private final String type; + private final String group; private final String name; private final String query; - MetricQuery(String type, String name, String query) { - this.type = type; - this.name = name; - this.query = MetricsService.getQuery(type, name, query); + MetricQuery(String group, String name, AtlasGremlinQuery gremlinQuery) { + this.group = group; + this.name = name; + this.query = MetricsService.getQuery(group, name, gremlinQueryProvider.getQuery(gremlinQuery)); } @Override public String toString() { - return "MetricQuery{" + "type='" + type + '\'' + + return "MetricQuery{" + "group='" + group + '\'' + ", name='" + name + '\'' + ", query='" + query + '\'' + '}'; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/48c10133/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java new file mode 100644 index 0000000..798ce38 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider { + @Override + public String getQuery(final AtlasGremlinQuery gremlinQuery) { + switch (gremlinQuery) { + case TYPE_COUNT_METRIC: + return "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).count()"; + case TYPE_UNUSED_COUNT_METRIC: + return "g.V('__type', 'typeSystem').filter({ it.'__type.category'.name() != 'TRAIT' && it.inE.count() == 0}).count()"; + case ENTITY_COUNT_METRIC: + return "g.V().has('__superTypeNames', T.in, ['Referenceable']).count()"; + case TAG_COUNT_METRIC: + return "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"; + case ENTITY_DELETED_METRIC: + return "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"; + case ENTITIES_PER_TYPE_METRIC: + return "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"; + case TAGGED_ENTITIES_METRIC: + return "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"; + case ENTITIES_FOR_TAG_METRIC: + return "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.'__type.category'.name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"; + case EXPORT_BY_GUID: + return "g.V('__guid', startGuid).bothE().bothV().has('__guid').__guid.dedup().toList()"; + case EXPORT_TYPE_STARTS_WITH: + return "g.V().has('__typeName','%s').filter({it.'%s'.startsWith(attrValue)}).has('__guid').__guid.toList()"; + case EXPORT_TYPE_ENDS_WITH: + return "g.V().has('__typeName','%s').filter({it.'%s'.endsWith(attrValue)}).has('__guid').__guid.toList()"; + case EXPORT_TYPE_CONTAINS: + return "g.V().has('__typeName','%s').filter({it.'%s'.contains(attrValue)}).has('__guid').__guid.toList()"; + case EXPORT_TYPE_MATCHES: + return "g.V().has('__typeName','%s').filter({it.'%s'.matches(attrValue)}).has('__guid').__guid.toList()"; + case EXPORT_TYPE_DEFAULT: + return "g.V().has('__typeName','%s').has('%s', attrValue).has('__guid').__guid.toList()"; + case FULL_LINEAGE: + return "g.V('__guid', '%s').as('src').in('%s').out('%s')." + + "loop('src', {((it.path.contains(it.object)) ? false : true)}, " + + "{((it.object.'__superTypeNames') ? " + + "(it.object.'__superTypeNames'.contains('DataSet')) : false)})." + + "path().toList()"; + case PARTIAL_LINEAGE: + return "g.V('__guid', '%s').as('src').in('%s').out('%s')." + + "loop('src', {it.loops <= %s}, {((it.object.'__superTypeNames') ? " + + "(it.object.'__superTypeNames'.contains('DataSet')) : false)})." + + "path().toList()"; + } + // Should never reach this point + return null; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/48c10133/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java new file mode 100644 index 0000000..9884569 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +public class AtlasGremlin3QueryProvider extends AtlasGremlin2QueryProvider { + @Override + public String getQuery(final AtlasGremlinQuery gremlinQuery) { + // In case any overrides are necessary, a specific switch case can be added here to + // return Gremlin 3 specific query otherwise delegate to super.getQuery + return super.getQuery(gremlinQuery); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/48c10133/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java new file mode 100644 index 0000000..ad22bf7 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graphdb.GremlinVersion; + +/** + * Generic Gremlin query provider which is agnostic of the Gremlin/TinkerPop version being used in Atlas + */ +public abstract class AtlasGremlinQueryProvider { + public static final AtlasGremlinQueryProvider INSTANCE = + AtlasGraphProvider.getGraphInstance().getSupportedGremlinVersion() == GremlinVersion.THREE ? + new AtlasGremlin3QueryProvider() : new AtlasGremlin2QueryProvider(); + + abstract public String getQuery(final AtlasGremlinQuery gremlinQuery); + + public enum AtlasGremlinQuery { + // Metrics related Queries + TYPE_COUNT_METRIC, + TYPE_UNUSED_COUNT_METRIC, + ENTITY_COUNT_METRIC, + TAG_COUNT_METRIC, + ENTITY_DELETED_METRIC, + ENTITIES_PER_TYPE_METRIC, + TAGGED_ENTITIES_METRIC, + ENTITIES_FOR_TAG_METRIC, + + // Import Export related Queries + EXPORT_BY_GUID, + EXPORT_TYPE_STARTS_WITH, + EXPORT_TYPE_ENDS_WITH, + EXPORT_TYPE_CONTAINS, + EXPORT_TYPE_MATCHES, + EXPORT_TYPE_DEFAULT, + + // Lineage Queries + FULL_LINEAGE, + PARTIAL_LINEAGE; + // Discovery Queries + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/48c10133/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java index cf85b2f..1531c9c 100644 --- a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java @@ -63,7 +63,7 @@ public class MetricsServiceTest { when(mockTypeRegistry.getAllEntityDefNames()).thenReturn(Arrays.asList("a", "b", "c")); setupMockGraph(); - metricsService = new MetricsService(mockConfig, mockGraph, mockTypeRegistry); + metricsService = new MetricsService(mockConfig, mockGraph); } private void setupMockGraph() throws ScriptException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/48c10133/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java b/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java index 1e98232..df8bf33 100644 --- a/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java @@ -37,6 +37,8 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.atlas.util.AtlasGremlinQueryProvider; +import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -47,7 +49,12 @@ import javax.script.Bindings; import javax.script.ScriptContext; import javax.script.ScriptEngine; import javax.script.ScriptException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class ExportService { @@ -62,21 +69,17 @@ public class ExportService { private final AtlasTypeRegistry typeRegistry; private final AtlasGraph atlasGraph; private final EntityGraphRetriever entityGraphRetriever; + private final AtlasGremlinQueryProvider gremlinQueryProvider; // query engine support private final ScriptEngine scriptEngine; private final Bindings bindings; - private final String queryByGuid = "g.V('__guid', startGuid).bothE().bothV().has('__guid').__guid.dedup().toList()"; - final private String queryByAttrEquals = "g.V().has('__typeName','%s').has('%s', attrValue).has('__guid').__guid.toList()"; - final private String queryByAttrStartWith = "g.V().has('__typeName','%s').filter({it.'%s'.startsWith(attrValue)}).has('__guid').__guid.toList()"; - final private String queryByAttrEndsWith = "g.V().has('__typeName','%s').filter({it.'%s'.endsWith(attrValue)}).has('__guid').__guid.toList()"; - final private String queryByAttrContains = "g.V().has('__typeName','%s').filter({it.'%s'.contains(attrValue)}).has('__guid').__guid.toList()"; - final private String queryByAttrMatches = "g.V().has('__typeName','%s').filter({it.'%s'.matches(attrValue)}).has('__guid').__guid.toList()"; public ExportService(final AtlasTypeRegistry typeRegistry) throws AtlasBaseException { this.typeRegistry = typeRegistry; this.entityGraphRetriever = new EntityGraphRetriever(this.typeRegistry); this.atlasGraph = AtlasGraphProvider.getGraphInstance(); + this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; this.scriptEngine = new GremlinGroovyScriptEngine(); @@ -186,15 +189,15 @@ public class ExportService { final String queryTemplate; if (StringUtils.equalsIgnoreCase(matchType, MATCH_TYPE_STARTS_WITH)) { - queryTemplate = queryByAttrStartWith; + queryTemplate = gremlinQueryProvider.getQuery(AtlasGremlinQuery.EXPORT_TYPE_STARTS_WITH); } else if (StringUtils.equalsIgnoreCase(matchType, MATCH_TYPE_ENDS_WITH)) { - queryTemplate = queryByAttrEndsWith; + queryTemplate = gremlinQueryProvider.getQuery(AtlasGremlinQuery.EXPORT_TYPE_ENDS_WITH); } else if (StringUtils.equalsIgnoreCase(matchType, MATCH_TYPE_CONTAINS)) { - queryTemplate = queryByAttrContains; + queryTemplate = gremlinQueryProvider.getQuery(AtlasGremlinQuery.EXPORT_TYPE_CONTAINS); } else if (StringUtils.equalsIgnoreCase(matchType, MATCH_TYPE_MATCHES)) { - queryTemplate = queryByAttrMatches; + queryTemplate = gremlinQueryProvider.getQuery(AtlasGremlinQuery.EXPORT_TYPE_MATCHES); } else { // default - queryTemplate = queryByAttrEquals; + queryTemplate = gremlinQueryProvider.getQuery(AtlasGremlinQuery.EXPORT_TYPE_DEFAULT); } for (Map.Entry<String, Object> e : item.getUniqueAttributes().entrySet()) { @@ -320,7 +323,8 @@ public class ExportService { } private List<String> executeGremlinScriptForHive(String guid) throws ScriptException { - return executeGremlinScriptFor(this.queryByGuid, "startGuid", guid); + String queryByGuid = gremlinQueryProvider.getQuery(AtlasGremlinQuery.EXPORT_BY_GUID); + return executeGremlinScriptFor(queryByGuid, "startGuid", guid); } private List<String> executeGremlinScriptFor(String query, String parameterName, String parameterValue) {
