Repository: incubator-atlas Updated Branches: refs/heads/0.8-incubating 47d11cc38 -> f800575ef
ATLAS-1981: Cache escaped type-query string to avoid repeated computation (cherry picked from commit e0d2cdc27c32742ebecd24db4cca62dc04dcdf4b) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f800575e Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f800575e Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f800575e Branch: refs/heads/0.8-incubating Commit: f800575eff23f7fd05f3f79b59dd6567b4a33f1c Parents: 47d11cc Author: Madhan Neethiraj <mad...@apache.org> Authored: Fri Jul 21 09:28:44 2017 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Fri Jul 21 12:25:39 2017 -0700 ---------------------------------------------------------------------- .../org/apache/atlas/repository/Constants.java | 4 +- .../org/apache/atlas/model/TypeCategory.java | 2 +- .../atlas/type/AtlasClassificationType.java | 30 ++++++++--- .../org/apache/atlas/type/AtlasEntityType.java | 28 +++++++--- .../org/apache/atlas/type/AtlasStructType.java | 40 +++++++++++++++ .../java/org/apache/atlas/type/AtlasType.java | 3 ++ .../apache/atlas/type/AtlasTypeRegistry.java | 4 ++ .../ClassificationSearchProcessor.java | 17 +++--- .../atlas/discovery/EntityDiscoveryService.java | 32 ++++++------ .../atlas/discovery/EntitySearchProcessor.java | 18 ++++--- .../discovery/FullTextSearchProcessor.java | 23 ++++----- .../apache/atlas/discovery/SearchProcessor.java | 54 ++++---------------- .../graph/v1/AtlasTypeDefGraphStoreV1.java | 3 ++ .../typestore/GraphBackedTypeStore.java | 8 +++ .../atlas/typesystem/types/DataTypes.java | 3 +- 15 files changed, 162 insertions(+), 107 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/common/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 7156d56..62d74f8 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -92,8 +92,8 @@ public final class Constants { public static final String QUALIFIED_NAME = "Referenceable.qualifiedName"; public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; - public static final String INDEX_SEARCH_MAX_TYPES_COUNT = "atlas.graph.index.search.max-types-count"; - public static final String INDEX_SEARCH_MAX_TAGS_COUNT = "atlas.graph.index.search.max-tags-count"; + public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length"; + public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.tags.max-query-str-length"; private Constants() { } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/model/TypeCategory.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/TypeCategory.java b/intg/src/main/java/org/apache/atlas/model/TypeCategory.java index e47a8a7..f06f64f 100644 --- a/intg/src/main/java/org/apache/atlas/model/TypeCategory.java +++ b/intg/src/main/java/org/apache/atlas/model/TypeCategory.java @@ -18,5 +18,5 @@ package org.apache.atlas.model; public enum TypeCategory { - PRIMITIVE, OBJECT_ID_TYPE, ENUM, STRUCT, CLASSIFICATION, ENTITY, ARRAY, MAP + PRIMITIVE, OBJECT_ID_TYPE, ENUM, STRUCT, CLASSIFICATION, ENTITY, ARRAY, MAP, RELATIONSHIP } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index c1c3add..56c3ed3 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -45,10 +45,11 @@ public class AtlasClassificationType extends AtlasStructType { private final AtlasClassificationDef classificationDef; - private List<AtlasClassificationType> superTypes = Collections.emptyList(); - private Set<String> allSuperTypes = Collections.emptySet(); - private Set<String> allSubTypes = Collections.emptySet(); - private Set<String> typeAndAllSubTypes = Collections.emptySet(); + private List<AtlasClassificationType> superTypes = Collections.emptyList(); + private Set<String> allSuperTypes = Collections.emptySet(); + private Set<String> allSubTypes = Collections.emptySet(); + private Set<String> typeAndAllSubTypes = Collections.emptySet(); + private String typeAndAllSubTypesQryStr = ""; public AtlasClassificationType(AtlasClassificationDef classificationDef) { super(classificationDef); @@ -108,6 +109,13 @@ public class AtlasClassificationType extends AtlasStructType { } } + @Override + public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + allSubTypes = Collections.unmodifiableSet(allSubTypes); + typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes); + typeAndAllSubTypesQryStr = ""; // will be computed on next access + } + private void addSubType(AtlasClassificationType subType) { allSubTypes.add(subType.getTypeName()); typeAndAllSubTypes.add(subType.getTypeName()); @@ -119,11 +127,17 @@ public class AtlasClassificationType extends AtlasStructType { public Set<String> getAllSuperTypes() { return allSuperTypes; } - public Set<String> getAllSubTypes() { - return Collections.unmodifiableSet(allSubTypes); - } + public Set<String> getAllSubTypes() { return allSubTypes; } - public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); } + public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; } + + public String getTypeAndAllSubTypesQryStr() { + if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) { + typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes); + } + + return typeAndAllSubTypesQryStr; + } public boolean isSuperTypeOf(AtlasClassificationType classificationType) { return classificationType != null && allSubTypes.contains(classificationType.getTypeName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 6516d48..2ad8f1c 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -47,10 +47,11 @@ public class AtlasEntityType extends AtlasStructType { private final AtlasEntityDef entityDef; - private List<AtlasEntityType> superTypes = Collections.emptyList(); - private Set<String> allSuperTypes = Collections.emptySet(); - private Set<String> allSubTypes = Collections.emptySet(); - private Set<String> typeAndAllSubTypes = Collections.emptySet(); + private List<AtlasEntityType> superTypes = Collections.emptyList(); + private Set<String> allSuperTypes = Collections.emptySet(); + private Set<String> allSubTypes = Collections.emptySet(); + private Set<String> typeAndAllSubTypes = Collections.emptySet(); + private String typeAndAllSubTypesQryStr = ""; public AtlasEntityType(AtlasEntityDef entityDef) { super(entityDef); @@ -108,6 +109,13 @@ public class AtlasEntityType extends AtlasStructType { } } + @Override + public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + allSubTypes = Collections.unmodifiableSet(allSubTypes); + typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes); + typeAndAllSubTypesQryStr = ""; // will be computed on next access + } + public Set<String> getSuperTypes() { return entityDef.getSuperTypes(); } @@ -116,9 +124,9 @@ public class AtlasEntityType extends AtlasStructType { return allSuperTypes; } - public Set<String> getAllSubTypes() { return Collections.unmodifiableSet(allSubTypes); } + public Set<String> getAllSubTypes() { return allSubTypes; } - public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); } + public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; } public boolean isSuperTypeOf(AtlasEntityType entityType) { return entityType != null && allSubTypes.contains(entityType.getTypeName()); @@ -136,6 +144,14 @@ public class AtlasEntityType extends AtlasStructType { return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName); } + public String getTypeAndAllSubTypesQryStr() { + if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) { + typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes); + } + + return typeAndAllSubTypesQryStr; + } + @Override public AtlasEntity createDefaultValue() { AtlasEntity ret = new AtlasEntity(entityDef.getName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 0eeaf9c..e73b503 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * class that implements behaviour of a struct-type. @@ -683,6 +684,39 @@ public class AtlasStructType extends AtlasType { return key; } + public static String escapeIndexQueryValue(Set<String> values) { + StringBuilder sb = new StringBuilder(); + + sb.append(BRACE_OPEN_CHAR); + for (String value : values) { + sb.append(escapeIndexQueryValue(value)).append(SPACE_CHAR); + } + sb.append(BRACE_CLOSE_CHAR); + + return sb.toString(); + } + + public static String escapeIndexQueryValue(String value) { + String ret = value; + + if (StringUtils.containsAny(value, IDX_QRY_OFFENDING_CHARS)) { + boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE_CHAR; + boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE_CHAR; + + if (!isQuoteAtStart) { + if (!isQuoteAtEnd) { + ret = DOUBLE_QUOTE_CHAR + value + DOUBLE_QUOTE_CHAR; + } else { + ret = DOUBLE_QUOTE_CHAR + value; + } + } else if (!isQuoteAtEnd) { + ret = value + DOUBLE_QUOTE_CHAR; + } + } + + return ret; + } + private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) { final String typeName = structDef.getName(); return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName); @@ -695,5 +729,11 @@ public class AtlasStructType extends AtlasType { new String[] { "$", "_d" }, new String[] { "%", "_p" }, }; + + private static final char[] IDX_QRY_OFFENDING_CHARS = { '@', '/', ' ' }; + private static final char BRACE_OPEN_CHAR = '('; + private static final char BRACE_CLOSE_CHAR = ')'; + private static final char DOUBLE_QUOTE_CHAR = '"'; + private static final char SPACE_CHAR = ' '; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java index 79de9d5..747c9a6 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java @@ -57,6 +57,9 @@ public abstract class AtlasType { public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { } + public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + } + public String getTypeName() { return typeName; } public TypeCategory getTypeCategory() { return typeCategory; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index 1b3526b..dbd1038 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -343,6 +343,10 @@ public class AtlasTypeRegistry { for (AtlasType type : registryData.allTypes.getAllTypes()) { type.resolveReferencesPhase2(this); } + + for (AtlasType type : registryData.allTypes.getAllTypes()) { + type.resolveReferencesPhase3(this); + } } public void clear() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java index 745f9d7..dcdcbbb 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java @@ -42,23 +42,24 @@ public class ClassificationSearchProcessor extends SearchProcessor { public ClassificationSearchProcessor(SearchContext context) { super(context); - AtlasClassificationType classificationType = context.getClassificationType(); - FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters(); - Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes(); - Set<String> solrAttributes = new HashSet<>(); - Set<String> gremlinAttributes = new HashSet<>(); - Set<String> allAttributes = new HashSet<>(); + final AtlasClassificationType classificationType = context.getClassificationType(); + final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters(); + final Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes(); + final String typeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr(); + final Set<String> solrAttributes = new HashSet<>(); + final Set<String> gremlinAttributes = new HashSet<>(); + final Set<String> allAttributes = new HashSet<>(); processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes); // for classification search, if any attribute can't be handled by Solr - switch to all Gremlin - boolean useSolrSearch = typeAndSubTypes.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false); + boolean useSolrSearch = typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false); if (useSolrSearch) { StringBuilder solrQuery = new StringBuilder(); - constructTypeTestQuery(solrQuery, typeAndSubTypes); + constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr); constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes); String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index dbe11a6..b183c72 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -87,8 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private final GraphBackedSearchIndexer indexer; private final SearchTracker searchTracker; private final int maxResultSetSize; - private final int maxTypesCountInIdxQuery; - private final int maxTagsCountInIdxQuery; + private final int maxTypesLengthInIdxQuery; + private final int maxTagsLengthInIdxQuery; @Inject EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry, @@ -101,8 +101,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; this.typeRegistry = typeRegistry; this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); - this.maxTypesCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10); - this.maxTagsCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10); + this.maxTypesLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512); + this.maxTagsLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512); } @Override @@ -490,8 +490,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) { - String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery); - String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery); + String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesLengthInIdxQuery); + String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsLengthInIdxQuery); StringBuilder queryText = new StringBuilder(); @@ -619,23 +619,23 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED; } - private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesCountInIdxQuery) { - AtlasClassificationType classification = typeRegistry.getClassificationTypeByName(classificationName); - Set<String> typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes() : null; + private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) { + AtlasClassificationType type = typeRegistry.getClassificationTypeByName(classificationName); + String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null; - if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) { - return String.format("(%s)", StringUtils.join(typeAndSubTypes, " ")); + if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) { + return typeAndSubTypesQryStr; } return ""; } - private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesCountInIdxQuery) { - AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName); - Set<String> typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes() : null; + private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesLengthInIdxQuery) { + AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName); + String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null; - if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) { - return String.format("(%s)", StringUtils.join(typeAndSubTypes, " ")); + if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) { + return typeAndSubTypesQryStr; } return ""; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java index efcfb7f..385951b 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java @@ -40,12 +40,13 @@ public class EntitySearchProcessor extends SearchProcessor { public EntitySearchProcessor(SearchContext context) { super(context); - final AtlasEntityType entityType = context.getEntityType(); - final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); - final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes(); - final Set<String> solrAttributes = new HashSet<>(); - final Set<String> gremlinAttributes = new HashSet<>(); - final Set<String> allAttributes = new HashSet<>(); + final AtlasEntityType entityType = context.getEntityType(); + final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); + final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes(); + final String typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr(); + final Set<String> solrAttributes = new HashSet<>(); + final Set<String> gremlinAttributes = new HashSet<>(); + final Set<String> allAttributes = new HashSet<>(); final AtlasClassificationType classificationType = context.getClassificationType(); final boolean filterClassification = classificationType != null && !context.needClassificationProcessor(); @@ -53,13 +54,13 @@ public class EntitySearchProcessor extends SearchProcessor { processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes); - final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY; + final boolean typeSearchBySolr = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES; final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, filterCriteria, false); StringBuilder solrQuery = new StringBuilder(); if (typeSearchBySolr) { - constructTypeTestQuery(solrQuery, typeAndSubTypes); + constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr); } if (attrSearchBySolr) { @@ -169,6 +170,7 @@ public class EntitySearchProcessor extends SearchProcessor { guidQuery.addConditionsFrom(graphQuery); + entityVertices.clear(); getVertices(guidQuery.vertices().iterator(), entityVertices); } } else { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java index 0d1c39b..d556bf1 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java @@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Set; public class FullTextSearchProcessor extends SearchProcessor { @@ -50,28 +49,26 @@ public class FullTextSearchProcessor extends SearchProcessor { // if search includes entity-type criteria, adding a filter here can help avoid unnecessary // processing (and rejection) by subsequent EntitySearchProcessor if (context.getEntityType() != null) { - Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes(); + String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr(); - if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) { - queryString.append(AND_STR); - appendIndexQueryValue(typeAndSubTypeNames, queryString); + if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) { + queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr); } else { - LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance", - context.getEntityType().getTypeName(), typeAndSubTypeNames.size()); + LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", + context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length()); } } // if search includes classification criteria, adding a filter here can help avoid unnecessary // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor if (context.getClassificationType() != null) { - Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes(); + String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr(); - if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) { - queryString.append(AND_STR); - appendIndexQueryValue(typeAndSubTypeNames, queryString); + if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) { + queryString.append(AND_STR).append(typeAndSubTypeNamesStr); } else { - LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance", - context.getClassificationType().getTypeName(), typeAndSubTypeNames.size()); + LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", + context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length()); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java index 2862165..7e09672 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java @@ -39,21 +39,19 @@ import java.util.regex.Pattern; public abstract class SearchProcessor { private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class); - public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)"); - public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)"); - public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)"); - public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); - public static final int MAX_ENTITY_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10); - public static final int MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10); + public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)"); + public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)"); + public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)"); + public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); + public static final int MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512); + public static final int MAX_QUERY_STR_LENGTH_TAGS = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512); public static final String AND_STR = " AND "; public static final String EMPTY_STRING = ""; public static final String SPACE_STRING = " "; public static final String BRACE_OPEN_STR = "("; public static final String BRACE_CLOSE_STR = ")"; - public static final char DOUBLE_QUOTE = '"'; private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>(); - private static final char[] OFFENDING_CHARS = { '@', '/', ' ' }; // This can grow as we discover corner cases static { @@ -181,14 +179,13 @@ public abstract class SearchProcessor { return ret; } - protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) { - if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) { + protected void constructTypeTestQuery(StringBuilder solrQuery, String typeAndAllSubTypesQryStr) { + if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) { if (solrQuery.length() > 0) { solrQuery.append(AND_STR); } - solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":"); - appendIndexQueryValue(typeAndAllSubTypes, solrQuery); + solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr); } } @@ -255,7 +252,7 @@ public abstract class SearchProcessor { if (OPERATOR_MAP.get(op) != null) { String qualifiedName = type.getQualifiedAttributeName(attrName); - ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue(attrVal)); + ret = String.format(OPERATOR_MAP.get(op), qualifiedName, AtlasStructType.AtlasAttribute.escapeIndexQueryValue(attrVal)); } } catch (AtlasBaseException ex) { LOG.warn(ex.getMessage()); @@ -389,16 +386,6 @@ public abstract class SearchProcessor { return ret; } - protected String appendIndexQueryValue(Set<String> values, StringBuilder sb) { - sb.append(BRACE_OPEN_STR); - for (String value : values) { - sb.append(escapeIndexQueryValue(value)).append(SPACE_STRING); - } - sb.append(BRACE_CLOSE_STR); - - return sb.toString(); - } - private static int getApplicationProperty(String propertyName, int defaultValue) { try { return ApplicationProperties.get().getInt(propertyName, defaultValue); @@ -408,25 +395,4 @@ public abstract class SearchProcessor { return defaultValue; } - - private String escapeIndexQueryValue(String value) { - String ret = value; - - if (StringUtils.containsAny(value, OFFENDING_CHARS)) { - boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE; - boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE; - - if (!isQuoteAtStart) { - if (!isQuoteAtEnd) { - ret = DOUBLE_QUOTE + value + DOUBLE_QUOTE; - } else { - ret = DOUBLE_QUOTE + value; - } - } else if (!isQuoteAtEnd) { - ret = value + DOUBLE_QUOTE; - } - } - - return ret; - } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java index f0c8380..fbd6314 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java @@ -385,6 +385,9 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { case ENUM: return TypeCategory.ENUM; + + case RELATIONSHIP: + return TypeCategory.RELATIONSHIP; } return null; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java index 7a064b6..1525d7b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java @@ -107,6 +107,10 @@ public class GraphBackedTypeStore implements ITypeStore { type.superTypes, visitor); break; + case RELATIONSHIP: + // ignore + break; + default: //Ignore primitive/collection types as they are covered under references break; } @@ -273,6 +277,10 @@ public class GraphBackedTypeStore implements ITypeStore { traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, superTypes, attributes)); break; + case RELATIONSHIP: + // v1 typesystem is not notified on new relation type + break; + default: throw new IllegalArgumentException("Unhandled type category " + typeCategory); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f800575e/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java index 21d5f1a..f9f4abe 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java @@ -86,7 +86,8 @@ public class DataTypes { MAP, STRUCT, TRAIT, - CLASS + CLASS, + RELATIONSHIP } public static abstract class PrimitiveType<T> extends AbstractDataType<T> {
