This is an automated email from the ASF dual-hosted git repository. nixon pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 11b73ee0a58e93bb2df78d15dac7b850a60ac799 Author: Pinal Shah <[email protected]> AuthorDate: Thu Jun 18 17:49:31 2020 +0530 ATLAS-3848 : Quick Search : Fixed incorrect aggregation metrics Signed-off-by: nixonrodrigues <[email protected]> (cherry picked from commit aa86d62c057d56f858dc7962aa158d23c82f972b) --- .../graphdb/janus/AtlasSolrQueryBuilder.java | 34 +++++++++++++++++++++- .../graphdb/janus/AtlasSolrQueryBuilderTest.java | 22 +++++++------- .../org/apache/atlas/type/AtlasTypeRegistry.java | 18 ++++++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java index 05db148..b8fd4a0 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java @@ -34,6 +34,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY; + public class AtlasSolrQueryBuilder { private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class); @@ -43,6 +45,8 @@ public class AtlasSolrQueryBuilder { private boolean excludeDeletedEntities; private boolean includeSubtypes; private Map<String, String> indexFieldNameCache; + public static final char CUSTOM_ATTR_SEPARATOR = '='; + public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\""; public AtlasSolrQueryBuilder() { @@ -211,6 +215,14 @@ public class AtlasSolrQueryBuilder { attributeValue = attributeValue.trim(); } + if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) { + // CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair, + // for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2 + // surrounding inverted commas to attributeValue works + operator = Operator.EQ; + attributeValue = getIndexQueryAttributeValue(attributeValue); + } + AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute == null) { @@ -231,6 +243,8 @@ public class AtlasSolrQueryBuilder { throw new AtlasBaseException(msg); } + beginCriteria(queryBuilder); + switch (operator) { case EQ: withEqual(queryBuilder, indexFieldName, attributeValue); @@ -274,9 +288,27 @@ public class AtlasSolrQueryBuilder { LOG.error(msg); throw new AtlasBaseException(msg); } + + endCriteria(queryBuilder); } } + private String getIndexQueryAttributeValue(String attributeValue) { + + if (StringUtils.isNotEmpty(attributeValue)) { + int separatorIdx = attributeValue.indexOf(CUSTOM_ATTR_SEPARATOR); + String key = separatorIdx != -1 ? attributeValue.substring(0, separatorIdx) : null; + String value = key != null ? attributeValue.substring(separatorIdx + 1) : null; + + if (key != null && value != null) { + return String.format(CUSTOM_ATTR_SEARCH_FORMAT, key, value); + } + } + + return attributeValue; + } + + private void beginCriteria(StringBuilder queryBuilder) { queryBuilder.append("( "); } @@ -294,7 +326,7 @@ public class AtlasSolrQueryBuilder { } private void withNotEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) { - queryBuilder.append("-").append(indexFieldName).append(":").append(attributeValue).append(" "); + queryBuilder.append("*:* -").append(indexFieldName).append(":").append(attributeValue).append(" "); } private void withEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) { diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java index 2af1818..60d8610 100644 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java @@ -114,7 +114,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 OR +comment_index:*t10* )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )"); } @Test @@ -124,7 +124,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); } @Test @@ -134,7 +134,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 AND +comment_index:*t10* )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )"); } @Test @@ -144,7 +144,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); } @Test @@ -154,7 +154,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND +name_index:t10 "); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )"); } @Test @@ -164,7 +164,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +comment_index:*United States* AND +descrption__index:*nothing* AND +name_index:*t100* )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )"); } @Test @@ -174,7 +174,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:{ 100 TO * ] )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )"); } @Test @@ -184,7 +184,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ 100 TO * ] AND +started__index:[ 100 TO * ] )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )"); } @Test @@ -194,7 +194,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ * TO100} )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )"); } @Test @@ -204,7 +204,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ * TO 100 ] AND +started__index:[ * TO 100 ] )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )"); } @Test @@ -214,7 +214,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +qualifiedName__index:testdb.t1* )"); + Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )"); } diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index 2c7cb2b..4a79b6f 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -62,6 +62,7 @@ public class AtlasTypeRegistry { commonIndexFieldNameCache = new HashMap<>(); resolveReferencesForRootTypes(); + resolveIndexFieldNamesForRootTypes(); } // used only by AtlasTransientTypeRegistry @@ -72,6 +73,7 @@ public class AtlasTypeRegistry { commonIndexFieldNameCache = other.commonIndexFieldNameCache; resolveReferencesForRootTypes(); + resolveIndexFieldNamesForRootTypes(); } public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); } @@ -275,6 +277,22 @@ public class AtlasTypeRegistry { } } + private void resolveIndexFieldNamesForRootTypes() { + for (AtlasStructType structType : Arrays.asList(AtlasEntityType.ENTITY_ROOT, AtlasClassificationType.CLASSIFICATION_ROOT)) { + for (AtlasAttribute attribute : structType.getAllAttributes().values()) { + String indexFieldName = getIndexFieldName(attribute.getVertexPropertyName()); + + if (StringUtils.isNotEmpty(indexFieldName)) { + attribute.setIndexFieldName(indexFieldName); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Attribute {} with index name {} is added", attribute.getVertexPropertyName(), attribute.getIndexFieldName()); + } + } + } + } + /** * retrieves the index field name for the common field passed in. * @param propertyName the name of the common field.
