This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new aa86d62 ATLAS-3848 : Quick Search : Fixed incorrect aggregation
metrics
aa86d62 is described below
commit aa86d62c057d56f858dc7962aa158d23c82f972b
Author: Pinal Shah <[email protected]>
AuthorDate: Thu Jun 18 17:49:31 2020 +0530
ATLAS-3848 : Quick Search : Fixed incorrect aggregation metrics
Signed-off-by: nixonrodrigues <[email protected]>
---
.../graphdb/janus/AtlasSolrQueryBuilder.java | 34 +++++++++++++++++++++-
.../graphdb/janus/AtlasSolrQueryBuilderTest.java | 22 +++++++-------
.../org/apache/atlas/type/AtlasTypeRegistry.java | 18 ++++++++++++
3 files changed, 62 insertions(+), 12 deletions(-)
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
index 05db148..b8fd4a0 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
@@ -34,6 +34,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import static
org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
+
public class AtlasSolrQueryBuilder {
private static final Logger LOG =
LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
@@ -43,6 +45,8 @@ public class AtlasSolrQueryBuilder {
private boolean excludeDeletedEntities;
private boolean includeSubtypes;
private Map<String, String> indexFieldNameCache;
+ public static final char CUSTOM_ATTR_SEPARATOR = '=';
+ public static final String CUSTOM_ATTR_SEARCH_FORMAT =
"\"\\\"%s\\\":\\\"%s\\\"\"";
public AtlasSolrQueryBuilder() {
@@ -211,6 +215,14 @@ public class AtlasSolrQueryBuilder {
attributeValue = attributeValue.trim();
}
+ if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) &&
operator.equals(Operator.CONTAINS)) {
+ // CustomAttributes stores key value pairs in String format,
so ideally it should be 'contains' operator to search for one pair,
+ // for use-case, E1 having key1=value1 and E2 having
key1=value2, searching key1=value1 results both E1,E2
+ // surrounding inverted commas to attributeValue works
+ operator = Operator.EQ;
+ attributeValue = getIndexQueryAttributeValue(attributeValue);
+ }
+
AtlasAttribute attribute = entityType.getAttribute(attributeName);
if (attribute == null) {
@@ -231,6 +243,8 @@ public class AtlasSolrQueryBuilder {
throw new AtlasBaseException(msg);
}
+ beginCriteria(queryBuilder);
+
switch (operator) {
case EQ:
withEqual(queryBuilder, indexFieldName, attributeValue);
@@ -274,9 +288,27 @@ public class AtlasSolrQueryBuilder {
LOG.error(msg);
throw new AtlasBaseException(msg);
}
+
+ endCriteria(queryBuilder);
}
}
+ private String getIndexQueryAttributeValue(String attributeValue) {
+
+ if (StringUtils.isNotEmpty(attributeValue)) {
+ int separatorIdx =
attributeValue.indexOf(CUSTOM_ATTR_SEPARATOR);
+ String key = separatorIdx != -1 ?
attributeValue.substring(0, separatorIdx) : null;
+ String value = key != null ?
attributeValue.substring(separatorIdx + 1) : null;
+
+ if (key != null && value != null) {
+ return String.format(CUSTOM_ATTR_SEARCH_FORMAT, key, value);
+ }
+ }
+
+ return attributeValue;
+ }
+
+
private void beginCriteria(StringBuilder queryBuilder) {
queryBuilder.append("( ");
}
@@ -294,7 +326,7 @@ public class AtlasSolrQueryBuilder {
}
private void withNotEqual(StringBuilder queryBuilder, String
indexFieldName, String attributeValue) {
-
queryBuilder.append("-").append(indexFieldName).append(":").append(attributeValue).append("
");
+ queryBuilder.append("*:*
-").append(indexFieldName).append(":").append(attributeValue).append(" ");
}
private void withEqual(StringBuilder queryBuilder, String indexFieldName,
String attributeValue) {
diff --git
a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
index 2af1818..60d8610 100644
---
a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
+++
b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
@@ -114,7 +114,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+name_index:t10 OR +comment_index:*t10* )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) OR ( +comment_index:*t10* ) )");
}
@Test
@@ -124,7 +124,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+name_index:t10 )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) )");
}
@Test
@@ -134,7 +134,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+name_index:t10 AND +comment_index:*t10* )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) AND ( +comment_index:*t10* ) )");
}
@Test
@@ -144,7 +144,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+name_index:t10 )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) )");
}
@Test
@@ -154,7 +154,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND
+name_index:t10 ");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+name_index:t10 )");
}
@Test
@@ -164,7 +164,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+comment_index:*United States* AND +descrption__index:*nothing* AND
+name_index:*t100* )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND (
+name_index:*t100* ) )");
}
@Test
@@ -174,7 +174,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+created__index:{ 100 TO * ] )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:{ 100 TO * ] ) )");
}
@Test
@@ -184,7 +184,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+created__index:[ 100 TO * ] AND +started__index:[ 100 TO * ] )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
}
@Test
@@ -194,7 +194,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+created__index:[ * TO100} )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ * TO100} ) )");
}
@Test
@@ -204,7 +204,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+created__index:[ * TO 100 ] AND +started__index:[ * TO 100 ] )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
}
@Test
@@ -214,7 +214,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND
+__typeName__index:(hive_table ) AND ( +qualifiedName__index:testdb.t1* )");
+ Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND
+__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* )
)");
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 2c7cb2b..4a79b6f 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -62,6 +62,7 @@ public class AtlasTypeRegistry {
commonIndexFieldNameCache = new HashMap<>();
resolveReferencesForRootTypes();
+ resolveIndexFieldNamesForRootTypes();
}
// used only by AtlasTransientTypeRegistry
@@ -72,6 +73,7 @@ public class AtlasTypeRegistry {
commonIndexFieldNameCache = other.commonIndexFieldNameCache;
resolveReferencesForRootTypes();
+ resolveIndexFieldNamesForRootTypes();
}
public Collection<String> getAllTypeNames() { return
registryData.allTypes.getAllTypeNames(); }
@@ -275,6 +277,22 @@ public class AtlasTypeRegistry {
}
}
+ private void resolveIndexFieldNamesForRootTypes() {
+ for (AtlasStructType structType :
Arrays.asList(AtlasEntityType.ENTITY_ROOT,
AtlasClassificationType.CLASSIFICATION_ROOT)) {
+ for (AtlasAttribute attribute :
structType.getAllAttributes().values()) {
+ String indexFieldName =
getIndexFieldName(attribute.getVertexPropertyName());
+
+ if (StringUtils.isNotEmpty(indexFieldName)) {
+ attribute.setIndexFieldName(indexFieldName);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Attribute {} with index name {} is added",
attribute.getVertexPropertyName(), attribute.getIndexFieldName());
+ }
+ }
+ }
+ }
+
/**
* retrieves the index field name for the common field passed in.
* @param propertyName the name of the common field.
