Repository: incubator-atlas Updated Branches: refs/heads/master 1e3029bc7 -> 2c0dc4068
ATLAS-690 Read timed out exceptions when tables are imported into Atlas (yhemanth via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/2c0dc406 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/2c0dc406 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/2c0dc406 Branch: refs/heads/master Commit: 2c0dc4068eac70a18f7cfa430b7e0bc7d84f7524 Parents: 1e3029b Author: Shwetha GS <[email protected]> Authored: Fri May 6 16:01:40 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Fri May 6 16:01:40 2016 +0530 ---------------------------------------------------------------------- release-log.txt | 1 + .../atlas/repository/graph/FullTextMapper.java | 19 ++- .../graph/GraphBackedSearchIndexer.java | 93 ++++++------ .../atlas/repository/graph/GraphHelper.java | 16 +- .../graph/TypedInstanceToGraphMapper.java | 11 +- .../graph/GraphBackedSearchIndexerMockTest.java | 94 ++++++++++++ .../graph/GraphBackedSearchIndexerTest.java | 151 +++++++++++++------ 7 files changed, 285 insertions(+), 100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2c0dc406/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 7c17a9d..186e93f 100644 --- a/release-log.txt +++ b/release-log.txt @@ -18,6 +18,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-690 Read timed out exceptions when tables are imported into Atlas (yhemanth via shwethags) ATLAS-585 NotificationHookConsumer creates new AtlasClient for every message (shwethags) ATLAS-682 Set HBase root dir to be relative to test target directory for HBaseBasedAuditRepositoryTest (shwethags via yhemanth) ATLAS-742 Avoid downloading hbase multiple times (shwethags via yhemanth) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2c0dc406/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java index 36d8034..37cacb0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java @@ -27,26 +27,41 @@ import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.EnumValue; import org.apache.atlas.typesystem.types.IDataType; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.List; import java.util.Map; public class FullTextMapper { + private static final Logger LOG = LoggerFactory.getLogger(FullTextMapper.class); + private final GraphToTypedInstanceMapper graphToTypedInstanceMapper; private static final GraphHelper graphHelper = GraphHelper.getInstance(); private static final String FULL_TEXT_DELIMITER = " "; + private final Map<String, ITypedReferenceableInstance> instanceCache; FullTextMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) { this.graphToTypedInstanceMapper = graphToTypedInstanceMapper; + instanceCache = new HashMap<>(); } public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException { String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); - ITypedReferenceableInstance typedReference = - graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); + ITypedReferenceableInstance typedReference; + if (instanceCache.containsKey(guid)) { + typedReference = instanceCache.get(guid); + LOG.debug("Cache hit: guid = {}, entityId = {}", guid, typedReference.getId()._getId()); + } else { + typedReference = + graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); + instanceCache.put(guid, typedReference); + LOG.debug("Cache miss: guid = {}, entityId = {}", guid, typedReference.getId().getId()); + } String fullText = forInstance(typedReference, followReferences); StringBuilder fullTextBuilder = new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2c0dc406/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 9b9fe35..c385df3 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -30,11 +30,11 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.discovery.SearchIndexer; import org.apache.atlas.ha.HAConfiguration; +import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.IndexCreationException; import org.apache.atlas.repository.IndexException; import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; @@ -70,8 +70,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang }}; @Inject - public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider) throws RepositoryException, - AtlasException { + public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider) throws AtlasException { this(graphProvider, ApplicationProperties.get()); } @@ -100,29 +99,27 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang management.buildIndex(Constants.EDGE_INDEX, Edge.class).buildMixedIndex(Constants.BACKING_INDEX); // create a composite index for guid as its unique - createCompositeAndMixedIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true, + createIndexes(management, Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, true); // create a composite index for entity state - createCompositeAndMixedIndex(management, Constants.STATE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true); - - // create a composite index for entity state - createCompositeAndMixedIndex(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true); + createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true); // create a composite index for entity state - createCompositeAndMixedIndex(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true); + createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false, + Cardinality.SINGLE, true); // create a composite and mixed index for type since it can be combined with other keys - createCompositeAndMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, + createIndexes(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true); // create a composite and mixed index for type since it can be combined with other keys - createCompositeAndMixedIndex(management, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET, + createIndexes(management, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET, true); // create a composite and mixed index for traitNames since it can be combined with other // keys. Traits must be a set and not a list. - createCompositeAndMixedIndex(management, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET, + createIndexes(management, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET, true); // Index for full text search @@ -151,12 +148,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang private void createTypeStoreIndexes(TitanManagement management) { //Create unique index on typeName - createCompositeAndMixedIndex(management, Constants.TYPENAME_PROPERTY_KEY, String.class, true, - Cardinality.SINGLE, true); + createIndexes(management, Constants.TYPENAME_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, + true); //create index on vertex type - createCompositeAndMixedIndex(management, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, - Cardinality.SINGLE, true); + createIndexes(management, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, + true); } /** @@ -233,13 +230,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang switch (field.dataType().getTypeCategory()) { case PRIMITIVE: Cardinality cardinality = getCardinality(field.multiplicity); - createCompositeAndMixedIndex(management, propertyName, getPrimitiveClass(field.dataType()), false, + createIndexes(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique, cardinality, false); break; case ENUM: cardinality = getCardinality(field.multiplicity); - createCompositeAndMixedIndex(management, propertyName, String.class, false, cardinality, false); + createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false); break; case ARRAY: @@ -312,43 +309,51 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang return Cardinality.LIST; } - - private PropertyKey createCompositeAndMixedIndex(TitanManagement management, String propertyName, - Class propertyClass, - boolean isUnique, Cardinality cardinality, boolean force) { - + private PropertyKey createIndexes(TitanManagement management, String propertyName, + Class propertyClass, boolean isUnique, Cardinality cardinality, + boolean isSystemProperty) { PropertyKey propertyKey = management.getPropertyKey(propertyName); if (propertyKey == null) { propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality) .make(); - if (checkIfMixedIndexApplicable(propertyClass, cardinality)) { - //Use backing index - LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName()); - TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX); - management.addIndexKey(vertexIndex, propertyKey); - LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName()); - } + enhanceMixedIndex(management, propertyName, propertyClass, cardinality, propertyKey); - //Create mixed index only for meta properties and unique constraints: - //Unique can't be achieved with backing/mixed index - //Creating composite index for every attribute will bloat up the index - if (force || isUnique) { - LOG.debug("Creating composite index for property {} of type {} ", propertyName, - propertyClass.getName()); - TitanManagement.IndexBuilder indexBuilder = - management.buildIndex(propertyName, Vertex.class).addKey(propertyKey); - if (isUnique) { - indexBuilder.unique(); - } - indexBuilder.buildCompositeIndex(); - LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName()); + if (isSystemProperty) { + createCompositeIndex(management, propertyName, propertyClass, propertyKey, isUnique); + } else if (isUnique) { + // send uniqueness as false because there can be many vertexes with the same property value + // but state can be active / deleted. + createCompositeIndex(management, propertyName, propertyClass, propertyKey, false); } } - return propertyKey; } + private void createCompositeIndex(TitanManagement management, String propertyName, Class propertyClass, + PropertyKey propertyKey, boolean enforceUniqueness) { + LOG.debug("Creating composite index for property {} of type {} ", propertyName, + propertyClass.getName()); + TitanManagement.IndexBuilder indexBuilder = + management.buildIndex(propertyName, Vertex.class).addKey(propertyKey); + if (enforceUniqueness) { + indexBuilder.unique(); + } + indexBuilder.buildCompositeIndex(); + LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName()); + } + + private void enhanceMixedIndex(TitanManagement management, String propertyName, Class propertyClass, + Cardinality cardinality, PropertyKey propertyKey) { + if (checkIfMixedIndexApplicable(propertyClass, cardinality)) { + //Use backing index + LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName()); + TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX); + management.addIndexKey(vertexIndex, propertyKey); + LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName()); + } + } + private boolean checkIfMixedIndexApplicable(Class propertyClass, Cardinality cardinality) { return !(MIXED_INDEX_EXCLUSIONS.contains(propertyClass) || cardinality == Cardinality.LIST || cardinality == Cardinality.SET); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2c0dc406/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index c542ec7..cccafc2 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -390,12 +390,20 @@ public final class GraphHelper { } public static String string(Vertex vertex) { - return String.format("vertex[id=%s type=%s guid=%s]", vertex.getId().toString(), getTypeName(vertex), - getIdFromVertex(vertex)); + if (LOG.isDebugEnabled()) { + return String.format("vertex[id=%s type=%s guid=%s]", vertex.getId().toString(), getTypeName(vertex), + getIdFromVertex(vertex)); + } else { + return String.format("vertex[id=%s]", vertex.getId().toString()); + } } public static String string(Edge edge) { - return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getId().toString(), edge.getLabel(), - string(edge.getVertex(Direction.OUT)), string(edge.getVertex(Direction.IN))); + if (LOG.isDebugEnabled()) { + return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getId().toString(), edge.getLabel(), + string(edge.getVertex(Direction.OUT)), string(edge.getVertex(Direction.IN))); + } else { + return String.format("edge[id=%s]", edge.getId().toString()); + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2c0dc406/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index a3dc7e5..a017536 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -96,11 +96,11 @@ public final class TypedInstanceToGraphMapper { createVerticesAndDiscoverInstances(newInstances); List<ITypedReferenceableInstance> entitiesToCreate = instancesPair.left; List<ITypedReferenceableInstance> entitiesToUpdate = instancesPair.right; - + FullTextMapper fulltextMapper = new FullTextMapper(graphToTypedInstanceMapper); switch (operation) { case CREATE: List<String> ids = addOrUpdateAttributesAndTraits(operation, entitiesToCreate); - addFullTextProperty(entitiesToCreate); + addFullTextProperty(entitiesToCreate, fulltextMapper); requestContext.recordCreatedEntities(ids); break; @@ -111,8 +111,8 @@ public final class TypedInstanceToGraphMapper { ids = addOrUpdateAttributesAndTraits(operation, entitiesToUpdate); requestContext.recordUpdatedEntities(ids); - addFullTextProperty(entitiesToCreate); - addFullTextProperty(entitiesToUpdate); + addFullTextProperty(entitiesToCreate, fulltextMapper); + addFullTextProperty(entitiesToUpdate, fulltextMapper); break; default: @@ -289,8 +289,7 @@ public final class TypedInstanceToGraphMapper { return TypeUtils.Pair.of(instancesToCreate, instancesToUpdate); } - private void addFullTextProperty(List<ITypedReferenceableInstance> instances) throws AtlasException { - FullTextMapper fulltextMapper = new FullTextMapper(graphToTypedInstanceMapper); + private void addFullTextProperty(List<ITypedReferenceableInstance> instances, FullTextMapper fulltextMapper) throws AtlasException { for (ITypedReferenceableInstance typedInstance : instances) { // Traverse Vertex instanceVertex = getClassVertex(typedInstance); String fullText = fulltextMapper.mapRecursive(instanceVertex, true); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2c0dc406/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerMockTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerMockTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerMockTest.java new file mode 100644 index 0000000..94dd298 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerMockTest.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graph; + +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.schema.TitanManagement; +import org.apache.atlas.AtlasException; +import org.apache.atlas.ha.HAConfiguration; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.IndexException; +import org.apache.atlas.repository.RepositoryException; +import org.apache.commons.configuration.Configuration; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +public class GraphBackedSearchIndexerMockTest { + + @Mock + private Configuration configuration; + + @Mock + private GraphProvider<TitanGraph> graphProvider; + + @Mock + private TitanGraph titanGraph; + + @Mock + private TitanManagement titanManagement; + + @BeforeMethod + public void setup() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testSearchIndicesAreInitializedOnConstructionWhenHAIsDisabled() throws IndexException, RepositoryException { + when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); + when(graphProvider.get()).thenReturn(titanGraph); + when(titanGraph.getManagementSystem()).thenReturn(titanManagement); + when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); + + GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration); + + verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY); + } + + @Test + public void testSearchIndicesAreNotInitializedOnConstructionWhenHAIsEnabled() throws IndexException, RepositoryException { + when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); + when(graphProvider.get()).thenReturn(titanGraph); + when(titanGraph.getManagementSystem()).thenReturn(titanManagement); + when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); + + GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration); + + verifyZeroInteractions(titanManagement); + + } + + @Test + public void testIndicesAreReinitializedWhenServerBecomesActive() throws AtlasException { + when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); + when(graphProvider.get()).thenReturn(titanGraph); + when(titanGraph.getManagementSystem()).thenReturn(titanManagement); + when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); + + GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration); + graphBackedSearchIndexer.instanceIsActive(); + + verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2c0dc406/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java index 87fdf87..3d9d45a 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java @@ -18,77 +18,140 @@ package org.apache.atlas.repository.graph; +import com.google.inject.Inject; +import com.thinkaurelius.titan.core.PropertyKey; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.schema.TitanGraphIndex; import com.thinkaurelius.titan.core.schema.TitanManagement; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.AtlasException; -import org.apache.atlas.ha.HAConfiguration; +import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.IndexException; -import org.apache.atlas.repository.RepositoryException; -import org.apache.commons.configuration.Configuration; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.annotations.BeforeMethod; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumType; +import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.commons.lang.RandomStringUtils; +import org.testng.annotations.Guice; import org.testng.annotations.Test; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; +import java.util.Arrays; -public class GraphBackedSearchIndexerTest { +import static junit.framework.Assert.assertTrue; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; - @Mock - private Configuration configuration; +@Guice(modules = RepositoryMetadataModule.class) +public class GraphBackedSearchIndexerTest { - @Mock + @Inject private GraphProvider<TitanGraph> graphProvider; - @Mock - private TitanGraph titanGraph; - - @Mock - private TitanManagement titanManagement; + @Inject + private GraphBackedSearchIndexer graphBackedSearchIndexer; - @BeforeMethod - public void setup() { - MockitoAnnotations.initMocks(this); + @Test + public void verifySystemMixedIndexes() { + TitanGraph titanGraph = graphProvider.get(); + TitanManagement managementSystem = titanGraph.getManagementSystem(); + + TitanGraphIndex vertexIndex = managementSystem.getGraphIndex(Constants.VERTEX_INDEX); + assertNotNull(vertexIndex); + assertTrue(vertexIndex.isMixedIndex()); + assertTrue(Vertex.class.isAssignableFrom(vertexIndex.getIndexedElement())); + + TitanGraphIndex edgeIndex = managementSystem.getGraphIndex(Constants.EDGE_INDEX); + assertNotNull(edgeIndex); + assertTrue(edgeIndex.isMixedIndex()); + assertTrue(Edge.class.isAssignableFrom(edgeIndex.getIndexedElement())); } @Test - public void testSearchIndicesAreInitializedOnConstructionWhenHAIsDisabled() throws IndexException, RepositoryException { - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false); - when(graphProvider.get()).thenReturn(titanGraph); - when(titanGraph.getManagementSystem()).thenReturn(titanManagement); - when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); + public void verifySystemCompositeIndexes() { + TitanGraph titanGraph = graphProvider.get(); + TitanManagement managementSystem = titanGraph.getManagementSystem(); + + verifySystemCompositeIndex(managementSystem, Constants.GUID_PROPERTY_KEY, true); + verifyVertexIndexContains(managementSystem, Constants.GUID_PROPERTY_KEY); - GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration); + verifySystemCompositeIndex(managementSystem, Constants.ENTITY_TYPE_PROPERTY_KEY, false); + verifyVertexIndexContains(managementSystem, Constants.ENTITY_TYPE_PROPERTY_KEY); - verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY); + verifySystemCompositeIndex(managementSystem, Constants.SUPER_TYPES_PROPERTY_KEY, false); + verifyVertexIndexContains(managementSystem, Constants.SUPER_TYPES_PROPERTY_KEY); + + verifySystemCompositeIndex(managementSystem, Constants.TRAIT_NAMES_PROPERTY_KEY, false); + verifyVertexIndexContains(managementSystem, Constants.TRAIT_NAMES_PROPERTY_KEY); } @Test - public void testSearchIndicesAreNotInitializedOnConstructionWhenHAIsEnabled() throws IndexException, RepositoryException { - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); - when(graphProvider.get()).thenReturn(titanGraph); - when(titanGraph.getManagementSystem()).thenReturn(titanManagement); - when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); + public void verifyFullTextIndex() { + TitanGraph titanGraph = graphProvider.get(); + TitanManagement managementSystem = titanGraph.getManagementSystem(); + + TitanGraphIndex fullTextIndex = managementSystem.getGraphIndex(Constants.FULLTEXT_INDEX); + assertTrue(fullTextIndex.isMixedIndex()); + + Arrays.asList(fullTextIndex.getFieldKeys()).contains( + managementSystem.getPropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY)); + } - GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration); + @Test + public void verifyTypeStoreIndexes() { + TitanGraph titanGraph = graphProvider.get(); + TitanManagement managementSystem = titanGraph.getManagementSystem(); - verifyZeroInteractions(titanManagement); + verifySystemCompositeIndex(managementSystem, Constants.TYPENAME_PROPERTY_KEY, true); + verifyVertexIndexContains(managementSystem, Constants.TYPENAME_PROPERTY_KEY); + verifySystemCompositeIndex(managementSystem, Constants.VERTEX_TYPE_PROPERTY_KEY, false); + verifyVertexIndexContains(managementSystem, Constants.VERTEX_TYPE_PROPERTY_KEY); } @Test - public void testIndicesAreReinitializedWhenServerBecomesActive() throws AtlasException { - when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(true); - when(graphProvider.get()).thenReturn(titanGraph); - when(titanGraph.getManagementSystem()).thenReturn(titanManagement); - when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true); + public void verifyUserDefinedTypeIndex() throws AtlasException { + TitanGraph titanGraph = graphProvider.get(); + TitanManagement managementSystem = titanGraph.getManagementSystem(); + + TypeSystem typeSystem = TypeSystem.getInstance(); + + String enumName = "randomEnum" + RandomStringUtils.randomAlphanumeric(10); + EnumType managedType = typeSystem.defineEnumType(enumName, new EnumValue("randomEnumValue", 0)); + + HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = + createClassTypeDef("Database", "Database type description", null, + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createUniqueRequiredAttrDef("managedType", managedType)); + + ClassType databaseType = typeSystem.defineClassType(databaseTypeDefinition); + graphBackedSearchIndexer.onAdd(Arrays.asList(databaseType)); - GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration); - graphBackedSearchIndexer.instanceIsActive(); + verifySystemCompositeIndex(managementSystem, "Database.name", false); + verifyVertexIndexContains(managementSystem, "Database.name"); + + verifySystemCompositeIndex(managementSystem, "Database.managedType", false); + verifyVertexIndexContains(managementSystem, "Database.managedType"); + } + + private void verifyVertexIndexContains(TitanManagement managementSystem, String indexName) { + TitanGraphIndex vertexIndex = managementSystem.getGraphIndex(Constants.VERTEX_INDEX); + PropertyKey[] fieldKeys = vertexIndex.getFieldKeys(); + Arrays.asList(fieldKeys).contains(managementSystem.getPropertyKey(indexName)); + } - verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY); + private void verifySystemCompositeIndex(TitanManagement managementSystem, String indexName, boolean isUnique) { + TitanGraphIndex guidIndex = managementSystem.getGraphIndex(indexName); + assertNotNull(guidIndex); + assertTrue(guidIndex.isCompositeIndex()); + if (isUnique) { + assertTrue(guidIndex.isUnique()); + } else { + assertFalse(guidIndex.isUnique()); + } } }
