Repository: incubator-atlas Updated Branches: refs/heads/master 96059e0a9 -> 39c52b2b7
ATLAS-79 Unique constraint is not honoured (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/39c52b2b Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/39c52b2b Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/39c52b2b Branch: refs/heads/master Commit: 39c52b2b7b1b27cd52dbba00599f74b7cf8c635e Parents: 96059e0 Author: Shwetha GS <[email protected]> Authored: Mon Aug 3 15:49:18 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Mon Aug 3 15:49:18 2015 +0530 ---------------------------------------------------------------------- release-log.txt | 1 + .../org/apache/atlas/repository/Constants.java | 4 - .../graph/GraphBackedSearchIndexer.java | 105 ++++++++----------- .../web/resources/EntityJerseyResourceIT.java | 43 ++++++++ 4 files changed, 89 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 3642095..0965bf6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ALL CHANGES: +ATLAS-79 Unique constraint is not honoured (shwethags) ATLAS-25 Fix Atlas on Java 8 (sandeep.samudrala via shwethags) ATLAS-86 Jenkins build failing as of build #41 (shwethags) ATLAS-80 Support for variables in application properties (shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/repository/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java index fe43ad0..d5d3b3c 100755 --- a/repository/src/main/java/org/apache/atlas/repository/Constants.java +++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java @@ -26,19 +26,16 @@ public final class Constants { public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__"; public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid"; - public static final String GUID_INDEX = "guid_index"; /** * Entity type name property key. */ public static final String ENTITY_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; - public static final String ENTITY_TYPE_INDEX = "type_index"; /** * Entity type's super types property key. */ public static final String SUPER_TYPES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames"; - public static final String SUPER_TYPES_INDEX = "super_types_index"; /** * Full-text for the entity for enabling full-text search. @@ -57,7 +54,6 @@ public final class Constants { * Trait names property key and index name. */ public static final String TRAIT_NAMES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "traitNames"; - public static final String TRAIT_NAMES_INDEX = "trait_names_index"; public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version"; public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp"; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index c43c64f..10babed 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -36,6 +36,7 @@ import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructType; import org.apache.atlas.typesystem.types.TraitType; import org.slf4j.Logger; @@ -44,8 +45,9 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.ArrayList; import java.util.Collection; -import java.util.Date; +import java.util.List; import java.util.Map; /** @@ -59,6 +61,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer { private TitanManagement management; + List<Class> MIXED_INDEX_EXCLUSIONS = new ArrayList() {{ + add(Boolean.class); + add(BigDecimal.class); + add(BigInteger.class); + }}; + @Inject public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider) throws RepositoryException { @@ -74,7 +82,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer { * Initializes the indices for the graph - create indices for Global Vertex Keys */ private void initialize() { - if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) { + if (management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)) { LOG.info("Global indexes already exist for graph"); return; } @@ -85,20 +93,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer { management.buildIndex(Constants.EDGE_INDEX, Edge.class).buildMixedIndex(Constants.BACKING_INDEX); // create a composite index for guid as its unique - createCompositeIndex(Constants.GUID_INDEX, Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE); + createCompositeAndMixedIndex(Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, true); // create a composite and mixed index for type since it can be combined with other keys - createCompositeAndMixedIndex(Constants.ENTITY_TYPE_INDEX, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, - false, Cardinality.SINGLE); + createCompositeAndMixedIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true); // create a composite and mixed index for type since it can be combined with other keys - createCompositeAndMixedIndex(Constants.SUPER_TYPES_INDEX, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, - false, Cardinality.SET); + createCompositeAndMixedIndex(Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET, true); // create a composite and mixed index for traitNames since it can be combined with other // keys. Traits must be a set and not a list. - createCompositeAndMixedIndex(Constants.TRAIT_NAMES_INDEX, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, - false, Cardinality.SET); + createCompositeAndMixedIndex(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET, true); // Index for full text search createFullTextIndex(); @@ -125,13 +130,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer { private void createTypeStoreIndexes() { //Create unique index on typeName - createCompositeIndex(Constants.TYPENAME_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY, String.class, true, - Cardinality.SINGLE); + createCompositeAndMixedIndex(Constants.TYPENAME_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, true); //create index on vertex type - createCompositeIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, - false, Cardinality.SINGLE); - + createCompositeAndMixedIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true); } /** @@ -201,11 +203,14 @@ public class GraphBackedSearchIndexer implements SearchIndexer { final String propertyName = typeName + "." + field.name; switch (field.dataType().getTypeCategory()) { case PRIMITIVE: - createVertexMixedIndex(propertyName, getPrimitiveClass(field.dataType())); + Cardinality cardinality = getCardinality(field.multiplicity); + createCompositeAndMixedIndex(propertyName, getPrimitiveClass(field.dataType()), field.isUnique, + cardinality, false); break; case ENUM: - createVertexMixedIndex(propertyName, String.class); + cardinality = getCardinality(field.multiplicity); + createCompositeAndMixedIndex(propertyName, String.class, field.isUnique, cardinality, false); break; case ARRAY: @@ -264,7 +269,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer { throw new IllegalArgumentException("unknown data type " + dataType); } -/* + private Cardinality getCardinality(Multiplicity multiplicity) { if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) { return Cardinality.SINGLE; @@ -277,66 +282,46 @@ public class GraphBackedSearchIndexer implements SearchIndexer { // todo - default to LIST as this is the most forgiving return Cardinality.LIST; } -*/ - - private void createCompositeAndMixedIndex(String indexName, String propertyName, Class propertyClass, - boolean isUnique, Cardinality cardinality) { - createCompositeIndex(indexName, propertyName, propertyClass, isUnique, cardinality); - createVertexMixedIndex(propertyName, propertyClass); - } - - private PropertyKey createCompositeIndex(String indexName, String propertyName, Class propertyClass, - boolean isUnique, Cardinality cardinality) { - PropertyKey propertyKey = management.getPropertyKey(propertyName); - if (propertyKey == null) { - propertyKey = - management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality).make(); - - TitanManagement.IndexBuilder indexBuilder = - management.buildIndex(indexName, Vertex.class).addKey(propertyKey); - - if (isUnique) { - indexBuilder = indexBuilder.unique(); - } - - indexBuilder.buildCompositeIndex(); - LOG.info("Created index for property {} in composite index {}", propertyName, indexName); - } - return propertyKey; - } - private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) { + private PropertyKey createCompositeAndMixedIndex(String propertyName, Class propertyClass, + boolean isUnique, Cardinality cardinality, boolean force) { PropertyKey propertyKey = management.getPropertyKey(propertyName); if (propertyKey == null) { - // ignored cardinality as Can only index single-valued property keys on vertices - propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).make(); + propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality) + .make(); - if (!checkIfMixedIndexApplicable(propertyClass)) { - LOG.debug("Creating composite index for property {} of type {} ", propertyName, - propertyClass.getName()); - //Use standard index as backing index only supports string, int and geo types - management.buildIndex(propertyName, Vertex.class).addKey(propertyKey).buildCompositeIndex(); - LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName()); - } else { + if (checkIfMixedIndexApplicable(propertyClass, cardinality)) { //Use backing index LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName()); TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX); management.addIndexKey(vertexIndex, propertyKey); LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName()); } - LOG.info("Created mixed vertex index for property {}", propertyName); + + //Create mixed index only for meta properties and unique constraints: + //Unique can't be achieved with backing/mixed index + //Creating composite index for every attribute will bloat up the index + if (force || isUnique) { + LOG.debug("Creating composite index for property {} of type {} ", propertyName, + propertyClass.getName()); + TitanManagement.IndexBuilder indexBuilder = + management.buildIndex(propertyName, Vertex.class).addKey(propertyKey); + if (isUnique) { + indexBuilder.unique(); + } + indexBuilder.buildCompositeIndex(); + LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName()); + } } return propertyKey; } - private boolean checkIfMixedIndexApplicable(Class propertyClass) { - if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class) { - return false; - } - return true; + private boolean checkIfMixedIndexApplicable(Class propertyClass, Cardinality cardinality) { + return !(MIXED_INDEX_EXCLUSIONS.contains(propertyClass) || cardinality == Cardinality.LIST || cardinality == + Cardinality.SET); } public void commit() throws IndexException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index 7361173..e6a7325 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -116,6 +116,49 @@ public class EntityJerseyResourceIT extends BaseResourceIT { } @Test + public void testUniqueAttribute() throws Exception { + //create type + String typeName = "type" + randomString(); + HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil + .createClassTypeDef(typeName, ImmutableList.<String>of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE)); + TypesDef typesDef = TypeUtils + .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(typeDefinition)); + createType(typesDef); + + //create entity + String name = "name" + randomString(); + Referenceable referenceable = new Referenceable(typeName); + referenceable.set("name", name); + createInstance(referenceable); + + //create entity with same name again - should fail + try { + createInstance(referenceable); + Assert.fail("Expected exception"); + } catch(Exception e) { + //expected exception + } + + //create another type with same attribute - should allow + typeName = "type" + randomString(); + typeDefinition = TypesUtil + .createClassTypeDef(typeName, ImmutableList.<String>of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE)); + typesDef = TypeUtils + .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(typeDefinition)); + createType(typesDef); + + referenceable = new Referenceable(typeName); + referenceable.set("name", name); + createInstance(referenceable); + } + + @Test public void testSubmitEntityWithBadDateFormat() throws Exception { try {
