Repository: incubator-atlas Updated Branches: refs/heads/master d671b1276 -> 91072c106
ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/91072c10 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/91072c10 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/91072c10 Branch: refs/heads/master Commit: 91072c10669c390338e39a9b705ffdd3bb5a3868 Parents: 89f25d4 Author: Shwetha GS <[email protected]> Authored: Fri Aug 5 18:38:29 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Fri Aug 5 18:39:07 2016 +0530 ---------------------------------------------------------------------- .../falcon/model/FalconDataModelGenerator.java | 10 +- .../org/apache/atlas/fs/model/FSDataModel.scala | 6 +- .../hive/model/HiveDataModelGenerator.java | 10 +- .../hive/bridge/HiveMetastoreBridgeIT.java | 2 +- .../org/apache/atlas/hive/hook/HiveHookIT.java | 2 +- .../sqoop/model/SqoopDataModelGenerator.java | 8 +- release-log.txt | 2 +- .../graph/GraphBackedSearchIndexer.java | 112 ++++++++++++------- .../atlas/services/DefaultMetadataService.java | 7 +- .../GraphBackedDiscoveryServiceTest.java | 4 + .../graph/GraphBackedSearchIndexerTest.java | 11 +- .../org/apache/atlas/query/GremlinTest.scala | 78 ++++++------- .../apache/atlas/query/LineageQueryTest.scala | 28 ++--- .../typesystem/types/AttributeDefinition.java | 2 +- 14 files changed, 158 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java index 3250a23..fc0a9b2 100644 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java @@ -97,7 +97,7 @@ public class FalconDataModelGenerator { private void createClusterEntityClass() throws AtlasException { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null), + false, true, null), // map of tags new AttributeDefinition(TAGS, DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), @@ -113,7 +113,7 @@ public class FalconDataModelGenerator { private void createFeedCreationEntityClass() throws AtlasException { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED, - false, null) + false, false, true, null) }; HierarchicalTypeDefinition<ClassType> definition = @@ -127,7 +127,7 @@ public class FalconDataModelGenerator { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE), new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED, - false, null), + false, false, true, null), new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), // map of tags new AttributeDefinition(TAGS, @@ -156,12 +156,12 @@ public class FalconDataModelGenerator { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE), new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED, - false, null), + false, false, true, null), // map of tags new AttributeDefinition(TAGS, DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), + new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null), // wf properties new AttributeDefinition(WFPROPERTIES, DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala ---------------------------------------------------------------------- diff --git a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala index 4941a5f..ad40340 100644 --- a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala +++ b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala @@ -45,9 +45,9 @@ object FSDataModel extends App { //Is a regular file or a directory. If true, it is a file else a directory "isFile" ~ (boolean, optional, indexed) //Is a symlink or not - "isSymlink" ~ (boolean, optional, indexed) + "isSymlink" ~ (boolean, optional) //Optional and may not be set for a directory - "fileSize" ~ (long, optional, indexed) + "fileSize" ~ (long, optional) "group" ~ (string, optional, indexed) "posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed) } @@ -66,7 +66,7 @@ object FSDataModel extends App { //Making cluster optional since path is already unique containing the namenode URI AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed) "numberOfReplicas" ~ (int, optional, indexed) - "extendedAttributes" ~ (map(string, string), optional, indexed) + "extendedAttributes" ~ (map(string, string), optional) } //TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java index a3d97eb..aac3924 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java @@ -204,7 +204,7 @@ public class HiveDataModelGenerator { private void createDBClass() throws AtlasException { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null), + false, true, null), new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null), @@ -220,7 +220,7 @@ public class HiveDataModelGenerator { private void createColumnClass() throws AtlasException { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), //Making this optional since this is an incompatible change //Reverse attribute to 'columns' in Table @@ -258,7 +258,7 @@ public class HiveDataModelGenerator { new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false, - null),}; + false, true, null),}; HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null, ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); @@ -271,9 +271,9 @@ public class HiveDataModelGenerator { new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null), + false, true, null), new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null), + false, true, null), new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java index 57f93a8..d09db1b 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java @@ -54,7 +54,7 @@ public class HiveMetastoreBridgeIT extends HiveITBase { int tableCount = atlasClient.listEntities(HiveDataTypes.HIVE_TABLE.getName()).size(); - //Now import using import tool - should be no-op + //Now import using import tool - should be no-op. This also tests update since table exists hiveMetaStoreBridge.importTable(atlasClient.getEntity(dbId), DEFAULT_DB, tableName, true); String tableId2 = assertTableIsRegistered(DEFAULT_DB, tableName); assertEquals(tableId2, tableId); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 9258b3e..a5838b4 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -1502,7 +1502,7 @@ public class HiveHookIT extends HiveITBase { final String fmtQuery = "alter database %s set OWNER %s %s"; String query = String.format(fmtQuery, dbName, "USER", owner); - runCommand(query); + runCommandWithDelay(query, 1000); assertDatabaseIsRegistered(dbName, new AssertPredicate() { @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java index 0ea4d7f..3373246 100644 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java +++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java @@ -119,13 +119,13 @@ public class SqoopDataModelGenerator { private void createSqoopDbStoreClass() throws AtlasException { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ new AttributeDefinition(DB_STORE_TYPE, - DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), new AttributeDefinition(DB_STORE_USAGE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition(STORE_URI, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition(SOURCE, - DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null) + DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null) }; HierarchicalTypeDefinition<ClassType> definition = @@ -139,12 +139,12 @@ public class SqoopDataModelGenerator { private void createSqoopProcessClass() throws AtlasException { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ new AttributeDefinition(OPERATION, - DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition(USER, - DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), + DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null), }; HierarchicalTypeDefinition<ClassType> definition = http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index f6daa1c..294e6b3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -3,7 +3,7 @@ Apache Atlas Release Notes --trunk - unreleased INCOMPATIBLE CHANGES: - +ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags) ALL CHANGES: ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index c77004d..f2e40f9 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -101,32 +101,32 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang // create a composite index for guid as its unique createIndexes(management, Constants.GUID_PROPERTY_KEY, String.class, true, - Cardinality.SINGLE, true); + Cardinality.SINGLE, true, true); // create a composite index for entity state - createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true); + createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true, true); // create a mixed index for entity state. Set systemProperty flag deliberately to false // so that it doesnt create a composite index which has issues with // titan 0.5.4 - Refer https://groups.google.com/forum/#!searchin/aureliusgraphs/hemanth/aureliusgraphs/bx7T843mzXU/fjAsclx7GAAJ - createIndexes(management, Constants.STATE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, false); + createStateMixedIndex(management); // create a composite index for entity state createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false, - Cardinality.SINGLE, true); + Cardinality.SINGLE, false, false); // create a composite and mixed index for type since it can be combined with other keys createIndexes(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, - true); + true, true); // create a composite and mixed index for type since it can be combined with other keys createIndexes(management, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET, - true); + true, true); // create a composite and mixed index for traitNames since it can be combined with other // keys. Traits must be a set and not a list. createIndexes(management, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET, - true); + true, true); // Index for full text search createFullTextIndex(management); @@ -142,6 +142,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } } + private void createStateMixedIndex(TitanManagement management) { + PropertyKey propertyKey = management.getPropertyKey(Constants.STATE_PROPERTY_KEY); + if (propertyKey == null) { + propertyKey = management.makePropertyKey(Constants.STATE_PROPERTY_KEY).dataType(String.class).cardinality(Cardinality.SINGLE) + .make(); + } + enhanceMixedIndex(management, Constants.STATE_PROPERTY_KEY, String.class, Cardinality.SINGLE, propertyKey); + } + + private void createFullTextIndex(TitanManagement management) { PropertyKey fullText = management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make(); @@ -155,11 +165,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang private void createTypeStoreIndexes(TitanManagement management) { //Create unique index on typeName createIndexes(management, Constants.TYPENAME_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, - true); + true, true); //create index on vertex type createIndexes(management, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, - true); + true, true); } /** @@ -172,7 +182,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang public void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException { TitanManagement management = titanGraph.getManagementSystem(); for (IDataType dataType : dataTypes) { - LOG.info("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass()); + LOG.debug("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass()); try { addIndexForType(management, dataType); LOG.info("Index creation for type {} complete", dataType.getName()); @@ -225,9 +235,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang private void createIndexForFields(TitanManagement management, IDataType dataType, Map<String, AttributeInfo> fields) { for (AttributeInfo field : fields.values()) { - if (field.isIndexable) { - createIndexForAttribute(management, dataType.getName(), field); - } + createIndexForAttribute(management, dataType.getName(), field); } } @@ -237,12 +245,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang case PRIMITIVE: Cardinality cardinality = getCardinality(field.multiplicity); createIndexes(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique, - cardinality, false); + cardinality, false, field.isIndexable); break; case ENUM: cardinality = getCardinality(field.multiplicity); - createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false); + createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false, field.isIndexable); break; case ARRAY: @@ -317,7 +325,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang private PropertyKey createIndexes(TitanManagement management, String propertyName, Class propertyClass, boolean isUnique, Cardinality cardinality, - boolean isSystemProperty) { + boolean createCompositeForAttribute, boolean createCompositeWithTypeandSuperTypes) { PropertyKey propertyKey = management.getPropertyKey(propertyName); if (propertyKey == null) { propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality) @@ -325,52 +333,74 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang enhanceMixedIndex(management, propertyName, propertyClass, cardinality, propertyKey); - if (isSystemProperty) { - createCompositeIndex(management, propertyName, propertyClass, propertyKey, isUnique); - } else if (isUnique) { - // send uniqueness as false because there can be many vertexes with the same property value - // but state can be active / deleted. - createCompositeIndexWithTypeName(management, propertyName, propertyClass, propertyKey); - } + } + + if (createCompositeForAttribute) { + createCompositeIndex(management, propertyName, propertyClass, propertyKey, isUnique); + } else if (createCompositeWithTypeandSuperTypes) { + //Index with typename since typename+property key queries need to speed up + createCompositeIndexWithTypeName(management, propertyName, propertyClass, propertyKey); + createCompositeIndexWithSuperTypeName(management, propertyName, propertyClass, propertyKey); } return propertyKey; } private void createCompositeIndex(TitanManagement management, String propertyName, Class propertyClass, - PropertyKey propertyKey, boolean enforceUniqueness) { - LOG.info("Creating composite index for property {} of type {} ", propertyName, - propertyClass.getName()); - TitanManagement.IndexBuilder indexBuilder = + PropertyKey propertyKey, boolean enforceUniqueness) { + LOG.debug("Creating composite index for property {} of type {} ", propertyName, + propertyClass.getName()); + + TitanGraphIndex existingIndex = management.getGraphIndex(propertyName); + if ( existingIndex == null) { + TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class).addKey(propertyKey); - if (enforceUniqueness) { - indexBuilder.unique(); + if (enforceUniqueness) { + LOG.debug("Enabling unique index for property {} of type {} ", propertyName, + propertyClass.getName()); + indexBuilder.unique(); + } + indexBuilder.buildCompositeIndex(); } - indexBuilder.buildCompositeIndex(); LOG.info("Created composite index for property {} of type {} ", propertyName, propertyClass.getName()); } private void createCompositeIndexWithTypeName(TitanManagement management, String propertyName, Class propertyClass, PropertyKey propertyKey) { - LOG.info("Creating composite index for property {} of type {} and {}", propertyName, - propertyClass.getName(), Constants.ENTITY_TYPE_PROPERTY_KEY); - PropertyKey typePropertyKey = management.getPropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY); + createCompositeIndexWithSystemProperty(management, propertyName, propertyClass, propertyKey, Constants.ENTITY_TYPE_PROPERTY_KEY, Cardinality.SINGLE); + } + + private void createCompositeIndexWithSuperTypeName(TitanManagement management, String propertyName, Class propertyClass, + PropertyKey propertyKey) { + createCompositeIndexWithSystemProperty(management, propertyName, propertyClass, propertyKey, Constants.SUPER_TYPES_PROPERTY_KEY, Cardinality.SET); + } + + private void createCompositeIndexWithSystemProperty(TitanManagement management, String propertyName, Class propertyClass, + PropertyKey propertyKey, final String systemPropertyKey, Cardinality cardinality) { + LOG.debug("Creating composite index for property {} of type {} and {}", propertyName, + propertyClass.getName(), systemPropertyKey); + PropertyKey typePropertyKey = management.getPropertyKey(systemPropertyKey); if (typePropertyKey == null) { - typePropertyKey = management.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY). - dataType(String.class).cardinality(Cardinality.SINGLE) + typePropertyKey = management.makePropertyKey(systemPropertyKey). + dataType(String.class).cardinality(cardinality) .make(); } - TitanManagement.IndexBuilder indexBuilder = - management.buildIndex(propertyName + Constants.ENTITY_TYPE_PROPERTY_KEY, Vertex.class). - addKey(propertyKey).addKey(typePropertyKey); - indexBuilder.buildCompositeIndex(); - LOG.info("Created composite index for property {} of type {} and {}", propertyName, propertyClass.getName(), Constants.ENTITY_TYPE_PROPERTY_KEY); + final String indexName = propertyName + systemPropertyKey; + TitanGraphIndex existingIndex = management.getGraphIndex(indexName); + + if ( existingIndex == null) { + TitanManagement.IndexBuilder indexBuilder = + management.buildIndex(indexName, Vertex.class). + addKey(propertyKey).addKey(typePropertyKey); + indexBuilder.buildCompositeIndex(); + LOG.info("Created composite index for property {} of type {} and {}", propertyName, propertyClass.getName(), systemPropertyKey); + } } private void enhanceMixedIndex(TitanManagement management, String propertyName, Class propertyClass, Cardinality cardinality, PropertyKey propertyKey) { if (checkIfMixedIndexApplicable(propertyClass, cardinality)) { //Use backing index - LOG.info("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName()); + LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName()); TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX); management.addIndexKey(vertexIndex, propertyKey); LOG.info("Created backing index for property {} of type {} ", propertyName, propertyClass.getName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index e6b75f4..4d05d49 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -185,15 +185,14 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang private void createSuperTypes() throws AtlasException { HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(), - TypesUtil.createUniqueRequiredAttrDef(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, - DataTypes.STRING_TYPE)); + new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null)); createType(referenceableType); HierarchicalTypeDefinition<ClassType> assetType = TypesUtil .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(), - TypesUtil.createRequiredAttrDef(AtlasClient.NAME, DataTypes.STRING_TYPE), + new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE), - TypesUtil.createOptionalAttrDef(AtlasClient.OWNER, DataTypes.STRING_TYPE)); + new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)); createType(assetType); HierarchicalTypeDefinition<ClassType> infraType = TypesUtil http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index df3fe87..74438cd 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -385,6 +385,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_table where name='sales_fact', db where name='Reporting'", 0}, {"hive_partition as p where values = ['2015-01-01']", 1}, // {"StorageDesc select cols", 6} //Not working since loading of lists needs to be fixed yet + + //check supertypeNames + {"DataSet where name='sales_fact'", 1}, + {"Asset where name='sales_fact'", 1} }; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java index 1fa0619..3291e72 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java @@ -135,6 +135,7 @@ public class GraphBackedSearchIndexerTest { verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY, false); verifyVertexIndexContains(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY); + verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.SUPER_TYPES_PROPERTY_KEY, false); verifyVertexIndexContains(managementSystem, "Database.managedType"); } @@ -146,13 +147,13 @@ public class GraphBackedSearchIndexerTest { } private void verifySystemCompositeIndex(TitanManagement managementSystem, String indexName, boolean isUnique) { - TitanGraphIndex guidIndex = managementSystem.getGraphIndex(indexName); - assertNotNull(guidIndex); - assertTrue(guidIndex.isCompositeIndex()); + TitanGraphIndex systemIndex = managementSystem.getGraphIndex(indexName); + assertNotNull(systemIndex); + assertTrue(systemIndex.isCompositeIndex()); if (isUnique) { - assertTrue(guidIndex.isUnique()); + assertTrue(systemIndex.isUnique()); } else { - assertFalse(guidIndex.isUnique()); + assertFalse(systemIndex.isUnique()); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala index 8fbcdbc..fa48c0e 100755 --- a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala +++ b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala @@ -78,7 +78,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -91,7 +91,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -104,7 +104,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | | }, @@ -118,7 +118,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | } | ] @@ -174,7 +174,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -187,7 +187,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -200,7 +200,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -213,7 +213,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | } | ] @@ -253,7 +253,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -266,7 +266,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -279,7 +279,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -292,7 +292,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | } | ] @@ -331,7 +331,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -344,7 +344,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | } | ] @@ -380,7 +380,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -393,7 +393,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -406,7 +406,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -419,7 +419,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] @@ -516,7 +516,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -529,7 +529,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -542,7 +542,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -555,7 +555,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] @@ -606,7 +606,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -619,7 +619,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -632,7 +632,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -645,7 +645,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] @@ -781,7 +781,7 @@ class GremlinTest extends BaseGremlinTest { @Test def testArith { val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))). select(id("name"), id("createTime") + int(1)), g, gp) - validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _src1.name, (_src1.createTime + 1) as (_src1.createTime + 1)\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct3\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_src1.name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"(_src1.createTime + 1)\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \" $typeName$\":\"__tempQueryResultStruct3\",\n \"(_src1.createTime + 1)\":1501,\n \"_src1.name\":\"Reporting\"\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _src1.name, (_src1.createTime + 1) as (_src1.createTime + 1)\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct3\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_src1.name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"(_src1.createTime + 1)\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct3\",\n \"(_src1.createTime + 1)\":1501,\n \"_src1.name\":\"Reporting\"\n }\n ]\n}") } @Test def testComparisonLogical { @@ -806,7 +806,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -819,7 +819,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -832,7 +832,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | }, | { @@ -845,7 +845,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite": false, | "isUnique": false, - | "isIndexable": true, + | "isIndexable": false, | "reverseAttributeName": null | } | ] @@ -872,7 +872,7 @@ class GremlinTest extends BaseGremlinTest { where((isTrait("Dimension"))). select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp ) - validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct5\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\ n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct5\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n }\n ]\n}") } @Test def testJoinAndSelect2 { @@ -881,7 +881,7 @@ class GremlinTest extends BaseGremlinTest { .or(id("name").`=`(string("Reporting")))).field("Table").as("tab") .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp ) - validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"_ _tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\ "__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}") } @Test def testJoinAndSelect3 { @@ -891,7 +891,7 @@ class GremlinTest extends BaseGremlinTest { .or(id("db1").hasField("owner"))).field("Table").as("tab") .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp ) - validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}") } @Test def testJoinAndSelect4 { @@ -900,7 +900,7 @@ class GremlinTest extends BaseGremlinTest { where((isTrait("Dimension"))). select(id("db1").as("dbO"), id("tab").field("name").as("tabName")), g, gp ) - validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1 as dbO, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbO\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$ty peName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"customer_dim\"\n }\n ]\n}") + validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1 as dbO, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbO\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$ typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"customer_dim\"\n }\n ]\n}") } @Test def testArrayComparision { @@ -924,7 +924,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] @@ -963,7 +963,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] @@ -1003,7 +1003,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -1016,7 +1016,7 @@ class GremlinTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala index 021a8d2..c8b635a 100755 --- a/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala +++ b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala @@ -81,7 +81,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -94,7 +94,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -107,7 +107,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -120,7 +120,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] @@ -213,7 +213,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -226,7 +226,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -239,7 +239,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -252,7 +252,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] @@ -365,7 +365,7 @@ class LineageQueryTest extends BaseGremlinTest { }, "isComposite":false, "isUnique":false, - "isIndexable":true, + "isIndexable":false, "reverseAttributeName":null }, { @@ -378,7 +378,7 @@ class LineageQueryTest extends BaseGremlinTest { }, "isComposite":false, "isUnique":false, - "isIndexable":true, + "isIndexable":false, "reverseAttributeName":null } ] @@ -434,7 +434,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -447,7 +447,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -460,7 +460,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | }, | { @@ -473,7 +473,7 @@ class LineageQueryTest extends BaseGremlinTest { | }, | "isComposite":false, | "isUnique":false, - | "isIndexable":true, + | "isIndexable":false, | "reverseAttributeName":null | } | ] http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java index 29c3450..f556223 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java @@ -38,7 +38,7 @@ public final class AttributeDefinition { public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite, String reverseAttributeName) { - this(name, dataTypeName, multiplicity, isComposite, false, true, reverseAttributeName); + this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName); }
