ATLAS-806 Create default taxonomy at server startup (jspeidel via yhemanth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/41ebb5ec Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/41ebb5ec Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/41ebb5ec Branch: refs/heads/0.7-incubating Commit: 41ebb5ec37a46a3f1e11dbc4f1654b5e9b3855ef Parents: 79f208e Author: Shwetha GS <[email protected]> Authored: Wed Jun 29 11:57:32 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Wed Jun 29 11:57:32 2016 +0530 ---------------------------------------------------------------------- .../atlas/catalog/TaxonomyResourceProvider.java | 142 +++++++++++++++---- .../catalog/TaxonomyResourceProviderTest.java | 141 ++++++++++++++++-- distro/src/conf/atlas-application.properties | 5 +- release-log.txt | 1 + 4 files changed, 251 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ebb5ec/catalog/src/main/java/org/apache/atlas/catalog/TaxonomyResourceProvider.java ---------------------------------------------------------------------- diff --git a/catalog/src/main/java/org/apache/atlas/catalog/TaxonomyResourceProvider.java b/catalog/src/main/java/org/apache/atlas/catalog/TaxonomyResourceProvider.java index 3a5d9be..ee71e47 100644 --- a/catalog/src/main/java/org/apache/atlas/catalog/TaxonomyResourceProvider.java +++ b/catalog/src/main/java/org/apache/atlas/catalog/TaxonomyResourceProvider.java @@ -18,9 +18,14 @@ package org.apache.atlas.catalog; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; import org.apache.atlas.catalog.definition.TaxonomyResourceDefinition; import org.apache.atlas.catalog.exception.*; import org.apache.atlas.catalog.query.AtlasQuery; +import org.apache.commons.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.*; @@ -28,8 +33,18 @@ import java.util.*; * Provider for taxonomy resources. */ public class TaxonomyResourceProvider extends BaseResourceProvider implements ResourceProvider { + private static final Logger LOG = LoggerFactory.getLogger(TaxonomyResourceProvider.class); + public static final String DEFAULT_TAXONOMY_NAME = "Catalog"; + public static final String DEFAULT_TAXONOMY_DESCRIPTION = "Business Catalog"; private final TermResourceProvider termResourceProvider; + // This is a cached value to prevent checking for taxonomy objects in every API call. + // It is updated once per lifetime of the application. + // TODO: If a taxonomy is deleted outside of this application, this value is not updated + // TODO: and there is no way in which a taxonomy will be auto-created. + // TODO: Assumption is that if a taxonomy is deleted externally, it will be created externally as well. + private static boolean taxonomyAutoInitializationChecked = false; + public TaxonomyResourceProvider(AtlasTypeSystem typeSystem) { super(typeSystem, new TaxonomyResourceDefinition()); termResourceProvider = new TermResourceProvider(typeSystem); @@ -37,38 +52,36 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re @Override public Result getResourceById(Request request) throws ResourceNotFoundException { - AtlasQuery atlasQuery; - try { - atlasQuery = queryFactory.createTaxonomyQuery(request); - } catch (InvalidQueryException e) { - throw new CatalogRuntimeException("Unable to compile internal Taxonomy query: " + e, e); + synchronized (TaxonomyResourceProvider.class) { + createDefaultTaxonomyIfNeeded(); } - Collection<Map<String, Object>> results = atlasQuery.execute(); - if (results.isEmpty()) { - throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.", - request.getProperty(resourceDefinition.getIdPropertyName()))); - } - return new Result(results); + return doGetResourceById(request); } @Override public Result getResources(Request request) throws InvalidQueryException, ResourceNotFoundException { - AtlasQuery atlasQuery = queryFactory.createTaxonomyQuery(request); - return new Result(atlasQuery.execute()); + synchronized (TaxonomyResourceProvider.class) { + createDefaultTaxonomyIfNeeded(); + } + return doGetResources(request); } @Override - public synchronized void createResource(Request request) + public void createResource(Request request) throws InvalidPayloadException, ResourceAlreadyExistsException { + // not checking for default taxonomy in create per requirements resourceDefinition.validateCreatePayload(request); - ensureTaxonomyDoesntExist(request); - typeSystem.createEntity(resourceDefinition, request); + synchronized (TaxonomyResourceProvider.class) { + ensureTaxonomyDoesntExist(request); + doCreateResource(request); + } } @Override public Collection<String> createResources(Request request) throws InvalidQueryException, ResourceNotFoundException { - throw new UnsupportedOperationException("Creating multiple Taxonomies in a request is not currently supported"); + throw new UnsupportedOperationException( + "Creating multiple Taxonomies in a request is not currently supported"); } @Override @@ -81,21 +94,36 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re @Override public void updateResourceById(Request request) throws ResourceNotFoundException, InvalidPayloadException { resourceDefinition.validateUpdatePayload(request); + AtlasQuery atlasQuery; try { atlasQuery = queryFactory.createTaxonomyQuery(request); } catch (InvalidQueryException e) { throw new CatalogRuntimeException("Unable to compile internal Term query: " + e, e); } - if (atlasQuery.execute(request.getUpdateProperties()).isEmpty()) { - throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.", - request.getQueryProperties().get("name"))); + + synchronized (TaxonomyResourceProvider.class) { + createDefaultTaxonomyIfNeeded(); + if (atlasQuery.execute(request.getUpdateProperties()).isEmpty()) { + throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.", + request.getQueryProperties().get("name"))); + } } } + private String getResourceId(Request request) throws ResourceNotFoundException { + request.addAdditionalSelectProperties(Collections.singleton("id")); + // will result in expected ResourceNotFoundException if taxonomy doesn't exist + Result result = getResourceById(request); + return String.valueOf(result.getPropertyMaps().iterator().next().get("id")); + } + + //todo: this is currently required because the expected exception isn't thrown by the Atlas repository + //todo: when an attempt is made to create an entity that already exists + // must be called from within class monitor private void ensureTaxonomyDoesntExist(Request request) throws ResourceAlreadyExistsException { try { - getResourceById(request); + doGetResourceById(request); throw new ResourceAlreadyExistsException(String.format("Taxonomy '%s' already exists.", request.getProperty("name"))); } catch (ResourceNotFoundException e) { @@ -103,11 +131,73 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re } } - private String getResourceId(Request request) throws ResourceNotFoundException { - request.addAdditionalSelectProperties(Collections.singleton("id")); - // will result in expected ResourceNotFoundException if taxonomy doesn't exist - Result result = getResourceById(request); - return String.valueOf(result.getPropertyMaps().iterator().next().get("id")); + // must be called from within class monitor + private Result doGetResourceById(Request request) throws ResourceNotFoundException { + AtlasQuery atlasQuery; + try { + atlasQuery = queryFactory.createTaxonomyQuery(request); + } catch (InvalidQueryException e) { + throw new CatalogRuntimeException("Unable to compile internal Taxonomy query: " + e, e); + } + + Collection<Map<String, Object>> resultSet = atlasQuery.execute(); + if (resultSet.isEmpty()) { + throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.", + request.getProperty(resourceDefinition.getIdPropertyName()))); + } + return new Result(resultSet); + } + + // must be called from within class monitor + private Result doGetResources(Request request) throws InvalidQueryException, ResourceNotFoundException { + AtlasQuery atlasQuery = queryFactory.createTaxonomyQuery(request); + return new Result(atlasQuery.execute()); + } + + // must be called from within class monitor + private void doCreateResource(Request request) throws ResourceAlreadyExistsException { + typeSystem.createEntity(resourceDefinition, request); + taxonomyAutoInitializationChecked = true; + } + + // must be called from within class monitor + private void createDefaultTaxonomyIfNeeded() { + if (! autoInitializationChecked()) { + try { + LOG.info("Checking if default taxonomy needs to be created."); + // if any business taxonomy has been created, don't create one more - hence searching to + // see if any taxonomy exists. + if (doGetResources(new CollectionRequest(null, null)).getPropertyMaps().isEmpty()) { + LOG.info("No taxonomies found - going to create default taxonomy."); + Map<String, Object> requestProperties = new HashMap<>(); + String defaultTaxonomyName = DEFAULT_TAXONOMY_NAME; + try { + Configuration configuration = ApplicationProperties.get(); + defaultTaxonomyName = configuration.getString("atlas.taxonomy.default.name", + defaultTaxonomyName); + } catch (AtlasException e) { + LOG.warn("Unable to read Atlas configuration, will use {} as default taxonomy name", + defaultTaxonomyName, e); + } + requestProperties.put("name", defaultTaxonomyName); + requestProperties.put("description", DEFAULT_TAXONOMY_DESCRIPTION); + + doCreateResource(new InstanceRequest(requestProperties)); + LOG.info("Successfully created default taxonomy {}.", defaultTaxonomyName); + } else { + taxonomyAutoInitializationChecked = true; + LOG.info("Some taxonomy exists, not creating default taxonomy"); + } + } catch (InvalidQueryException | ResourceNotFoundException e) { + LOG.error("Unable to query for existing taxonomies due to internal error.", e); + } catch (ResourceAlreadyExistsException e) { + LOG.info("Attempted to create default taxonomy and it already exists."); + } + } + } + + protected boolean autoInitializationChecked() { + return taxonomyAutoInitializationChecked; } protected TermResourceProvider getTermResourceProvider() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ebb5ec/catalog/src/test/java/org/apache/atlas/catalog/TaxonomyResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/catalog/src/test/java/org/apache/atlas/catalog/TaxonomyResourceProviderTest.java b/catalog/src/test/java/org/apache/atlas/catalog/TaxonomyResourceProviderTest.java index b833c6e..5a03414 100644 --- a/catalog/src/test/java/org/apache/atlas/catalog/TaxonomyResourceProviderTest.java +++ b/catalog/src/test/java/org/apache/atlas/catalog/TaxonomyResourceProviderTest.java @@ -58,7 +58,110 @@ public class TaxonomyResourceProviderTest { expect(query.execute()).andReturn(queryResult); replay(typeSystem, queryFactory, query); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); + provider.setQueryFactory(queryFactory); + + Map<String, Object> requestProperties = new HashMap<>(); + requestProperties.put("name", "taxonomyName"); + Request userRequest = new InstanceRequest(requestProperties); + + Result result = provider.getResourceById(userRequest); + + assertEquals(1, result.getPropertyMaps().size()); + assertEquals(queryResultRow, result.getPropertyMaps().iterator().next()); + + Request request = requestCapture.getValue(); + assertNull(request.getQueryString()); + assertEquals(0, request.getAdditionalSelectProperties().size()); + assertEquals(requestProperties, request.getQueryProperties()); + + verify(typeSystem, queryFactory, query); + } + + @Test + public void testGetResourceById_notInitialized_createDefaultTaxonomy() throws Exception { + AtlasTypeSystem typeSystem = createStrictMock(AtlasTypeSystem.class); + QueryFactory queryFactory = createStrictMock(QueryFactory.class); + AtlasQuery query = createStrictMock(AtlasQuery.class); + Capture<Request> checkForAnyTaxonomiesCapture = newCapture(); + Capture<Request> createDefaultTaxonomyRequestCapture = newCapture(); + Capture<Request> requestCapture = newCapture(); + Capture<ResourceDefinition> resourceDefinitionCapture = newCapture(); + + Collection<Map<String, Object>> queryResult = new ArrayList<>(); + Map<String, Object> queryResultRow = new HashMap<>(); + queryResult.add(queryResultRow); + queryResultRow.put("name", "taxonomyName"); + queryResultRow.put("description", "test taxonomy description"); + queryResultRow.put("creation_time", "04/20/2016"); + + // mock expectations + expect(queryFactory.createTaxonomyQuery(capture(checkForAnyTaxonomiesCapture))).andReturn(query); + expect(query.execute()).andReturn(Collections.<Map<String, Object>>emptySet()); + typeSystem.createEntity(capture(resourceDefinitionCapture), capture(createDefaultTaxonomyRequestCapture)); + expect(queryFactory.createTaxonomyQuery(capture(requestCapture))).andReturn(query); + expect(query.execute()).andReturn(queryResult); + replay(typeSystem, queryFactory, query); + + TestTaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); + provider.setInitialized(false); + provider.setQueryFactory(queryFactory); + + Map<String, Object> requestProperties = new HashMap<>(); + requestProperties.put("name", "taxonomyName"); + Request userRequest = new InstanceRequest(requestProperties); + + Result result = provider.getResourceById(userRequest); + + assertEquals(1, result.getPropertyMaps().size()); + assertEquals(queryResultRow, result.getPropertyMaps().iterator().next()); + + Request request = requestCapture.getValue(); + assertNull(request.getQueryString()); + assertEquals(0, request.getAdditionalSelectProperties().size()); + assertEquals(requestProperties, request.getQueryProperties()); + + Request checkForAnyTaxonomiesRequest = checkForAnyTaxonomiesCapture.getValue(); + assertNull(checkForAnyTaxonomiesRequest.getQueryString()); + assertEquals(checkForAnyTaxonomiesRequest.getAdditionalSelectProperties().size(), 0); + assertEquals(checkForAnyTaxonomiesRequest.getQueryProperties().size(), 0); + + Request createDefaultTaxonomyRequest = createDefaultTaxonomyRequestCapture.getValue(); + assertNull(createDefaultTaxonomyRequest.getQueryString()); + assertEquals(createDefaultTaxonomyRequest.getAdditionalSelectProperties().size(), 0); + assertEquals(createDefaultTaxonomyRequest.getQueryProperties().size(), 2); + assertEquals(createDefaultTaxonomyRequest.getQueryProperties().get("name"), + TaxonomyResourceProvider.DEFAULT_TAXONOMY_NAME); + assertEquals(createDefaultTaxonomyRequest.getQueryProperties().get("description"), + TaxonomyResourceProvider.DEFAULT_TAXONOMY_DESCRIPTION); + + verify(typeSystem, queryFactory, query); + } + + @Test + public void testGetResourceById_notInitialized_taxonomyAlreadyExists() throws Exception { + AtlasTypeSystem typeSystem = createStrictMock(AtlasTypeSystem.class); + QueryFactory queryFactory = createStrictMock(QueryFactory.class); + AtlasQuery query = createStrictMock(AtlasQuery.class); + Capture<Request> checkForAnyTaxonomiesCapture = newCapture(); + Capture<Request> requestCapture = newCapture(); + + Collection<Map<String, Object>> queryResult = new ArrayList<>(); + Map<String, Object> queryResultRow = new HashMap<>(); + queryResult.add(queryResultRow); + queryResultRow.put("name", "taxonomyName"); + queryResultRow.put("description", "test taxonomy description"); + queryResultRow.put("creation_time", "04/20/2016"); + + // mock expectations + expect(queryFactory.createTaxonomyQuery(capture(checkForAnyTaxonomiesCapture))).andReturn(query); + expect(query.execute()).andReturn(queryResult); + expect(queryFactory.createTaxonomyQuery(capture(requestCapture))).andReturn(query); + expect(query.execute()).andReturn(queryResult); + replay(typeSystem, queryFactory, query); + + TestTaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); + provider.setInitialized(false); provider.setQueryFactory(queryFactory); Map<String, Object> requestProperties = new HashMap<>(); @@ -93,7 +196,7 @@ public class TaxonomyResourceProviderTest { expect(query.execute()).andReturn(emptyResponse); replay(typeSystem, queryFactory, query); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); Map<String, Object> requestProperties = new HashMap<>(); @@ -130,7 +233,7 @@ public class TaxonomyResourceProviderTest { expect(query.execute()).andReturn(queryResult); replay(typeSystem, queryFactory, query); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); Request userRequest = new CollectionRequest(Collections.<String, Object>emptyMap(), "name:taxonomy*"); @@ -163,7 +266,7 @@ public class TaxonomyResourceProviderTest { expect(query.execute()).andReturn(queryResult); replay(typeSystem, queryFactory, query); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); Request userRequest = new CollectionRequest(Collections.<String, Object>emptyMap(), "name:taxonomy*"); @@ -193,7 +296,7 @@ public class TaxonomyResourceProviderTest { requestProperties.put("description", "test"); Request userRequest = new InstanceRequest(requestProperties); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); provider.createResource(userRequest); @@ -225,7 +328,7 @@ public class TaxonomyResourceProviderTest { requestProperties.put("name", "taxonomyName"); Request userRequest = new InstanceRequest(requestProperties); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); provider.createResource(userRequest); @@ -252,7 +355,7 @@ public class TaxonomyResourceProviderTest { requestProperties.put("name", "taxonomyName"); Request userRequest = new InstanceRequest(requestProperties); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); provider.createResource(userRequest); @@ -279,7 +382,7 @@ public class TaxonomyResourceProviderTest { requestProperties.put("name", "taxonomyName"); Request userRequest = new InstanceRequest(requestProperties); - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); provider.createResources(userRequest); @@ -385,7 +488,7 @@ public class TaxonomyResourceProviderTest { replay(typeSystem, queryFactory, query); // instantiate resource provider and invoke method being tested - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); provider.updateResourceById(userRequest); @@ -424,7 +527,7 @@ public class TaxonomyResourceProviderTest { replay(typeSystem, queryFactory, query); // instantiate resource provider and invoke method being tested - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); provider.updateResourceById(userRequest); @@ -450,7 +553,7 @@ public class TaxonomyResourceProviderTest { replay(typeSystem, queryFactory, query); // instantiate resource provider and invoke method being tested - TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem); + TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem); provider.setQueryFactory(queryFactory); provider.updateResourceById(userRequest); @@ -459,14 +562,30 @@ public class TaxonomyResourceProviderTest { private static class TestTaxonomyResourceProvider extends TaxonomyResourceProvider { private final TermResourceProvider termResourceProvider; + private boolean isInitialized = true; + + public TestTaxonomyResourceProvider(AtlasTypeSystem typeSystem) { + super(typeSystem); + this.termResourceProvider = null; + } + public TestTaxonomyResourceProvider(AtlasTypeSystem typeSystem, TermResourceProvider termResourceProvider) { super(typeSystem); this.termResourceProvider = termResourceProvider; } + public void setInitialized(boolean isInitialized) { + this.isInitialized = isInitialized; + } + @Override protected synchronized TermResourceProvider getTermResourceProvider() { return termResourceProvider; } + + @Override + protected boolean autoInitializationChecked() { + return isInitialized; + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ebb5ec/distro/src/conf/atlas-application.properties ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index 8135eac..79a4982 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -175,4 +175,7 @@ atlas.authorizer.impl=SIMPLE ######### Performance Configs ######### #atlas.graph.storage.lock.retries=10 -#atlas.graph.storage.cache.db-cache-time=120000 \ No newline at end of file +#atlas.graph.storage.cache.db-cache-time=120000 + +######### Business Catalog ######### +atlas.taxonomy.default.name=Catalog \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ebb5ec/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index bc8370d..8ccbac9 100644 --- a/release-log.txt +++ b/release-log.txt @@ -31,6 +31,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-806 Create default taxonomy at server startup (jspeidel via yhemanth) ATLAS-942 Jenkins build failure - GraphRepoMapperScaleTest (shwethags) ATLAS-920 Lineage graph is broken when there are multiple paths from same source table (kevalbhatt18 via sumasai) ATLAS-940 Type cache implementation property name in atlas-application.properties is incorrect ( dkantor via sumasai)
