ATLAS-746 : After updating a set of entities, response contains only the first entity definition
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/5f508c97 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/5f508c97 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/5f508c97 Branch: refs/heads/master Commit: 5f508c97c7ea9436e16dbacb6770f14d0114eca3 Parents: acfe9a4 Author: Jeff Hagelberg <[email protected]> Authored: Tue Feb 7 15:31:02 2017 -0500 Committer: Jeff Hagelberg <[email protected]> Committed: Tue Feb 7 15:31:02 2017 -0500 ---------------------------------------------------------------------- .../apache/atlas/catalog/DefaultTypeSystem.java | 3 +- .../java/org/apache/atlas/AtlasBaseClient.java | 65 ++++-- .../main/java/org/apache/atlas/AtlasClient.java | 3 +- .../org/apache/atlas/AtlasEntitiesClientV2.java | 67 ++++-- .../atlas/CreateUpdateEntitiesResult.java | 122 ++++++++++ .../model/instance/EntityMutationResponse.java | 37 +-- .../atlas/model/instance/GuidMapping.java | 84 +++++++ release-log.txt | 1 + .../atlas/repository/MetadataRepository.java | 9 +- .../graph/GraphBackedMetadataRepository.java | 28 ++- .../graph/TypedInstanceToGraphMapper.java | 15 +- .../store/graph/v1/AtlasStructDefStoreV1.java | 8 +- .../graph/v1/AtlasTypeDefGraphStoreV1.java | 7 +- .../atlas/services/DefaultMetadataService.java | 51 ++-- .../org/apache/atlas/BaseRepositoryTest.java | 2 +- .../test/java/org/apache/atlas/TestUtils.java | 222 ++++++++++++++---- .../GraphBackedDiscoveryServiceTest.java | 71 ++++-- ...hBackedMetadataRepositoryDeleteTestBase.java | 24 +- .../GraphBackedMetadataRepositoryTest.java | 232 ++++++++++++++++--- .../atlas/repository/graph/GraphHelperTest.java | 6 +- .../graph/GraphRepoMapperScaleTest.java | 12 +- .../typestore/GraphBackedTypeStoreTest.java | 3 +- .../service/DefaultMetadataServiceTest.java | 4 +- .../apache/atlas/services/MetadataService.java | 27 +-- .../atlas/typesystem/types/TypeSystem.java | 2 +- .../org/apache/atlas/examples/QuickStartV2.java | 5 +- .../java/org/apache/atlas/util/RestUtils.java | 32 +-- .../web/adapters/AtlasInstanceRestAdapters.java | 23 +- .../atlas/web/resources/EntityResource.java | 58 +++-- .../org/apache/atlas/web/rest/EntitiesREST.java | 3 +- .../org/apache/atlas/web/rest/EntityREST.java | 3 +- .../apache/atlas/examples/QuickStartV2IT.java | 11 +- .../org/apache/atlas/util/RestUtilsTest.java | 212 +++++++++++++++++ .../atlas/web/resources/BaseResourceIT.java | 116 ++++++++-- .../web/resources/EntityJerseyResourceIT.java | 101 ++++++-- .../web/resources/EntityV2JerseyResourceIT.java | 115 +++++++-- 36 files changed, 1448 insertions(+), 336 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/catalog/src/main/java/org/apache/atlas/catalog/DefaultTypeSystem.java ---------------------------------------------------------------------- diff --git a/catalog/src/main/java/org/apache/atlas/catalog/DefaultTypeSystem.java b/catalog/src/main/java/org/apache/atlas/catalog/DefaultTypeSystem.java index 726351a..ea967f9 100644 --- a/catalog/src/main/java/org/apache/atlas/catalog/DefaultTypeSystem.java +++ b/catalog/src/main/java/org/apache/atlas/catalog/DefaultTypeSystem.java @@ -105,7 +105,8 @@ public class DefaultTypeSystem implements AtlasTypeSystem { entity.set(TaxonomyResourceProvider.NAMESPACE_ATTRIBUTE_NAME, TaxonomyResourceProvider.TAXONOMY_NS); ITypedReferenceableInstance typedInstance = metadataService.getTypedReferenceableInstance(entity); - final List<String> entities = metadataService.createEntities(Collections.singletonList(typedInstance).toArray(new ITypedReferenceableInstance[1])); + ITypedReferenceableInstance[] entitiesToCreate = Collections.singletonList(typedInstance).toArray(new ITypedReferenceableInstance[1]); + final List<String> entities = metadataService.createEntities(entitiesToCreate).getCreatedEntities(); return entities != null && entities.size() > 0 ? entities.get(0) : null; } catch (EntityExistsException e) { throw new ResourceAlreadyExistsException( http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/client/src/main/java/org/apache/atlas/AtlasBaseClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasBaseClient.java b/client/src/main/java/org/apache/atlas/AtlasBaseClient.java index 9b69991..cec5a11 100644 --- a/client/src/main/java/org/apache/atlas/AtlasBaseClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasBaseClient.java @@ -17,15 +17,19 @@ */ package org.apache.atlas; -import com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter; -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; +import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; + import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.security.SecureClientUtils; import org.apache.atlas.type.AtlasType; @@ -38,17 +42,16 @@ import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriBuilder; -import java.io.IOException; -import java.net.ConnectException; -import java.util.List; -import java.util.Map; - -import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED; +import com.google.common.annotations.VisibleForTesting; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter; +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; public abstract class AtlasBaseClient { public static final String BASE_URI = "api/atlas/"; @@ -277,6 +280,14 @@ public abstract class AtlasBaseClient { } protected <T> T callAPIWithResource(APIInfo api, WebResource resource, Object requestObject, Class<T> responseType) throws AtlasServiceException { + GenericType<T> genericType = null; + if(responseType != null) { + genericType = new GenericType<>(responseType); + } + return callAPIWithResource(api, resource, requestObject, genericType); + } + + protected <T> T callAPIWithResource(APIInfo api, WebResource resource, Object requestObject, GenericType<T> responseType) throws AtlasServiceException { ClientResponse clientResponse = null; int i = 0; do { @@ -297,7 +308,7 @@ public abstract class AtlasBaseClient { return null; } try { - if (responseType == JSONObject.class) { + if (responseType.getRawClass() == JSONObject.class) { String stringEntity = clientResponse.getEntity(String.class); try { JSONObject jsonObject = new JSONObject(stringEntity); @@ -419,6 +430,12 @@ public abstract class AtlasBaseClient { return callAPIWithResource(api, getResource(api, params), requestObject, responseType); } + public <T> T callAPI(APIInfo api, Object requestObject, GenericType<T> responseType, String... params) + throws AtlasServiceException { + return callAPIWithResource(api, getResource(api, params), requestObject, responseType); + } + + public <T> T callAPI(APIInfo api, Object requestBody, Class<T> responseType, MultivaluedMap<String, String> queryParams, String... params) throws AtlasServiceException { WebResource resource = getResource(api, queryParams, params); @@ -431,6 +448,12 @@ public abstract class AtlasBaseClient { return callAPIWithResource(api, resource, null, responseType); } + public <T> T callAPI(APIInfo api, GenericType<T> responseType, MultivaluedMap<String, String> queryParams, String... params) + throws AtlasServiceException { + WebResource resource = getResource(api, queryParams, params); + return callAPIWithResource(api, resource, null, responseType); + } + protected WebResource getResource(APIInfo api, String... pathParams) { return getResource(service, api, pathParams); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/client/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java index 154644d..1955f2a 100755 --- a/client/src/main/java/org/apache/atlas/AtlasClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasClient.java @@ -61,6 +61,7 @@ public class AtlasClient extends AtlasBaseClient { public static final String TYPENAME = "typeName"; public static final String GUID = "GUID"; public static final String ENTITIES = "entities"; + public static final String GUID_ASSIGNMENTS = "guidAssignments"; public static final String DEFINITION = "definition"; public static final String ERROR = "error"; @@ -616,7 +617,7 @@ public class AtlasClient extends AtlasBaseClient { /** * Delete the specified entities from the repository - * + * * @param guids guids of entities to delete * @return List of entity ids updated/deleted * @throws AtlasServiceException http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java b/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java index 2b3669d..2d6e708 100644 --- a/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java +++ b/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java @@ -17,29 +17,32 @@ */ package org.apache.atlas; -import com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.core.util.MultivaluedMapImpl; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; + import org.apache.atlas.model.SearchFilter; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntities; import org.apache.atlas.model.instance.AtlasEntityWithAssociations; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.commons.configuration.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.atlas.model.instance.AtlasEntity.AtlasEntities; +import com.google.common.annotations.VisibleForTesting; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.util.MultivaluedMapImpl; public class AtlasEntitiesClientV2 extends AtlasBaseClient { + private static final GenericType<List<AtlasEntityWithAssociations>> ENTITY_WITH_ASSOCIATIONS_LIST_TYPE = new GenericType<List<AtlasEntityWithAssociations>>(){}; public static final String ENTITY_API = BASE_URI + "v2/entity/"; public static final String ENTITIES_API = BASE_URI + "v2/entities/"; @@ -84,22 +87,25 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient { super(service, configuration); } - public AtlasEntity getEntityByGuid(String guid) throws AtlasServiceException { - return callAPI(GET_ENTITY_BY_GUID, null, AtlasEntity.class, guid); + public List<AtlasEntityWithAssociations> getEntityByGuid(String guid) throws AtlasServiceException { + + return callAPI(GET_ENTITY_BY_GUID, null, ENTITY_WITH_ASSOCIATIONS_LIST_TYPE, guid); } public AtlasEntities getEntityByGuids(List<String> guids) throws AtlasServiceException { return callAPI(GET_ENTITY_BY_GUID, AtlasEntities.class, "guid", guids); } - public AtlasEntityWithAssociations getEntityWithAssociationByGuid(String guid) throws AtlasServiceException { - return callAPI(formatPathForPathParams(GET_ENTITY_WITH_ASSOCIATION_BY_GUID, guid), null, AtlasEntityWithAssociations.class); + public List<AtlasEntityWithAssociations> getEntityWithAssociationByGuid(String guid) throws AtlasServiceException { + + return callAPI(formatPathForPathParams(GET_ENTITY_WITH_ASSOCIATION_BY_GUID, guid), null, ENTITY_WITH_ASSOCIATIONS_LIST_TYPE); } - public AtlasEntity getEntityByAttribute(String type, String attribute, String value) throws AtlasServiceException { + public List<AtlasEntityWithAssociations> getEntityByAttribute(String type, String attribute, String value) throws AtlasServiceException { + MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl(); queryParams.add("value", value); - return callAPI(formatPathForPathParams(GET_ENTITY_BY_ATTRIBUTE, type, attribute), AtlasEntity.class, queryParams); + return callAPI(formatPathForPathParams(GET_ENTITY_BY_ATTRIBUTE, type, attribute), ENTITY_WITH_ASSOCIATIONS_LIST_TYPE, queryParams); } public EntityMutationResponse updateEntityByAttribute(String type, String attribute, String value, AtlasEntity entity) throws AtlasServiceException { @@ -115,11 +121,11 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient { } public EntityMutationResponse createEntity(final AtlasEntity atlasEntity) throws AtlasServiceException { - return callAPI(CREATE_ENTITY, new HashMap<String, AtlasEntity>() {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class); + return callAPI(CREATE_ENTITY, new HashMap<String, AtlasEntity>(1) {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class); } public EntityMutationResponse updateEntity(final AtlasEntity atlasEntity) throws AtlasServiceException { - return callAPI(UPDATE_ENTITY, new HashMap<String, AtlasEntity>() {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class); + return callAPI(UPDATE_ENTITY, new HashMap<String, AtlasEntity>(1) {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class); } public AtlasEntity deleteEntityByGuid(String guid) throws AtlasServiceException { @@ -135,7 +141,7 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient { } public void addClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException { - callAPI(formatPathForPathParams(ADD_CLASSIFICATIONS, guid), classifications, null, (String[]) null); + callAPI(formatPathForPathParams(ADD_CLASSIFICATIONS, guid), classifications, (Class<?>)null, (String[]) null); } public void updateClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException { @@ -156,11 +162,24 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient { return null; } - public List<AtlasEntity> createEntities(Map<String, AtlasEntity> atlasEntities) throws AtlasServiceException { - return (List<AtlasEntity>)callAPI(CREATE_ENTITIES, atlasEntities, List.class); + public EntityMutationResponse createEntities(List<AtlasEntity> atlasEntities) throws AtlasServiceException { + + return callAPI(CREATE_ENTITIES, entityListToMap(atlasEntities), EntityMutationResponse.class); + } + + private Map<String, AtlasEntity> entityListToMap(List<AtlasEntity> atlasEntities) { + Map<String,AtlasEntity> toSend = new HashMap<String, AtlasEntity>(atlasEntities.size()); + for(AtlasEntity entity : atlasEntities) { + toSend.put(entity.getGuid(), entity); + } + return toSend; + } + + public EntityMutationResponse updateEntities(List<AtlasEntity> atlasEntities) throws AtlasServiceException { + return callAPI(UPDATE_ENTITIES, entityListToMap(atlasEntities), EntityMutationResponse.class); } - public List<AtlasEntity> updateEntities(Map<String, AtlasEntity> atlasEntities) throws AtlasServiceException { - return (List<AtlasEntity>)callAPI(UPDATE_ENTITIES, atlasEntities, List.class); + public AtlasEntity.AtlasEntities searchEntities(SearchFilter searchFilter) throws AtlasServiceException { + return callAPI(GET_ENTITIES, AtlasEntity.AtlasEntities.class, searchFilter.getParams()); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/client/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java b/client/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java new file mode 100644 index 0000000..a1aef61 --- /dev/null +++ b/client/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas; + +import java.util.Collections; +import java.util.List; + +import org.apache.atlas.AtlasClient.EntityResult; +import org.apache.atlas.model.instance.GuidMapping; + +/** + * Result from creating or updating entities. + */ +public class CreateUpdateEntitiesResult { + + /** + * Guid mapping for the entities that were created/updated + */ + private GuidMapping guidMapping; + + /** + * Entity result + */ + private EntityResult entityResult; + + /** + * Gets the guid mapping + */ + public GuidMapping getGuidMapping() { + return guidMapping; + } + + /** + * Sets the guid mapping + */ + public void setGuidMapping(GuidMapping guidMapping) { + this.guidMapping = guidMapping; + } + + /** + * Gets the entity result + */ + public EntityResult getEntityResult() { + return entityResult; + } + + /** + * Sets the entity result + */ + public void setEntityResult(EntityResult entityResult) { + this.entityResult = entityResult; + } + + /** + * Deserializes the given json into an instance of + * CreateUpdateEntitiesResult. + * + * @param json + * the (unmodified) json that comes back from Atlas. + * @return + * @throws AtlasServiceException + */ + public static CreateUpdateEntitiesResult fromJson(String json) throws AtlasServiceException { + + GuidMapping guidMapping = GuidMapping.fromString(json); + EntityResult entityResult = EntityResult.fromString(json); + CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); + result.setEntityResult(entityResult); + result.setGuidMapping(guidMapping); + return result; + } + + /** + * Convenience method to get the guids of the created entities from + * the EntityResult. + */ + public List<String> getCreatedEntities() { + if(entityResult == null) { + return Collections.emptyList(); + } + return getEntityResult().getCreatedEntities(); + } + + /** + * Convenience method to get the guids of the updated entities from + * the EntityResult. + */ + public List<String> getUpdatedEntities() { + if(entityResult == null) { + return Collections.emptyList(); + } + return getEntityResult().getUpdateEntities(); + } + + + /** + * Convenience method to get the guids of the deleted entities + * from the EntityResult. + */ + public List<String> getDeletedEntities() { + if (entityResult == null) { + return Collections.emptyList(); + } + return getEntityResult().getDeletedEntities(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java index 874a43a..2f2d44f 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java @@ -18,25 +18,24 @@ package org.apache.atlas.model.instance; -import org.apache.atlas.model.typedef.AtlasBaseTypeDef; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.codehaus.jackson.annotate.JsonAutoDetect; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; -import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; @JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @@ -46,6 +45,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL public class EntityMutationResponse { Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated; + Map<String,String> guidAssignments; public EntityMutationResponse() { } @@ -148,16 +148,25 @@ public class EntityMutationResponse { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; EntityMutationResponse that = (EntityMutationResponse) o; - return Objects.equals(entitiesMutated, that.entitiesMutated); + return Objects.equals(entitiesMutated, that.entitiesMutated) && + Objects.equals(guidAssignments, that.guidAssignments); } @Override public int hashCode() { - return Objects.hash(entitiesMutated); + return Objects.hash(entitiesMutated, guidAssignments); } @Override public String toString() { return toString(new StringBuilder()).toString(); } + + public void setGuidAssignments(Map<String,String> guidAssignments) { + this.guidAssignments = guidAssignments; + } + + public Map<String,String> getGuidAssignments() { + return guidAssignments; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/intg/src/main/java/org/apache/atlas/model/instance/GuidMapping.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/GuidMapping.java b/intg/src/main/java/org/apache/atlas/model/instance/GuidMapping.java new file mode 100644 index 0000000..43ce3b0 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/GuidMapping.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * This stores a mapping of guid assignments that were made during the processing + * of a create or update entity request. + *. + */ +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class GuidMapping { + + @JsonIgnore + private static final Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + private Map<String,String> guidAssignments; + + + public GuidMapping() { + + } + + public GuidMapping(Map<String,String> guidAssignments) { + this.guidAssignments = guidAssignments; + } + + + public Map<String,String> getGuidAssignments() { + return guidAssignments; + } + + public void setGuidAssignments(Map<String,String> guidAssignments) { + this.guidAssignments = guidAssignments; + } + /** + * Converts the GuidMapping to json + */ + @Override + public String toString() { + return gson.toJson(this); + } + + @JsonIgnore + public static GuidMapping fromString(String json) { + return gson.fromJson(json, GuidMapping.class); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index e6272fb..5a12669 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-746 After updating a set of entities, response contains only the first entity definition (jnhagelb) ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb) ATLAS-1388 Cache entities that are created/updated (jnhagelb) ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java index 1d61ea8..3f31f25 100755 --- a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java @@ -20,6 +20,7 @@ package org.apache.atlas.repository; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.exception.EntityExistsException; @@ -91,11 +92,11 @@ public interface MetadataRepository { * Creates an entity definition (instance) corresponding to a given type. * * @param entities entity (typed instance) - * @return a globally unique identifier + * @return CreateOrUpdateEntitiesResult with the guids of the entities that were created * @throws RepositoryException * @throws EntityExistsException */ - List<String> createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, EntityExistsException; + CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, EntityExistsException; /** * Fetch the complete definition of an entity given its GUID. @@ -166,13 +167,13 @@ public interface MetadataRepository { * Adds/Updates the property to the entity that corresponds to the GUID * Supports only primitive attribute/Class Id updations. */ - AtlasClient.EntityResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException; + CreateUpdateEntitiesResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException; /** * Adds the property to the entity that corresponds to the GUID * @param entitiesToBeUpdated The entities to be updated */ - AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException; + CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException; /** * Returns the entity for the given type and qualified name http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index 27bf6d7..943edef 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -27,8 +27,10 @@ import java.util.Map; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.GraphTransaction; import org.apache.atlas.RequestContext; +import org.apache.atlas.model.instance.GuidMapping; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.RepositoryException; @@ -143,7 +145,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Override @GraphTransaction - public List<String> createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, + public CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, EntityExistsException { if (LOG.isDebugEnabled()) { LOG.debug("adding entities={}", entities); @@ -152,7 +154,13 @@ public class GraphBackedMetadataRepository implements MetadataRepository { try { TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities); - return RequestContext.get().getCreatedEntityIds(); + List<String> createdGuids = RequestContext.get().getCreatedEntityIds(); + CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); + AtlasClient.EntityResult entityResult = new AtlasClient.EntityResult(createdGuids, null, null); + GuidMapping mapping = instanceToGraphMapper.createGuidMapping(); + result.setEntityResult(entityResult); + result.setGuidMapping(mapping); + return result; } catch (EntityExistsException e) { throw e; } catch (AtlasException e) { @@ -360,7 +368,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Override @GraphTransaction - public AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException { + public CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException { if (LOG.isDebugEnabled()) { LOG.debug("updating entity {}", entitiesUpdated); } @@ -369,8 +377,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL, entitiesUpdated); + CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); RequestContext requestContext = RequestContext.get(); - return createEntityResultFromContext(requestContext); + result.setEntityResult(createEntityResultFromContext(requestContext)); + GuidMapping mapping = instanceToGraphMapper.createGuidMapping(); + result.setGuidMapping(mapping); + return result; } catch (AtlasException e) { throw new RepositoryException(e); } @@ -378,7 +390,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Override @GraphTransaction - public AtlasClient.EntityResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException { + public CreateUpdateEntitiesResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException { if (LOG.isDebugEnabled()) { LOG.debug("updating entity {}", entity); } @@ -387,7 +399,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository { TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity); RequestContext requestContext = RequestContext.get(); - return createEntityResultFromContext(requestContext); + CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); + GuidMapping mapping = instanceToGraphMapper.createGuidMapping(); + result.setEntityResult(createEntityResultFromContext(requestContext)); + result.setGuidMapping(mapping); + return result; } catch (AtlasException e) { throw new RepositoryException(e); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index 1a88251..d9c7feb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -32,6 +32,7 @@ import java.util.Set; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; +import org.apache.atlas.model.instance.GuidMapping; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.graphdb.AtlasEdge; @@ -847,4 +848,16 @@ public final class TypedInstanceToGraphMapper { context.cache(instance); } } -} + + public GuidMapping createGuidMapping() { + Map<String,String> mapping = new HashMap<>(idToVertexMap.size()); + for(Map.Entry<Id, AtlasVertex> entry : idToVertexMap.entrySet()) { + Id id = entry.getKey(); + if (id.isUnassigned()) { + AtlasVertex classVertex = entry.getValue(); + mapping.put(id._getId(), GraphHelper.getGuid(classVertex)); + } + } + return new GuidMapping(mapping); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index d48c87e..69cf60a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -43,6 +43,8 @@ import org.codehaus.jettison.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -498,7 +500,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At } } - private static String toJsonFromAttribute(AtlasAttribute attribute) { + @VisibleForTesting + public static String toJsonFromAttribute(AtlasAttribute attribute) { AtlasAttributeDef attributeDef = attribute.getAttributeDef(); boolean isComposite = attribute.legacyIsComposite(); String reverseAttribName = attribute.legacyReverseAttribute(); @@ -539,7 +542,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At return AtlasType.toJson(attribInfo); } - private static AtlasAttributeDef toAttributeDefFromJson(AtlasStructDef structDef, + @VisibleForTesting + public static AtlasAttributeDef toAttributeDefFromJson(AtlasStructDef structDef, Map attribInfo, AtlasTypeDefGraphStoreV1 typeDefStore) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java index 88197ac..b76dfef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.store.graph.v1; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.inject.Inject; @@ -110,7 +111,8 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { AtlasGraph getAtlasGraph() { return atlasGraph; } - AtlasVertex findTypeVertexByName(String typeName) { + @VisibleForTesting + public AtlasVertex findTypeVertexByName(String typeName) { Iterator results = atlasGraph.query().has(VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE) .has(Constants.TYPENAME_PROPERTY_KEY, typeName) .vertices().iterator(); @@ -276,7 +278,8 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { return VERTEX_TYPE.equals(vertexType); } - boolean isTypeVertex(AtlasVertex vertex, TypeCategory category) { + @VisibleForTesting + public boolean isTypeVertex(AtlasVertex vertex, TypeCategory category) { boolean ret = false; if (isTypeVertex(vertex)) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index b14531f..c6cd93c 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -26,6 +26,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; @@ -293,7 +294,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang * @return guids - list of guids */ @Override - public List<String> createEntities(String entityInstanceDefinition) throws AtlasException { + public CreateUpdateEntitiesResult createEntities(String entityInstanceDefinition) throws AtlasException { entityInstanceDefinition = ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition"); ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition); @@ -301,10 +302,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang return createEntities(typedInstances); } - public List<String> createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException { - final List<String> guids = repository.createEntities(typedInstances); - onEntitiesAdded(guids); - return guids; + public CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException { + final CreateUpdateEntitiesResult result = repository.createEntities(typedInstances); + onEntitiesAdded(result.getCreatedEntities()); + return result; } private ITypedReferenceableInstance[] deserializeClassInstances(String entityInstanceDefinition) throws AtlasException { @@ -397,13 +398,13 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang * @return guids - json array of guids */ @Override - public AtlasClient.EntityResult updateEntities(String entityInstanceDefinition) throws AtlasException { + public CreateUpdateEntitiesResult updateEntities(String entityInstanceDefinition) throws AtlasException { entityInstanceDefinition = ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition"); ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition); - AtlasClient.EntityResult entityResult = repository.updateEntities(typedInstances); - onEntitiesAddedUpdated(entityResult); - return entityResult; + CreateUpdateEntitiesResult result = repository.updateEntities(typedInstances); + onEntitiesAddedUpdated(result.getEntityResult()); + return result; } /** @@ -413,10 +414,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang * @return guids - json array of guids */ @Override - public AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance[] entityInstanceDefinitions) throws AtlasException { - AtlasClient.EntityResult entityResult = repository.updateEntities(entityInstanceDefinitions); - onEntitiesAddedUpdated(entityResult); - return entityResult; + public CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance[] entityInstanceDefinitions) throws AtlasException { + CreateUpdateEntitiesResult result = repository.updateEntities(entityInstanceDefinitions); + onEntitiesAddedUpdated(result.getEntityResult()); + return result; } private void onEntitiesAddedUpdated(AtlasClient.EntityResult entityResult) throws AtlasException { @@ -427,7 +428,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } @Override - public AtlasClient.EntityResult updateEntityAttributeByGuid(String guid, String attributeName, + public CreateUpdateEntitiesResult updateEntityAttributeByGuid(String guid, String attributeName, String value) throws AtlasException { guid = ParamChecker.notEmpty(guid, "entity id"); attributeName = ParamChecker.notEmpty(attributeName, "attribute name"); @@ -457,9 +458,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName())); - AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance); - onEntitiesAddedUpdated(entityResult); - return entityResult; + CreateUpdateEntitiesResult result = repository.updatePartial(newInstance); + onEntitiesAddedUpdated(result.getEntityResult()); + return result; } private ITypedReferenceableInstance validateEntityExists(String guid) @@ -472,7 +473,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } @Override - public AtlasClient.EntityResult updateEntityPartialByGuid(String guid, Referenceable newEntity) + public CreateUpdateEntitiesResult updateEntityPartialByGuid(String guid, Referenceable newEntity) throws AtlasException { guid = ParamChecker.notEmpty(guid, "guid cannot be null"); newEntity = ParamChecker.notNull(newEntity, "updatedEntity cannot be null"); @@ -481,9 +482,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang ITypedReferenceableInstance newInstance = convertToTypedInstance(newEntity, existInstance.getTypeName()); ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName())); - AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance); - onEntitiesAddedUpdated(entityResult); - return entityResult; + CreateUpdateEntitiesResult result = repository.updatePartial(newInstance); + onEntitiesAddedUpdated(result.getEntityResult()); + return result; } private ITypedReferenceableInstance convertToTypedInstance(Referenceable updatedEntity, String typeName) @@ -530,7 +531,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } @Override - public AtlasClient.EntityResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, + public CreateUpdateEntitiesResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue, Referenceable updatedEntity) throws AtlasException { typeName = ParamChecker.notEmpty(typeName, "typeName"); @@ -543,9 +544,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang final ITypedReferenceableInstance newInstance = convertToTypedInstance(updatedEntity, typeName); ((ReferenceableInstance)newInstance).replaceWithNewId(oldInstance.getId()); - AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance); - onEntitiesAddedUpdated(entityResult); - return entityResult; + CreateUpdateEntitiesResult result = repository.updatePartial(newInstance); + onEntitiesAddedUpdated(result.getEntityResult()); + return result; } private void validateTypeExists(String entityType) throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java index 9714a8b..6656dc6 100644 --- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java @@ -395,7 +395,7 @@ public class BaseRepositoryTest { } private Id createInstance(Referenceable referenceable, ClassType clsType) throws Exception { ITypedReferenceableInstance typedInstance = clsType.convert(referenceable, Multiplicity.REQUIRED); - List<String> guids = repository.createEntities(typedInstance); + List<String> guids = repository.createEntities(typedInstance).getCreatedEntities(); // return the reference to created instance with guid return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index 1d1a5e0..e5abd77 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -24,6 +24,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAt import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef; +import static org.testng.Assert.assertEquals; import java.io.File; import java.io.FileOutputStream; @@ -38,7 +39,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.TypesChangeListener; @@ -53,14 +58,17 @@ import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.MetadataService; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.typesystem.IInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.atlas.typesystem.types.EnumTypeDefinition; import org.apache.atlas.typesystem.types.EnumValue; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; @@ -89,6 +97,32 @@ public final class TestUtils { public static final long TEST_DATE_IN_LONG = 1418265358440L; + + public static final String EMPLOYEES_ATTR = "employees"; + public static final String DEPARTMENT_ATTR = "department"; + public static final String ASSETS_ATTR = "assets"; + + public static final String POSITIONS_ATTR = "positions"; + public static final String ASSET_TYPE = "TestAsset"; + + public static final String DATABASE_TYPE = "hive_database"; + public static final String DATABASE_NAME = "foo"; + public static final String TABLE_TYPE = "hive_table"; + public static final String PROCESS_TYPE = "hive_process"; + public static final String COLUMN_TYPE = "column_type"; + public static final String TABLE_NAME = "bar"; + public static final String CLASSIFICATION = "classification"; + public static final String PII = "PII"; + public static final String SUPER_TYPE_NAME = "Base"; + public static final String STORAGE_DESC_TYPE = "hive_storagedesc"; + public static final String PARTITION_STRUCT_TYPE = "partition_struct_type"; + public static final String PARTITION_CLASS_TYPE = "partition_class_type"; + public static final String SERDE_TYPE = "serdeType"; + public static final String COLUMNS_MAP = "columnsMap"; + public static final String COLUMNS_ATTR_NAME = "columns"; + + public static final String NAME = "name"; + private TestUtils() { } @@ -141,17 +175,21 @@ public final class TestUtils { createRequiredAttrDef("city", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(), - createRequiredAttrDef("name", DataTypes.STRING_TYPE), - new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL, - true, "department")); + createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), + new AttributeDefinition(EMPLOYEES_ATTR, String.format("array<%s>", "Person"), Multiplicity.OPTIONAL, + true, DEPARTMENT_ATTR), + new AttributeDefinition(POSITIONS_ATTR, String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL, + false, null) + ); HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(), - createRequiredAttrDef("name", DataTypes.STRING_TYPE), + createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("address", "Address"), - new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"), + new AttributeDefinition(DEPARTMENT_ATTR, "Department", Multiplicity.REQUIRED, false, EMPLOYEES_ATTR), new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"), new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null), + new AttributeDefinition(ASSETS_ATTR, String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null), createOptionalAttrDef("birthday", DataTypes.DATE_TYPE), createOptionalAttrDef("hasPets", DataTypes.BOOLEAN_TYPE), createOptionalAttrDef("numberOfCars", DataTypes.BYTE_TYPE), @@ -165,6 +203,12 @@ public final class TestUtils { createOptionalAttrDef("isOrganDonor", DataTypes.BOOLEAN_TYPE) ); + + HierarchicalTypeDefinition<ClassType> assetTypeDef = createClassTypeDef(ASSET_TYPE, "Asset"+_description, ImmutableSet.<String>of(), + createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), + new AttributeDefinition("childAssets", String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null) + ); + HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Person"), new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")); @@ -175,7 +219,7 @@ public final class TestUtils { ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), ImmutableList.of(securityClearanceTypeDef), - ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef)); + ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef, assetTypeDef)); } public static final String DEPARTMENT_TYPE = "Department"; @@ -193,9 +237,9 @@ public final class TestUtils { Referenceable max = new Referenceable("Person"); Referenceable maxAddr = new Referenceable("Address"); - hrDept.set("name", "hr"); - john.set("name", "John"); - john.set("department", hrDept); + hrDept.set(NAME, "hr"); + john.set(NAME, "John"); + john.set(DEPARTMENT_ATTR, hrDept); johnAddr.set("street", "Stewart Drive"); johnAddr.set("city", "Sunnyvale"); john.set("address", johnAddr); @@ -212,22 +256,22 @@ public final class TestUtils { john.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000")); john.set("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); - jane.set("name", "Jane"); - jane.set("department", hrDept); + jane.set(NAME, "Jane"); + jane.set(DEPARTMENT_ATTR, hrDept); janeAddr.set("street", "Great America Parkway"); janeAddr.set("city", "Santa Clara"); jane.set("address", janeAddr); janeAddr.set("street", "Great America Parkway"); - julius.set("name", "Julius"); - julius.set("department", hrDept); + julius.set(NAME, "Julius"); + julius.set(DEPARTMENT_ATTR, hrDept); juliusAddr.set("street", "Madison Ave"); juliusAddr.set("city", "Newtonville"); julius.set("address", juliusAddr); julius.set("subordinates", ImmutableList.<Referenceable>of()); - max.set("name", "Max"); - max.set("department", hrDept); + max.set(NAME, "Max"); + max.set(DEPARTMENT_ATTR, hrDept); maxAddr.set("street", "Ripley St"); maxAddr.set("city", "Newton"); max.set("address", maxAddr); @@ -247,7 +291,7 @@ public final class TestUtils { john.set("manager", jane); john.set("mentor", max); - hrDept.set("employees", ImmutableList.of(john, jane, julius, max)); + hrDept.set(EMPLOYEES_ATTR, ImmutableList.of(john, jane, julius, max)); jane.set("subordinates", ImmutableList.of(john, max)); @@ -260,23 +304,7 @@ public final class TestUtils { return hrDept2; } - public static final String DATABASE_TYPE = "hive_database"; - public static final String DATABASE_NAME = "foo"; - public static final String TABLE_TYPE = "hive_table"; - public static final String PROCESS_TYPE = "hive_process"; - public static final String COLUMN_TYPE = "column_type"; - public static final String TABLE_NAME = "bar"; - public static final String CLASSIFICATION = "classification"; - public static final String PII = "PII"; - public static final String SUPER_TYPE_NAME = "Base"; - public static final String STORAGE_DESC_TYPE = "hive_storagedesc"; - public static final String PARTITION_STRUCT_TYPE = "partition_struct_type"; - public static final String PARTITION_CLASS_TYPE = "partition_class_type"; - public static final String SERDE_TYPE = "serdeType"; - public static final String COLUMNS_MAP = "columnsMap"; - public static final String COLUMNS_ATTR_NAME = "columns"; - public static final String NAME = "name"; public static TypesDef simpleType(){ HierarchicalTypeDefinition<ClassType> superTypeDefinition = @@ -284,7 +312,7 @@ public final class TestUtils { createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", - new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)}); + new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)}); HierarchicalTypeDefinition<TraitType> traitTypeDefinition = createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); @@ -306,7 +334,7 @@ public final class TestUtils { createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", - new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)}); + new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)}); HierarchicalTypeDefinition<TraitType> traitTypeDefinition = createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); @@ -344,7 +372,7 @@ public final class TestUtils { StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description, - new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE), + new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createRequiredAttrDef("serde", DataTypes.STRING_TYPE), createOptionalAttrDef("description", DataTypes.STRING_TYPE)}); @@ -354,11 +382,11 @@ public final class TestUtils { HierarchicalTypeDefinition<ClassType> columnsDefinition = createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(), - createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE)); StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description, - new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),}); + new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),}); AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, @@ -403,7 +431,7 @@ public final class TestUtils { HierarchicalTypeDefinition<ClassType> tableTypeDefinition = createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), - TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createRequiredAttrDef("description", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE), createOptionalAttrDef("created", DataTypes.DATE_TYPE), @@ -498,19 +526,75 @@ public final class TestUtils { return entity; } + /** + * Creates an entity in the graph and does basic validation + * of the GuidMapping that was created in the process. + * + */ public static String createInstance(MetadataService metadataService, Referenceable entity) throws Exception { RequestContext.createContext(); String entityjson = InstanceSerialization.toJson(entity, true); JSONArray entitiesJson = new JSONArray(); entitiesJson.put(entityjson); - List<String> guids = metadataService.createEntities(entitiesJson.toString()); + CreateUpdateEntitiesResult creationResult = metadataService.createEntities(entitiesJson.toString()); + Map<String,String> guidMap = creationResult.getGuidMapping().getGuidAssignments(); + Map<Id, Referenceable> referencedObjects = findReferencedObjects(entity); + + for(Map.Entry<Id,Referenceable> entry : referencedObjects.entrySet()) { + Id foundId = entry.getKey(); + if(foundId.isUnassigned()) { + String guid = guidMap.get(entry.getKey()._getId()); + Referenceable obj = entry.getValue(); + loadAndDoSimpleValidation(guid,obj, metadataService); + } + } + List<String> guids = creationResult.getCreatedEntities(); if (guids != null && guids.size() > 0) { return guids.get(guids.size() - 1); } return null; } + private static Map<Id,Referenceable> findReferencedObjects(Referenceable ref) { + Map<Id, Referenceable> result = new HashMap<>(); + findReferencedObjects(ref, result); + return result; + } + + private static void findReferencedObjects(Referenceable ref, Map<Id, Referenceable> seen) { + + Id guid = ref.getId(); + if(seen.containsKey(guid)) { + return; + } + seen.put(guid, ref); + for(Map.Entry<String, Object> attr : ref.getValuesMap().entrySet()) { + Object value = attr.getValue(); + if(value instanceof Referenceable) { + findReferencedObjects((Referenceable)value, seen); + } + else if(value instanceof List) { + for(Object o : (List)value) { + if(o instanceof Referenceable) { + findReferencedObjects((Referenceable)o, seen); + } + } + } + else if(value instanceof Map) { + for(Object o : ((Map)value).values()) { + if(o instanceof Referenceable) { + findReferencedObjects((Referenceable)o, seen); + } + } + } + } + } + + /** + * Clears the state in the request context. + * + */ public static void resetRequestContext() { //reset the context while preserving the user String user = RequestContext.get().getUser(); @@ -518,6 +602,10 @@ public final class TestUtils { RequestContext.get().setUser(user); } + /** + * Triggers the Atlas initialization process using the specified MetadataRepository. + * This causes the built-in types and their indices to be created. + */ public static void setupGraphProvider(MetadataRepository repo) throws AtlasException { TypeCache typeCache = null; try { @@ -634,4 +722,60 @@ public final class TestUtils { }); } + + /** + * Loads the entity and does sanity testing of the GuidMapping that was + * created during the operation. + * + */ + public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataRepository repositoryService) throws AtlasException { + ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid); + doSimpleValidation(original, loaded); + return loaded; + } + + /** + * Loads the entity and does sanity testing of the GuidMapping that was + * created during the operation. + * + */ + public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataService repositoryService) throws AtlasException { + ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid); + doSimpleValidation(original, loaded); + return loaded; + + } + + private static void doSimpleValidation(Referenceable original, IInstance loaded) throws AtlasException { + + assertEquals(loaded.getTypeName(), original.getTypeName()); + ClassType ct = TypeSystem.getInstance().getDataType(ClassType.class, loaded.getTypeName()); + + //compare primitive fields + for(AttributeInfo field : ct.fieldMapping.fields.values()) { + if(field.dataType().getTypeCategory() == TypeCategory.PRIMITIVE) { + if(original.get(field.name) != null) { + Object rawLoadedValue = loaded.get(field.name); + Object rawProvidedValue = original.get(field.name); + Object convertedLoadedValue = field.dataType().convert(rawLoadedValue, Multiplicity.REQUIRED); + Object convertedProvidedValue = field.dataType().convert(rawProvidedValue, Multiplicity.REQUIRED); + + assertEquals(convertedLoadedValue, convertedProvidedValue); + } + } + } + } + + /** + * Validates that the two String Collections contain the same items, without + * regard to order. + * + */ + public static void assertContentsSame(Collection<String> actual, Collection<String> expected) { + assertEquals(actual.size(), expected.size()); + Set<String> checker = new HashSet<>(); + checker.addAll(expected); + checker.removeAll(actual); + assertEquals(checker.size(), 0); + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index ce87c9e..ffda984 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -121,7 +121,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { } } - + //We need to commit the transaction before creating the indices to release the locks held by the transaction. //otherwise, the index commit will fail while waiting for the those locks to be released. AtlasGraphProvider.getGraphInstance().commit(); @@ -302,7 +302,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { for (String name : Arrays.asList("John", "Max")) { Assert.assertTrue(names.contains(name)); } - + // Query for all Vertices modified after 01/01/2015 00:00:00 GMT r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY + " > 1420070400000}.toList()"); Assert.assertTrue(r instanceof List); @@ -524,55 +524,55 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db, hive_table limit 5", 5}, {"hive_db, hive_table limit 5 offset 0", 5}, {"hive_db, hive_table limit 5 offset 5", 5}, - + {"View is JdbcAccess", 2}, {"View is JdbcAccess limit 1", 1}, {"View is JdbcAccess limit 2 offset 1", 1}, {"hive_db as db1, hive_table where db1.name = \"Reporting\"", 0}, //Not working - ATLAS-145 - - + + {"from hive_table", 10}, {"from hive_table limit 5", 5}, {"from hive_table limit 5 offset 5", 5}, - + {"hive_table", 10}, {"hive_table limit 5", 5}, {"hive_table limit 5 offset 5", 5}, - + {"hive_table isa Dimension", 3}, {"hive_table isa Dimension limit 2", 2}, {"hive_table isa Dimension limit 2 offset 0", 2}, {"hive_table isa Dimension limit 2 offset 1", 2}, {"hive_table isa Dimension limit 3 offset 1", 2}, - + {"hive_column where hive_column isa PII", 8}, {"hive_column where hive_column isa PII limit 5", 5}, {"hive_column where hive_column isa PII limit 5 offset 1", 5}, {"hive_column where hive_column isa PII limit 5 offset 5", 3}, - - + + {"View is Dimension" , 2}, {"View is Dimension limit 1" , 1}, {"View is Dimension limit 1 offset 1" , 1}, {"View is Dimension limit 10 offset 1" , 1}, - + {"hive_column select hive_column.name", 37}, {"hive_column select hive_column.name limit 5", 5}, {"hive_column select hive_column.name limit 5 offset 36", 1}, - + {"hive_column select name", 37}, {"hive_column select name limit 5", 5}, {"hive_column select name limit 5 offset 36 ", 1}, - + {"hive_column where hive_column.name=\"customer_id\"", 6}, {"hive_column where hive_column.name=\"customer_id\" limit 2", 2}, {"hive_column where hive_column.name=\"customer_id\" limit 2 offset 1", 2}, {"hive_column where hive_column.name=\"customer_id\" limit 10 offset 3", 3}, - + {"from hive_table select hive_table.name", 10}, {"from hive_table select hive_table.name limit 5", 5}, {"from hive_table select hive_table.name limit 5 offset 5", 5}, - + {"hive_db where (name = \"Reporting\")", 1}, {"hive_db where (name = \"Reporting\") limit 10", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1}, @@ -585,18 +585,18 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db where hive_db has name limit 5", 3}, {"hive_db where hive_db has name limit 2 offset 0", 2}, {"hive_db where hive_db has name limit 2 offset 1", 2}, - + {"hive_db as db1 hive_table where (db1.name = \"Reporting\")", 0}, //Not working -> ATLAS-145 {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 limit 10", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 limit 10 offset 1", 0}, {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 limit 10 offset 0", 1}, - + {"hive_table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 ", 1}, {"hive_table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 limit 10 ", 1}, {"hive_table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 limit 10 offset 0", 1}, {"hive_table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 limit 10 offset 5", 0}, - + {"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1 ", 1}, {"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1 limit 10 offset 0", 1}, {"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1 limit 10 offset 1", 0}, @@ -608,25 +608,25 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"Dimension limit 2", 2}, {"Dimension limit 2 offset 1", 2}, {"Dimension limit 5 offset 4", 1}, - + {"JdbcAccess", 2}, {"JdbcAccess limit 5 offset 0", 2}, {"JdbcAccess limit 2 offset 1", 1}, {"JdbcAccess limit 1", 1}, - + {"ETL", 5}, {"ETL limit 2", 2}, {"ETL limit 1", 1}, {"ETL limit 1 offset 0", 1}, {"ETL limit 2 offset 1", 2}, - + {"Metric", 9}, {"Metric limit 10", 9}, {"Metric limit 2", 2}, {"Metric limit 10 offset 1", 8}, - - - + + + {"PII", 8}, {"PII limit 10", 8}, {"PII limit 2", 2}, @@ -933,7 +933,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows"); } - + @Test(dataProvider = "dslQueriesProvider") public void testSearchByDSLQueries(String dslQuery, Integer expectedNumRows) throws Exception { runQuery(dslQuery, expectedNumRows, 40, 0); @@ -1207,6 +1207,27 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { } + @Test + public void testSearchForTypeWithNoInstances() throws Exception { + + HierarchicalTypeDefinition EMPTY = createClassTypeDef("EmptyType", null, + createRequiredAttrDef("a", DataTypes.INT_TYPE)); + TypeSystem.getInstance().defineClassTypes(EMPTY); + + String dslQuery = "EmptyType"; + String jsonResults = searchByDSL(dslQuery); + assertNotNull(jsonResults); + JSONObject results = new JSONObject(jsonResults); + + assertEquals(results.length(), 3); + + JSONArray rows = results.getJSONArray("rows"); + assertNotNull(rows); + + // query should not return any rows + assertEquals(rows.length(), 0); + } + private FieldValueValidator makeCountValidator(int count) { return new FieldValueValidator().withFieldNames("count()").withExpectedValues(count); }
