Repository: incubator-atlas Updated Branches: refs/heads/master a67e89625 -> 1e105174c
ATLAS-1527 : Batch entity retrievals - DefaultMetadataService.loadEntities Signed-off-by: Jeff Hagelberg <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/1e105174 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/1e105174 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/1e105174 Branch: refs/heads/master Commit: 1e105174cfc0ba82ad254efaeb666ba880956dea Parents: a67e896 Author: Wojciech Wojcik <[email protected]> Authored: Thu Feb 9 09:22:53 2017 -0500 Committer: Jeff Hagelberg <[email protected]> Committed: Thu Feb 9 09:23:38 2017 -0500 ---------------------------------------------------------------------- release-log.txt | 1 + .../atlas/repository/MetadataRepository.java | 11 ++++ .../graph/GraphBackedMetadataRepository.java | 69 ++++++++++++++++++-- .../atlas/services/DefaultMetadataService.java | 9 +-- .../GraphBackedRepositoryHardDeleteTest.java | 2 +- 5 files changed, 77 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index c907e67..a444c7a 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1527 Batch entity retrievals - DefaultMetadataService.loadEntities ATLAS-1385 Add configuration property to disable full text mapper (wwojcik via jnhagelb) ATLAS-746 After updating a set of entities, response contains only the first entity definition (jnhagelb) ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java old mode 100755 new mode 100644 index 3f31f25..fab1859 --- a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java @@ -104,10 +104,21 @@ public interface MetadataRepository { * @param guid globally unique identifier for the entity * @return entity (typed instance) definition * @throws RepositoryException + * @throws EntityNotFoundException */ ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException; /** + * Fetch the complete entity definitions for the entities with the given GUIDs + * + * @param guids globally unique identifiers for the entities + * @return entity (typed instance) definitions list + * @throws RepositoryException + * @throws EntityNotFoundException + */ + List<ITypedReferenceableInstance> getEntityDefinitions(String... guids) throws RepositoryException, EntityNotFoundException; + + /** * Gets the list of entities for a given entity type. * * @param entityType name of a type which is unique http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index 943edef..4461447 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -19,11 +19,15 @@ package org.apache.atlas.repository.graph; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; @@ -169,19 +173,70 @@ public class GraphBackedMetadataRepository implements MetadataRepository { } @Override - @GraphTransaction public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException { + return getEntityDefinitions(guid).get(0); + } + + @Override + @GraphTransaction + public List<ITypedReferenceableInstance> getEntityDefinitions(String... guids) throws RepositoryException, EntityNotFoundException { if (LOG.isDebugEnabled()) { - LOG.debug("Retrieving entity with guid={}", guid); + LOG.debug("Retrieving entities with guids={}", Arrays.toString(guids)); } - AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid); + RequestContext context = RequestContext.get(); + ITypedReferenceableInstance[] result = new ITypedReferenceableInstance[guids.length]; + + // Map of the guids of instances not in the cache to their index(es) in the result. + // This is used to put the loaded instances into the location(s) corresponding + // to their guid in the result. Note that a set is needed since guids can + // appear more than once in the list. + Map<String, Set<Integer>> uncachedGuids = new HashMap<>(); + + for (int i = 0; i < guids.length; i++) { + String guid = guids[i]; + + // First, check the cache. + ITypedReferenceableInstance cached = context.getInstance(guid); + if (cached != null) { + result[i] = cached; + } else { + Set<Integer> indices = uncachedGuids.get(guid); + if (indices == null) { + indices = new HashSet<>(1); + uncachedGuids.put(guid, indices); + } + indices.add(i); + } + } - try { - return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); - } catch (AtlasException e) { - throw new RepositoryException(e); + List<String> guidsToFetch = new ArrayList<>(uncachedGuids.keySet()); + Map<String, AtlasVertex> instanceVertices = graphHelper.getVerticesForGUIDs(guidsToFetch); + + // search for missing entities + if (instanceVertices.size() != guidsToFetch.size()) { + Set<String> missingGuids = new HashSet<String>(guidsToFetch); + missingGuids.removeAll(instanceVertices.keySet()); + if (!missingGuids.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to find guids={}", missingGuids); + } + throw new EntityNotFoundException( + "Could not find entities in the repository with guids: " + missingGuids.toString()); + } + } + + for (String guid : guidsToFetch) { + try { + ITypedReferenceableInstance entity = graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertices.get(guid)); + for(int index : uncachedGuids.get(guid)) { + result[index] = entity; + } + } catch (AtlasException e) { + throw new RepositoryException(e); + } } + return Arrays.asList(result); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index c6cd93c..5127f74 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -705,13 +705,8 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } } - private List<ITypedReferenceableInstance> loadEntities(List<String> guids) throws EntityNotFoundException, - RepositoryException { - List<ITypedReferenceableInstance> entities = new ArrayList<>(); - for (String guid : guids) { - entities.add(repository.getEntityDefinition(guid)); - } - return entities; + private List<ITypedReferenceableInstance> loadEntities(List<String> guids) throws RepositoryException, EntityNotFoundException { + return repository.getEntityDefinitions(guids.toArray(new String[guids.size()])); } private void onTypesUpdated(Map<String, IDataType> typesUpdated) throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java index 43b08c4..174b8cb 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java @@ -80,7 +80,7 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo repositoryService.getEntityDefinition(id); fail("Expected EntityNotFoundException"); } catch(EntityNotFoundException e) { - //expected + // expected } }
