http://git-wip-us.apache.org/repos/asf/atlas/blob/8a57e657/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveBuilder.java ---------------------------------------------------------------------- diff --git a/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveBuilder.java b/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveBuilder.java new file mode 100644 index 0000000..ab2b2f6 --- /dev/null +++ b/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveBuilder.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.omrs.archivemanager; + +import org.apache.atlas.omrs.archivemanager.properties.*; +import org.apache.atlas.omrs.metadatacollection.properties.instances.EntityDetail; +import org.apache.atlas.omrs.metadatacollection.properties.instances.Relationship; +import org.apache.atlas.omrs.metadatacollection.properties.typedefs.*; + +import java.util.*; + +/** + * OMRSArchiveBuilder creates an in-memory copy of an open metadata archive that can be saved to disk or processed + * by a server. + */ +public class OMRSArchiveBuilder +{ + /* + * Archive properties supplied on the constructor + */ + private OpenMetadataArchiveProperties archiveProperties = null; + + /* + * Hash maps for accumulating TypeDefs and instances as the content of the archive is built up. + */ + private Map<String, PrimitiveDef> primitiveDefMap = new HashMap<>(); + private Map<String, CollectionDef> collectionDefMap = new HashMap<>(); + private Map<String, EnumDef> enumDefMap = new HashMap<>(); + private Map<String, ClassificationDef> classificationDefMap = new HashMap<>(); + private Map<String, EntityDef> entityDefMap = new HashMap<>(); + private Map<String, RelationshipDef> relationshipDefMap = new HashMap<>(); + private Map<String, TypeDefPatch> typeDefPatchMap = new HashMap<>(); + private Map<String, EntityDetail> entityDetailMap = new HashMap<>(); + private Map<String, Relationship> relationshipMap = new HashMap<>(); + + + /** + * Typical constructor passes parameters used to build the open metadata archive's property header. + * + * @param archiveGUID - unique identifier for this open metadata archive. + * @param archiveName - name of the open metadata archive. + * @param archiveDescription - description of the open metadata archive. + * @param archiveType - enum describing the type of archive this is. + * @param originatorName - name of the originator (person or organization) of the archive. + * @param creationDate - data that this archive was created. + * @param dependsOnArchives - list of GUIDs for archives that this archive depends on (null for no dependencies). + */ + public OMRSArchiveBuilder(String archiveGUID, + String archiveName, + String archiveDescription, + OpenMetadataArchiveType archiveType, + String originatorName, + Date creationDate, + ArrayList<String> dependsOnArchives) + { + this.archiveProperties = new OpenMetadataArchiveProperties(); + + this.archiveProperties.setArchiveGUID(archiveGUID); + this.archiveProperties.setArchiveName(archiveName); + this.archiveProperties.setArchiveDescription(archiveDescription); + this.archiveProperties.setArchiveType(archiveType); + this.archiveProperties.setOriginatorName(originatorName); + this.archiveProperties.setCreationDate(creationDate); + this.archiveProperties.setDependsOnArchives(dependsOnArchives); + } + + + /** + * Add a new PrimitiveDef to the archive. + * + * @param primitiveDef - type to add + */ + public void addPrimitiveDef(PrimitiveDef primitiveDef) + { + if (primitiveDef != null) + { + primitiveDefMap.put(primitiveDef.getName(), primitiveDef); + } + } + + + /** + * Add a new CollectionDef to the archive. + * + * @param collectionDef - type to add + */ + public void addCollectionDef(CollectionDef collectionDef) + { + if (collectionDef != null) + { + collectionDefMap.put(collectionDef.getName(), collectionDef); + } + } + + + /** + * Add a new EnumDef to the archive. + * + * @param enumDef - type to add + */ + public void addEnumDef(EnumDef enumDef) + { + if (enumDef != null) + { + enumDefMap.put(enumDef.getName(), enumDef); + } + } + + + /** + * Add a new ClassificationDef to the archive. + * + * @param classificationDef - type to add + */ + public void addClassificationDef(ClassificationDef classificationDef) + { + if (classificationDef != null) + { + classificationDefMap.put(classificationDef.getName(), classificationDef); + } + } + + + /** + * Add a new EntityDef to the archive. + * + * @param entityDef - type to add + */ + public void addEntityDef(EntityDef entityDef) + { + if (entityDef != null) + { + EntityDef previousDef = entityDefMap.put(entityDef.getName(), entityDef); + + if (previousDef != null) + { + // TODO log a duplicate + } + } + } + + + /** + * Retrieve the entityDef - or null if it is not defined. + * + * @param entityDefName - name of the entity + * @return the retrieved Entity def + */ + public EntityDef getEntityDef(String entityDefName) + { + if (entityDefName != null) + { + EntityDef retrievedEntityDef = entityDefMap.get(entityDefName); + + if (retrievedEntityDef != null) + { + return retrievedEntityDef; + } + else + { + // TODO Throw exception + return null; /* temporary */ + } + } + else + { + return null; + } + } + + + /** + * Add a new RelationshipDef to the archive. + * + * @param relationshipDef - type to add + */ + public void addRelationshipDef(RelationshipDef relationshipDef) + { + if (relationshipDef != null) + { + relationshipDefMap.put(relationshipDef.getName(), relationshipDef); + } + } + + + /** + * Add a new entity to the archive. + * + * @param entity - instance to add + */ + public void addEntity(EntityDetail entity) + { + if (entity != null) + { + entityDetailMap.put(entity.getGUID(), entity); + } + } + + + /** + * Add a new relationship to the archive. + * + * @param relationship - instance to add + */ + public void addRelationship(Relationship relationship) + { + if (relationship != null) + { + relationshipMap.put(relationship.getGUID(), relationship); + } + } + + + /** + * Once the content of the archive has been added to the archive builder, an archive object can be retrieved. + * + * @return open metadata archive object with all of the supplied content in it. + */ + public OpenMetadataArchive getOpenMetadataArchive() + { + OpenMetadataArchive archive = new OpenMetadataArchive(); + + /* + * Set up the archive properties + */ + archive.setArchiveProperties(this.archiveProperties); + + /* + * Set up the TypeStore. The types are added in a strict order to ensure that the dependencies are resolved. + */ + ArrayList<AttributeTypeDef> attributeTypeDefs = null; + ArrayList<TypeDef> typeDefs = new ArrayList<>(); + ArrayList<TypeDefPatch> typeDefPatches = new ArrayList<>(); + + if (! primitiveDefMap.isEmpty()) + { + attributeTypeDefs.addAll(primitiveDefMap.values()); + } + if (! collectionDefMap.isEmpty()) + { + attributeTypeDefs.addAll(collectionDefMap.values()); + } + if (! enumDefMap.isEmpty()) + { + attributeTypeDefs.addAll(enumDefMap.values()); + } + if (! entityDefMap.isEmpty()) + { + typeDefs.addAll(entityDefMap.values()); + } + if (! classificationDefMap.isEmpty()) + { + typeDefs.addAll(classificationDefMap.values()); + } + if (! relationshipDefMap.isEmpty()) + { + typeDefs.addAll(relationshipDefMap.values()); + } + + if (! typeDefPatchMap.isEmpty()) + { + typeDefPatches.addAll(typeDefPatchMap.values()); + } + + if ((! typeDefs.isEmpty()) || (! typeDefPatches.isEmpty())) + { + OpenMetadataArchiveTypeStore typeStore = new OpenMetadataArchiveTypeStore(); + + if (! attributeTypeDefs.isEmpty()) + { + typeStore.setAttributeTypeDefs(attributeTypeDefs); + } + + if (! typeDefs.isEmpty()) + { + typeStore.setNewTypeDefs(typeDefs); + } + + if (! typeDefPatches.isEmpty()) + { + typeStore.setTypeDefPatches(typeDefPatches); + } + + archive.setArchiveTypeStore(typeStore); + } + + + /* + * Finally set up the instance store + */ + ArrayList<EntityDetail> entities = new ArrayList<>(); + ArrayList<Relationship> relationships = new ArrayList<>(); + + if (! entityDetailMap.isEmpty()) + { + entities.addAll(entityDetailMap.values()); + } + if (! relationshipMap.isEmpty()) + { + relationships.addAll(relationshipMap.values()); + } + + if ((! entities.isEmpty()) || (! relationships.isEmpty())) + { + OpenMetadataArchiveInstanceStore instanceStore = new OpenMetadataArchiveInstanceStore(); + + if (! entities.isEmpty()) + { + instanceStore.setEntities(entities); + } + + if (! relationships.isEmpty()) + { + instanceStore.setRelationships(relationships); + } + + archive.setArchiveInstanceStore(instanceStore); + } + + return archive; + } +}
http://git-wip-us.apache.org/repos/asf/atlas/blob/8a57e657/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveManager.java ---------------------------------------------------------------------- diff --git a/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveManager.java b/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveManager.java new file mode 100644 index 0000000..437bd7b --- /dev/null +++ b/omrs/src/main/java/org/apache/atlas/omrs/archivemanager/OMRSArchiveManager.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.omrs.archivemanager; + +import org.apache.atlas.omrs.archivemanager.opentypes.OpenMetadataTypesArchive; +import org.apache.atlas.omrs.archivemanager.store.OpenMetadataArchiveStore; +import org.apache.atlas.omrs.archivemanager.properties.OpenMetadataArchive; +import org.apache.atlas.omrs.archivemanager.properties.OpenMetadataArchiveInstanceStore; +import org.apache.atlas.omrs.archivemanager.properties.OpenMetadataArchiveProperties; +import org.apache.atlas.omrs.archivemanager.properties.OpenMetadataArchiveTypeStore; +import org.apache.atlas.omrs.auditlog.OMRSAuditLog; +import org.apache.atlas.omrs.auditlog.OMRSAuditingComponent; +import org.apache.atlas.omrs.eventmanagement.events.OMRSInstanceEventProcessor; +import org.apache.atlas.omrs.eventmanagement.events.OMRSTypeDefEventProcessor; +import org.apache.atlas.omrs.localrepository.repositorycontentmanager.OMRSRepositoryContentManager; +import org.apache.atlas.omrs.metadatacollection.properties.instances.EntityDetail; +import org.apache.atlas.omrs.metadatacollection.properties.instances.Relationship; +import org.apache.atlas.omrs.metadatacollection.properties.typedefs.TypeDef; +import org.apache.atlas.omrs.metadatacollection.properties.typedefs.TypeDefPatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; + +/** + * OMRSArchiveManager manages the loading and unloading of open metadata archives from the local OMRS repository. + * An open metadata archive provides pre-built definitions for types and metadata instances. + */ +public class OMRSArchiveManager +{ + ArrayList<OpenMetadataArchiveStore> openMetadataArchiveStores = new ArrayList<>(); + OMRSRepositoryContentManager repositoryContentManager = null; + OMRSInstanceEventProcessor localInstanceEventProcessor = null; + + + /* + * The audit log provides a verifiable record of the open metadata archives that have been loaded into + * the open metadata repository. The Logger is for standard debug. + */ + private static final OMRSAuditLog auditLog = new OMRSAuditLog(OMRSAuditingComponent.ARCHIVE_MANAGER); + private static final Logger log = LoggerFactory.getLogger(OMRSArchiveManager.class); + + + /** + * Constructor to save the initial list of open metadata archives from the server startup configuration. + * These will be processed as soon as the event processors are supplied from the local repository. + * + * @param startUpOpenMetadataArchives - initial list of open metadata archives provided in startup configuration + */ + public OMRSArchiveManager(ArrayList<OpenMetadataArchiveStore> startUpOpenMetadataArchives) + { + if (startUpOpenMetadataArchives != null) + { + for (OpenMetadataArchiveStore archiveStore : startUpOpenMetadataArchives) + { + this.openMetadataArchiveStores.add(archiveStore); + } + } + } + + + /** + * Close down any open archives. + */ + public void close() + { + for (OpenMetadataArchiveStore archiveStore : openMetadataArchiveStores) + { + if (archiveStore != null) + { + archiveStore.closeArchive(); + } + } + } + + + /** + * The local repository is accessed through its inbound event processors. A server will always have + * the local Content Manager and TypeDef Processor but the local Instance Processor is only available + * if the local server has a metadata repository defined. + * + * @param repositoryContentManager - typeDef processor for the local repository + * @param instanceProcessor - instance processor for the local repository + */ + public void setLocalRepository(OMRSRepositoryContentManager repositoryContentManager, + OMRSInstanceEventProcessor instanceProcessor) + { + this.repositoryContentManager = repositoryContentManager; + this.localInstanceEventProcessor = instanceProcessor; + + /* + * The repository content manager is seeded with all of the open metadata types. + */ + processOpenMetadataTypes(); + + /* + * Once the open metadata types are in place, the archive stores are processed. + */ + for (OpenMetadataArchiveStore archiveStore : this.openMetadataArchiveStores) + { + processOpenMetadataArchiveStore(archiveStore, repositoryContentManager, instanceProcessor); + } + } + + + /** + * Add a new archive to the local repository. If there are problems processing the archive, an exception is thrown. + * This method allows archives to be loaded into a running server. + * + * @param archiveStore - new open metadata archive to process + */ + public void addOpenMetadataArchive(OpenMetadataArchiveStore archiveStore) + { + this.processOpenMetadataArchiveStore(archiveStore, repositoryContentManager, localInstanceEventProcessor); + this.openMetadataArchiveStores.add(archiveStore); + } + + + /** + * Unpack and process the contents an open metadata archive store, passing its contents to the local + * repository (if it exists). + */ + private void processOpenMetadataTypes() + { + OpenMetadataTypesArchive openMetadataTypesArchive = new OpenMetadataTypesArchive(); + OpenMetadataArchive openMetadataTypes = openMetadataTypesArchive.getOpenMetadataArchive(); + + repositoryContentManager.setOpenMetadataTypesOriginGUID(openMetadataTypesArchive.getArchiveGUID()); + processOpenMetadataArchive(openMetadataTypes, repositoryContentManager, localInstanceEventProcessor); + } + + /** + * Unpack and process the contents an open metadata archive store, passing its contents to the local + * repository (if it exists). + * + * @param archiveStore - open metadata archive store to process + */ + private void processOpenMetadataArchiveStore(OpenMetadataArchiveStore archiveStore, + OMRSTypeDefEventProcessor typeDefProcessor, + OMRSInstanceEventProcessor instanceProcessor) + { + if (archiveStore != null) + { + /* + * Each archive store has a header, a section of new type definitions (TypeDefs) and a section of + * metadata instances. + */ + OpenMetadataArchive archiveContent = archiveStore.getArchiveContents(); + + if (archiveContent == null) + { + // TODO log Error + } + else + { + processOpenMetadataArchive(archiveContent, typeDefProcessor, instanceProcessor); + } + } + else + { + // TODO log empty Archive store + } + } + + + /** + * Step through the content of an open metadata archive, passing its contents to the local repository (if it + * exists). + * + * @param archiveContent - open metadata archive to process + */ + private void processOpenMetadataArchive(OpenMetadataArchive archiveContent, + OMRSTypeDefEventProcessor typeDefProcessor, + OMRSInstanceEventProcessor instanceProcessor) + { + OpenMetadataArchiveProperties archiveProperties = archiveContent.getArchiveProperties(); + OpenMetadataArchiveTypeStore archiveTypeStore = archiveContent.getArchiveTypeStore(); + OpenMetadataArchiveInstanceStore archiveInstanceStore = archiveContent.getArchiveInstanceStore(); + + // TODO Log that the open metadata archive is being processed + + if (archiveProperties != null) + { + if (archiveTypeStore != null) + { + this.processTypeDefStore(archiveProperties, archiveTypeStore, typeDefProcessor); + } + + if (archiveInstanceStore != null) + { + this.processInstanceStore(archiveProperties, archiveInstanceStore, instanceProcessor); + } + } + else + { + // TODO Log that the archive is invalid + } + + + // TODO log that processing complete + } + + + /** + * The TypeStore is in two parts. First there is an optional list of patches to existing TypeDefs. + * Then an optional list of new TypeDefs. It is possible that this archive has been processed before + * and so any duplicates detected are ignored. However, conflicting TypeDefs are detected. + * Any problems found in applying the archive contents are recorded on the audit log. + * + * @param archiveProperties - properties of the archive used for logging + * @param archiveTypeStore - TypeStore from the archive + */ + private void processTypeDefStore(OpenMetadataArchiveProperties archiveProperties, + OpenMetadataArchiveTypeStore archiveTypeStore, + OMRSTypeDefEventProcessor typeDefProcessor) + { + ArrayList<TypeDefPatch> typeDefPatches = archiveTypeStore.getTypeDefPatches(); + ArrayList<TypeDef> newTypeDefs = archiveTypeStore.getNewTypeDefs(); + + if (typeDefProcessor != null) + { + String sourceName = OMRSAuditingComponent.ARCHIVE_MANAGER.getComponentName(); + String originatorMetadataCollectionId = archiveProperties.getArchiveGUID(); + String originatorServerName = archiveProperties.getArchiveName(); + String originatorServerType = null; + String originatorOrganizationName = archiveProperties.getOriginatorName(); + + if (archiveProperties.getArchiveType() != null) + { + originatorServerType = archiveProperties.getArchiveType().getArchiveTypeName(); + } + + if (typeDefPatches != null) + { + for (TypeDefPatch typeDefPatch : typeDefPatches) + { + if (typeDefPatch != null) + { + typeDefProcessor.processUpdatedTypeDefEvent(sourceName, + originatorMetadataCollectionId, + originatorServerName, + originatorServerType, + originatorOrganizationName, + typeDefPatch); + } + } + } + + if (newTypeDefs != null) + { + for (TypeDef newTypeDef : newTypeDefs) + { + if (newTypeDef != null) + { + typeDefProcessor.processNewTypeDefEvent(sourceName, + originatorMetadataCollectionId, + originatorServerName, + originatorServerType, + originatorOrganizationName, + newTypeDef); + } + } + } + } + } + + + /** + * The InstanceStore is in two parts - an optional list of entities followed by an optional list + * of relationships. It is possible that this archive has been processed before + * and so any duplicates detected are ignored. However, conflicting instances are detected. + * Any problems found in applying the archive contents are recorded on the audit log. + * + * @param archiveProperties - properties describing the archive used in logging + * @param archiveInstanceStore - the instance store to process + * @param instanceProcessor - the processor to add the instances to the local repository. It may be null + * if there is no local repository configured for this server. + */ + private void processInstanceStore(OpenMetadataArchiveProperties archiveProperties, + OpenMetadataArchiveInstanceStore archiveInstanceStore, + OMRSInstanceEventProcessor instanceProcessor) + { + ArrayList<EntityDetail> entities = archiveInstanceStore.getEntities(); + ArrayList<Relationship> relationships = archiveInstanceStore.getRelationships(); + + if (instanceProcessor != null) + { + String sourceName = OMRSAuditingComponent.ARCHIVE_MANAGER.getComponentName(); + String originatorMetadataCollectionId = archiveProperties.getArchiveGUID(); + String originatorServerName = archiveProperties.getArchiveName(); + String originatorServerType = null; + String originatorOrganizationName = archiveProperties.getOriginatorName(); + + if (archiveProperties.getArchiveType() != null) + { + originatorServerType = archiveProperties.getArchiveType().getArchiveTypeName(); + } + + if (entities != null) + { + for (EntityDetail entity : entities) + { + if (entity != null) + { + instanceProcessor.processNewEntityEvent(sourceName, + originatorMetadataCollectionId, + originatorServerName, + originatorServerType, + originatorOrganizationName, + entity); + } + } + } + + + if (relationships != null) + { + for (Relationship relationship : relationships) + { + if (relationship != null) + { + instanceProcessor.processNewRelationshipEvent(sourceName, + originatorMetadataCollectionId, + originatorServerName, + originatorServerType, + originatorOrganizationName, + relationship); + } + } + } + } + } +}
