http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala b/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala deleted file mode 100644 index 7caf5e8..0000000 --- a/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.storm.model - -import org.apache.atlas.AtlasClient -import org.apache.atlas.typesystem.TypesDef -import org.apache.atlas.typesystem.builders.TypesBuilder -import org.apache.atlas.typesystem.json.TypesSerialization - - -/** - * This represents the data model for a storm topology. - */ -object StormDataModel extends App { - - var typesDef : TypesDef = null - - val typesBuilder = new TypesBuilder - import typesBuilder._ - - typesDef = types { - - /** - * Model is represented as: - * Topology is a Process Super Type inheriting inputs/outputs - * Input DataSet(s) => Topology => Output DataSet(s) - * Also, Topology contains the Graph of Nodes - * Topology => Node(s) -> Spouts/Bolts - */ - _class(StormDataTypes.STORM_TOPOLOGY.getName, List(AtlasClient.PROCESS_SUPER_TYPE)) { - "id" ~ (string, required, indexed, unique) - "startTime" ~ date - "endTime" ~ date - "conf" ~ (map(string, string), optional) - "clusterName" ~ (string, optional, indexed) - - // Nodes in the Graph - "nodes" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite) - } - - // Base class for DataProducer aka Spouts and - // DataProcessor aka Bolts, also links from Topology - _class(StormDataTypes.STORM_NODE.getName) { - "name" ~ (string, required, indexed) - "description" ~ (string, optional, indexed) - // fully qualified driver java class name - "driverClass" ~ (string, required, indexed) - // spout or bolt configuration NVPs - "conf" ~ (map(string, string), optional) - } - - // Data Producer and hence only outputs - _class(StormDataTypes.STORM_SPOUT.getName, List(StormDataTypes.STORM_NODE.getName)) { - // "outputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite) - "outputs" ~ (array(string), collection) - } - - // Data Processor and hence both inputs and outputs (inherited from Spout) - _class(StormDataTypes.STORM_BOLT.getName, List(StormDataTypes.STORM_NODE.getName)) { - // "inputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite) - "inputs" ~ (array(string), collection) - "outputs" ~ (array(string), collection, optional) - } - - // Kafka Data Set - _class(StormDataTypes.KAFKA_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) { - "topic" ~ (string, required, unique, indexed) - "uri" ~ (string, required) - } - - // JMS Data Set - _class(StormDataTypes.JMS_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) { - "topic" ~ (string, required, unique, indexed) - "uri" ~ (string, required) - } - - // HBase Data Set - _class(StormDataTypes.HBASE_TABLE.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) { - "uri" ~ (string, required) - } - // Hive table data set already exists in atlas. - } - - // add the types to atlas - val typesAsJSON = TypesSerialization.toJson(typesDef) - println("Storm Data Model as JSON: ") - println(typesAsJSON) -}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java index fe4c001..e0800b8 100644 --- a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java +++ b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java @@ -18,19 +18,11 @@ package org.apache.atlas.storm.hook; -import com.sun.jersey.api.client.ClientResponse; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasException; -import org.apache.atlas.AtlasServiceException; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; -import org.apache.atlas.hive.model.HiveDataModelGenerator; -import org.apache.atlas.hive.model.HiveDataTypes; -import org.apache.atlas.storm.model.StormDataModel; import org.apache.atlas.storm.model.StormDataTypes; import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.utils.AuthenticationUtil; import org.apache.commons.configuration.Configuration; import org.apache.storm.ILocalCluster; @@ -67,37 +59,6 @@ public class StormAtlasHookIT { } else { atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT)); } - registerDataModel(new HiveDataModelGenerator()); - } - - private void registerDataModel(HiveDataModelGenerator dataModelGenerator) throws AtlasException, - AtlasServiceException { - try { - atlasClient.getType(HiveDataTypes.HIVE_PROCESS.getName()); - LOG.info("Hive data model is already registered! Going ahead with registration of Storm Data model"); - } catch(AtlasServiceException ase) { - if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) { - //Expected in case types do not exist - LOG.info("Registering Hive data model"); - atlasClient.createType(dataModelGenerator.getModelAsJson()); - } else { - throw ase; - } - } - - - try { - atlasClient.getType(StormDataTypes.STORM_TOPOLOGY.getName()); - } catch(AtlasServiceException ase) { - if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) { - LOG.info("Registering Storm/Kafka data model"); - StormDataModel.main(new String[]{}); - TypesDef typesDef = StormDataModel.typesDef(); - String stormTypesAsJSON = TypesSerialization.toJson(typesDef); - LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON); - atlasClient.createType(stormTypesAsJSON); - } - } } @@ -109,23 +70,6 @@ public class StormAtlasHookIT { atlasClient = null; } - @Test - public void testCreateDataModel() throws Exception { - StormDataModel.main(new String[]{}); - TypesDef stormTypesDef = StormDataModel.typesDef(); - - String stormTypesAsJSON = TypesSerialization.toJson(stormTypesDef); - LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON); - - registerDataModel(new HiveDataModelGenerator()); - - // verify types are registered - for (StormDataTypes stormDataType : StormDataTypes.values()) { - Assert.assertNotNull(atlasClient.getType(stormDataType.getName())); - } - } - - @Test (dependsOnMethods = "testCreateDataModel") public void testAddEntities() throws Exception { StormTopology stormTopology = StormTestUtil.createTestTopology(); StormTestUtil.submitTopology(stormCluster, TOPOLOGY_NAME, stormTopology); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/distro/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml index 39c6512..215cb23 100755 --- a/distro/src/main/assemblies/standalone-package.xml +++ b/distro/src/main/assemblies/standalone-package.xml @@ -106,9 +106,9 @@ <outputDirectory>examples</outputDirectory> </fileSet> - <!-- addons/hdfs-model --> + <!-- out-of-box-models --> <fileSet> - <directory>../addons/hdfs-model/target/models</directory> + <directory>../addons/models</directory> <outputDirectory>models</outputDirectory> </fileSet> @@ -130,55 +130,29 @@ <outputDirectory>hook</outputDirectory> </fileSet> - <fileSet> - <directory>../addons/hive-bridge/target/models</directory> - <outputDirectory>models</outputDirectory> - </fileSet> - <!-- addons/falcon --> <fileSet> <directory>../addons/falcon-bridge/target/dependency/hook</directory> <outputDirectory>hook</outputDirectory> </fileSet> - <fileSet> - <directory>../addons/falcon-bridge/target/models</directory> - <outputDirectory>models</outputDirectory> - </fileSet> - <!-- addons/sqoop --> <fileSet> <directory>../addons/sqoop-bridge/target/dependency/hook</directory> <outputDirectory>hook</outputDirectory> </fileSet> - <fileSet> - <directory>../addons/sqoop-bridge/target/models</directory> - <outputDirectory>models</outputDirectory> - </fileSet> - <!-- addons/storm --> <fileSet> <directory>../addons/storm-bridge/target/dependency/hook</directory> <outputDirectory>hook</outputDirectory> </fileSet> - <fileSet> - <directory>../addons/storm-bridge/target/models</directory> - <outputDirectory>models</outputDirectory> - </fileSet> - <!-- for kafka topic setup --> <fileSet> <directory>../notification/target/dependency/hook</directory> <outputDirectory>hook</outputDirectory> </fileSet> - - <!-- for patches --> - <fileSet> - <directory>../addons/hive-bridge/src/patches</directory> - <outputDirectory>models/patches</outputDirectory> - </fileSet> </fileSets> <files> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 709fcbc..fe38fba 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -53,7 +53,9 @@ public enum AtlasErrorCode { INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"), INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"), - INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}") + INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"), + PATCH_NOT_APPLICABLE_FOR_TYPE(500, "ATLAS5004E", "{0} - invalid patch for type {1}"), + PATCH_FOR_UNKNOWN_TYPE(500, "ATLAS5005E", "{0} - patch references unknown type {1}") ; private String errorCode; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index d5ed0bc..7421da8 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -279,12 +279,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { List<AtlasConstraintDef> constraintDefs) { setName(name); setTypeName(typeName); - setOptional(isOptional); + setIsOptional(isOptional); setCardinality(cardinality); setValuesMinCount(valuesMinCount); setValuesMaxCount(valuesMaxCount); - setUnique(isUnique); - setIndexable(isIndexable); + setIsUnique(isUnique); + setIsIndexable(isIndexable); setConstraintDefs(constraintDefs); } @@ -292,12 +292,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { if (other != null) { setName(other.getName()); setTypeName(other.getTypeName()); - setOptional(other.isOptional()); + setIsOptional(other.getIsOptional()); setCardinality(other.getCardinality()); setValuesMinCount(other.getValuesMinCount()); setValuesMaxCount(other.getValuesMaxCount()); - setUnique(other.isUnique()); - setIndexable(other.isIndexable()); + setIsUnique(other.getIsUnique()); + setIsIndexable(other.getIsIndexable()); setConstraintDefs(other.getConstraintDefs()); } } @@ -318,11 +318,11 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { this.typeName = typeName; } - public boolean isOptional() { + public boolean getIsOptional() { return isOptional; } - public void setOptional(boolean optional) { isOptional = optional; } + public void setIsOptional(boolean optional) { isOptional = optional; } public void setCardinality(Cardinality cardinality) { this.cardinality = cardinality; @@ -348,19 +348,19 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { this.valuesMaxCount = valuesMaxCount; } - public boolean isUnique() { + public boolean getIsUnique() { return isUnique; } - public void setUnique(boolean unique) { + public void setIsUnique(boolean unique) { isUnique = unique; } - public boolean isIndexable() { + public boolean getIsIndexable() { return isIndexable; } - public void setIndexable(boolean idexable) { + public void setIsIndexable(boolean idexable) { isIndexable = idexable; } @@ -399,7 +399,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { sb.append("AtlasAttributeDef{"); sb.append("name='").append(name).append('\''); sb.append(", typeName='").append(typeName).append('\''); - sb.append(", isOptional=").append(isOptional); + sb.append(", getIsOptional=").append(isOptional); sb.append(", cardinality=").append(cardinality); sb.append(", valuesMinCount=").append(valuesMinCount); sb.append(", valuesMaxCount=").append(valuesMaxCount); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java index fb2029d..899e53f 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java @@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef; import org.apache.commons.collections.CollectionUtils; import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnore; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -91,6 +92,7 @@ public class AtlasTypesDef { this.entityDefs = entityDefs; } + @JsonIgnore public boolean isEmpty() { return CollectionUtils.isEmpty(enumDefs) && CollectionUtils.isEmpty(structDefs) && http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 78b29fd..c8c2216 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -198,7 +198,7 @@ public class AtlasStructType extends AtlasType { if (value != null) { ret = dataType.validateValue(value, fieldName, messages) && ret; - } else if (!attributeDef.isOptional()) { + } else if (!attributeDef.getIsOptional()) { ret = false; messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName()); @@ -218,7 +218,7 @@ public class AtlasStructType extends AtlasType { if (value != null) { ret = dataType.validateValue(value, fieldName, messages) && ret; - } else if (!attributeDef.isOptional()) { + } else if (!attributeDef.getIsOptional()) { ret = false; messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName()); @@ -244,7 +244,7 @@ public class AtlasStructType extends AtlasType { Object attributeValue = getNormalizedValue(obj.getAttribute(attributeName), attributeDef); obj.setAttribute(attributeName, attributeValue); - } else if (!attributeDef.isOptional()) { + } else if (!attributeDef.getIsOptional()) { obj.setAttribute(attributeName, createDefaultValue(attributeDef)); } } @@ -260,7 +260,7 @@ public class AtlasStructType extends AtlasType { Object attributeValue = getNormalizedValue(obj.get(attributeName), attributeDef); obj.put(attributeName, attributeValue); - } else if (!attributeDef.isOptional()) { + } else if (!attributeDef.getIsOptional()) { obj.put(attributeName, createDefaultValue(attributeDef)); } } @@ -276,7 +276,7 @@ public class AtlasStructType extends AtlasType { } for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { - if (!attributeDef.isOptional()) { + if (!attributeDef.getIsOptional()) { attributes.put(attributeDef.getName(), createDefaultValue(attributeDef)); } } @@ -310,7 +310,7 @@ public class AtlasStructType extends AtlasType { ret = false; // invalid value } } - } else if (!attributeDef.isOptional()) { + } else if (!attributeDef.getIsOptional()) { ret = false; // mandatory attribute not present } @@ -322,7 +322,7 @@ public class AtlasStructType extends AtlasType { if (attrType != null) { if (value == null) { - if (!attributeDef.isOptional()) { + if (!attributeDef.getIsOptional()) { return attrType.createDefaultValue(); } } else { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index 8924d44..0c118f0 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -64,6 +64,10 @@ public class AtlasTypeRegistry { public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); } + public boolean isRegisteredType(String typeName) { + return registryData.allTypes.isKnownType(typeName); + } + public AtlasType getType(String typeName) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasTypeRegistry.getType({})", typeName); @@ -677,6 +681,10 @@ class TypeCache { } } + public boolean isKnownType(String typeName) { + return typeNameMap.containsKey(typeName); + } + public Collection<String> getAllTypeNames() { return Collections.unmodifiableCollection(typeNameMap.keySet()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index cd44318..aabf269 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -44,9 +44,7 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.service.Service; import org.apache.atlas.services.DefaultMetadataService; -import org.apache.atlas.services.IBootstrapTypesRegistrar; import org.apache.atlas.services.MetadataService; -import org.apache.atlas.services.ReservedTypesRegistrar; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.types.TypeSystem; @@ -92,8 +90,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { // bind the MetadataService interface to an implementation bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton(); - bind(IBootstrapTypesRegistrar.class).to(ReservedTypesRegistrar.class); - // bind the DiscoveryService interface to an implementation bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 67b5362..aea54fa 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -267,8 +267,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang AtlasAttributeDef attributeDef) { final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName()); AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality()); - boolean isUnique = attributeDef.isUnique(); - boolean isIndexable = attributeDef.isIndexable(); + boolean isUnique = attributeDef.getIsUnique(); + boolean isIndexable = attributeDef.getIsIndexable(); String attribTypeName = attributeDef.getTypeName(); boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java new file mode 100644 index 0000000..e52ac6d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java @@ -0,0 +1,364 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.bootstrap; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; + + +/** + * Class that handles initial loading of models and patches into typedef store + */ +public class AtlasTypeDefStoreInitializer { + private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class); + + public void initializeStore(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) { + File typesDir = new File(typesDirName); + File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null; + + if (typeDefFiles == null || typeDefFiles.length == 0) { + LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName); + + return; + } + + // sort the files by filename + Arrays.sort(typeDefFiles); + + for (File typeDefFile : typeDefFiles) { + if (!typeDefFile.isFile()) { + continue; + } + + try { + String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8); + AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class); + + if (typesDef == null || typesDef.isEmpty()) { + LOG.info("No type in file {}", typeDefFile.getAbsolutePath()); + + continue; + } + + AtlasTypesDef typesToCreate = new AtlasTypesDef(); + + if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { + for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { + if (!typeRegistry.isRegisteredType(enumDef.getName())) { + typesToCreate.getEnumDefs().add(enumDef); + } + } + } + + if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) { + for (AtlasStructDef structDef : typesDef.getStructDefs()) { + if (!typeRegistry.isRegisteredType(structDef.getName())) { + typesToCreate.getStructDefs().add(structDef); + } + } + } + + if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) { + for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) { + if (!typeRegistry.isRegisteredType(classificationDef.getName())) { + typesToCreate.getClassificationDefs().add(classificationDef); + } + } + } + + if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) { + for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) { + if (!typeRegistry.isRegisteredType(entityDef.getName())) { + typesToCreate.getEntityDefs().add(entityDef); + } + } + } + + if (typesToCreate.isEmpty()) { + LOG.info("No new type in file {}", typeDefFile.getAbsolutePath()); + + continue; + } + + LOG.info("Loading types defined in file {}", typeDefFile.getAbsolutePath()); + + typeDefStore.createTypesDef(typesDef); + } catch (Throwable t) { + LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), t); + } + } + + applyTypePatches(typeDefStore, typeRegistry, typesDirName); + } + + private void applyTypePatches(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) { + String typePatchesDirName = typesDirName + File.separator + "patches"; + File typePatchesDir = new File(typePatchesDirName); + File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null; + + if (typePatchFiles == null || typePatchFiles.length == 0) { + LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName); + + return; + } + + // sort the files by filename + Arrays.sort(typePatchFiles); + + PatchHandler[] patchHandlers = new PatchHandler[] { new AddAttributePatchHandler(typeDefStore, typeRegistry) }; + + Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>(); + + for (PatchHandler patchHandler : patchHandlers) { + for (String supportedAction : patchHandler.getSupportedActions()) { + patchHandlerRegistry.put(supportedAction, patchHandler); + } + } + + for (File typePatchFile : typePatchFiles) { + if (!typePatchFile.isFile()) { + continue; + } + + try { + String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8); + TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class); + + if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) { + LOG.info("No patches in file {}", typePatchFile.getAbsolutePath()); + + continue; + } + + for (TypeDefPatch patch : patches.getPatches()) { + PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction()); + + if (patchHandler == null) { + LOG.error("Unknown patch action {} in file {}. Ignored", + patch.getAction(), typePatchFile.getAbsolutePath()); + + continue; + } + + try { + patchHandler.applyPatch(patch); + } catch (AtlasBaseException excp) { + LOG.error("Failed to apply " + patch.getAction() + " patch in file " + + typePatchFile.getAbsolutePath() + ". Ignored", excp); + } + } + } catch (Throwable t) { + LOG.error("Failed to apply patches in file " + typePatchFile.getAbsolutePath() + ". Ignored", t); + } + } + } + + /** + * typedef patch details + */ + @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) + @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.PROPERTY) + static class TypeDefPatch { + private String action; + private String typeName; + private String applyToVersion; + private String updateToVersion; + private Map<String, Object> params; + private List<AtlasAttributeDef> attributeDefs; + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + public String getApplyToVersion() { + return applyToVersion; + } + + public void setApplyToVersion(String applyToVersion) { + this.applyToVersion = applyToVersion; + } + + public String getUpdateToVersion() { + return updateToVersion; + } + + public void setUpdateToVersion(String updateToVersion) { + this.updateToVersion = updateToVersion; + } + + public Map<String, Object> getParams() { + return params; + } + + public void setParams(Map<String, Object> params) { + this.params = params; + } + + public List<AtlasAttributeDef> getAttributeDefs() { + return attributeDefs; + } + + public void setAttributeDefs(List<AtlasAttributeDef> attributeDefs) { + this.attributeDefs = attributeDefs; + } + } + + /** + * list of typedef patches + */ + @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) + @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.PROPERTY) + static class TypeDefPatches { + private List<TypeDefPatch> patches; + + public List<TypeDefPatch> getPatches() { + return patches; + } + + public void setPatches(List<TypeDefPatch> patches) { + this.patches = patches; + } + } + + abstract class PatchHandler { + protected final AtlasTypeDefStore typeDefStore; + protected final AtlasTypeRegistry typeRegistry; + protected final String[] supportedActions; + + protected PatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String[] supportedActions) { + this.typeDefStore = typeDefStore; + this.typeRegistry = typeRegistry; + this.supportedActions = supportedActions; + } + + public String[] getSupportedActions() { return supportedActions; } + + public abstract void applyPatch(TypeDefPatch patch) throws AtlasBaseException; + + protected boolean isPatchApplicable(TypeDefPatch patch, AtlasBaseTypeDef currentTypeDef) { + String currentVersion = currentTypeDef.getTypeVersion(); + String applyToVersion = patch.getApplyToVersion(); + + return currentVersion == null || + currentVersion.equalsIgnoreCase(applyToVersion) || + currentVersion.startsWith(applyToVersion + "."); + } + } + + class AddAttributePatchHandler extends PatchHandler { + public AddAttributePatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) { + super(typeDefStore, typeRegistry, new String[] { "ADD_ATTRIBUTE" }); + } + + @Override + public void applyPatch(TypeDefPatch patch) throws AtlasBaseException { + String typeName = patch.getTypeName(); + + if (!typeRegistry.isRegisteredType(typeName)) { + throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName); + } + + AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName); + + if (isPatchApplicable(patch, typeDef)) { + if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef)typeDef); + + for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) { + updatedDef.addAttribute(attributeDef); + } + updatedDef.setTypeVersion(patch.getUpdateToVersion()); + + typeDefStore.updateEntityDefByName(typeName, updatedDef); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef updatedDef = new AtlasClassificationDef((AtlasClassificationDef)typeDef); + + for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) { + updatedDef.addAttribute(attributeDef); + } + updatedDef.setTypeVersion(patch.getUpdateToVersion()); + + typeDefStore.updateClassificationDefByName(typeName, updatedDef); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef); + + for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) { + updatedDef.addAttribute(attributeDef); + } + updatedDef.setTypeVersion(patch.getUpdateToVersion()); + + typeDefStore.updateStructDefByName(typeName, updatedDef); + } else { + throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE, + patch.getAction(), typeDef.getClass().getSimpleName()); + } + } else { + LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}", + patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion()); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java index 2163e01..816832b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java @@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumDefs; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs; import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.repository.util.FilterUtil; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; @@ -45,6 +46,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -89,6 +91,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ ttr.addTypes(typesDef); typeRegistry.commitTransientTypeRegistry(ttr); + + bootstrapTypes(); } @Override @@ -886,6 +890,15 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ LOG.info("Not reacting to a Passive state change"); } + private void bootstrapTypes() { + AtlasTypeDefStoreInitializer storeInitializer = new AtlasTypeDefStoreInitializer(); + + String atlasHomeDir = System.getProperty("atlas.home"); + String typesDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models"; + + storeInitializer.initializeStore(this, typeRegistry, typesDirName); + } + private void updateTypeRegistryPostCommit(AtlasTransientTypeRegistry ttr) { new TypeRegistryUpdateHook(ttr); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index 0dd7164..6dfe8cf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -421,7 +421,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) { // new attribute - only allow if optional - if (!attributeDef.isOptional()) { + if (!attributeDef.getIsOptional()) { throw new AtlasBaseException(AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE, structDef.getName(), attributeDef.getName()); } } @@ -510,13 +510,30 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At attribInfo.put("name", attributeDef.getName()); attribInfo.put("dataType", attributeDef.getTypeName()); - attribInfo.put("isUnique", attributeDef.isUnique()); - attribInfo.put("isIndexable", attributeDef.isIndexable()); + attribInfo.put("isUnique", attributeDef.getIsUnique()); + attribInfo.put("isIndexable", attributeDef.getIsIndexable()); attribInfo.put("isComposite", isComposite); attribInfo.put("reverseAttributeName", reverseAttribName); + + final int lower; + final int upper; + + if (attributeDef.getCardinality() == AtlasAttributeDef.Cardinality.SINGLE) { + lower = attributeDef.getIsOptional() ? 0 : 1; + upper = 1; + } else { + if(attributeDef.getIsOptional()) { + lower = 0; + } else { + lower = attributeDef.getValuesMinCount() < 1 ? 1 : attributeDef.getValuesMinCount(); + } + + upper = attributeDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attributeDef.getValuesMaxCount(); + } + Map<String, Object> multiplicity = new HashMap<>(); - multiplicity.put("lower", attributeDef.getValuesMinCount()); - multiplicity.put("upper", attributeDef.getValuesMaxCount()); + multiplicity.put("lower", lower); + multiplicity.put("upper", upper); multiplicity.put("isUnique", AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality())); attribInfo.put("multiplicity", AtlasType.toJson(multiplicity)); @@ -532,8 +549,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At ret.setName((String) attribInfo.get("name")); ret.setTypeName((String) attribInfo.get("dataType")); - ret.setUnique((Boolean) attribInfo.get("isUnique")); - ret.setIndexable((Boolean) attribInfo.get("isIndexable")); + ret.setIsUnique((Boolean) attribInfo.get("isUnique")); + ret.setIsIndexable((Boolean) attribInfo.get("isIndexable")); String attrTypeName = ret.getTypeName(); @@ -608,10 +625,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At Boolean isUnique = (Boolean) multiplicity.get("isUnique"); if (minCount == null || minCount.intValue() == 0) { - ret.setOptional(true); + ret.setIsOptional(true); ret.setValuesMinCount(0); } else { - ret.setOptional(false); + ret.setIsOptional(false); ret.setValuesMinCount(minCount.intValue()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java b/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java deleted file mode 100644 index f916169..0000000 --- a/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.services; - - -import com.google.gson.FieldNamingPolicy; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSyntaxException; -import org.apache.atlas.services.AtlasTypePatch.PatchContent; -import org.apache.atlas.services.AtlasTypePatch.PatchData; -import org.apache.atlas.services.AtlasTypePatch.PatchResult; -import org.apache.atlas.services.AtlasTypePatch.PatchStatus; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.TypeUpdateException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Type; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class AtlasPatchHandler { - - private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchHandler.class); - - public static void handlePatches(DefaultMetadataService metadataService, TypeSystem typeSystem) throws TypeUpdateException { - - Map<String, AtlasTypePatch> patchHandlerMap = initializePatchHandlerMap(metadataService, typeSystem); - - if (patchHandlerMap == null || patchHandlerMap.isEmpty()) - return; - - String patchDirName = ReservedTypesRegistrar.getTypesDir() + File.separator + "patches"; - LOG.info("Checking for any patches to be applied to the type system in " + patchDirName); - - File patchDir = new File(patchDirName); - if (!patchDir.exists()) { - LOG.info("Patch directory {} doesn't exist, not applying any patches", patchDirName); - return; - } - - File[] patchFiles = patchDir.listFiles(); - if (patchFiles == null || patchFiles.length == 0) { - LOG.info("No patch files found in {}, not applying any patches", patchDirName); - return; - } - - // Sort the patch files based on file name. - Arrays.sort(patchFiles, new Comparator<File>() { - public int compare(File f1, File f2) { - return String.valueOf(f1.getName()).compareTo(f2.getName()); - } - }); - - LOG.info("Found " + patchFiles.length + " patch files to process."); - int patchNumber = 0; - Gson gson = initializeGson(); - AtlasTypePatch typePatch; - - for (File patchFile : patchFiles) { - try { - LOG.info("Processing patch file " + (++patchNumber) + " - " + patchFile.getAbsolutePath()); - String content = new String(Files.readAllBytes(patchFile.toPath()), StandardCharsets.UTF_8); - PatchContent patchContent = gson.fromJson(content, PatchContent.class); - PatchData[] patchDatas = patchContent.getPatches(); - PatchResult result; - int patchCounter = 0; - - for (PatchData patch : patchDatas) { - typePatch = patchHandlerMap.get(patch.getAction()); - if (typePatch != null) { - result = typePatch.applyPatch(patch); - - if (result != null) { - LOG.info(result.getMessage() + " Patch " + (++patchCounter) + " out of " + patchDatas.length + " processed in : " + patchFile.toPath()); - if (result.getStatus().equals(PatchStatus.FAILED)) { - throw new TypeUpdateException(result.getMessage() + " patch " + patchNumber + " failed in :" + patchFile.getAbsolutePath()); - } - } - } - } - - } catch (IOException e) { - throw new TypeUpdateException("Unable to read patch file from " + patchFile.getAbsolutePath()); - } catch (JsonSyntaxException e) { - throw new TypeUpdateException("Invalid non-parseable JSON patch file in " + patchFile.getAbsolutePath()); - } - } - - LOG.info("Processed " + patchFiles.length + " patch files."); - } - - private static Map<String, AtlasTypePatch> initializePatchHandlerMap(DefaultMetadataService metadataService, TypeSystem typeSystem) { - Map<String, AtlasTypePatch> patchHandlerMap = new HashMap<String, AtlasTypePatch>(); - List<AtlasTypePatch> patchers = new ArrayList<AtlasTypePatch>(); - - // Register new patch classes here - patchers.add(new AtlasTypeAttributePatch(metadataService, typeSystem)); - - for (AtlasTypePatch patcher : patchers) { - for (String action : patcher.getSupportedActions()) { - patchHandlerMap.put(action, patcher); - } - } - - return patchHandlerMap; - } - - public static Gson initializeGson() { - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Multiplicity.class, new MultiplicityDeserializer()); - gsonBuilder.setFieldNamingPolicy(FieldNamingPolicy.IDENTITY); - Gson gson = gsonBuilder.create(); - - return gson; - } - - static class MultiplicityDeserializer implements JsonDeserializer<Multiplicity> { - @Override - public Multiplicity deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) - throws JsonParseException { - String multiplicityString = json.getAsString().toLowerCase(); - Multiplicity m = null; - switch (multiplicityString) { - case "optional": - m = Multiplicity.OPTIONAL; - break; - case "required": - m = Multiplicity.REQUIRED; - break; - case "collection": - m = Multiplicity.COLLECTION; - break; - case "set": - m = Multiplicity.SET; - break; - default: - break; - } - return m; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java b/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java deleted file mode 100644 index b918c87..0000000 --- a/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java +++ /dev/null @@ -1,296 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.services; - - -import com.google.common.collect.ImmutableList; -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.EnumTypeDefinition; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructTypeDefinition; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.codehaus.jettison.json.JSONException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -public class AtlasTypeAttributePatch extends AtlasTypePatch { - - private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeAttributePatch.class); - private static final String STRUCT_TYPE = "STRUCT"; - private static final String CLASS_TYPE = "CLASS"; - private static final String TRAIT_TYPE = "TRAIT"; - private static final String[] SUPPORTED_ACTIONS = new String[] { "ADD_ATTRIBUTE", "UPDATE_ATTRIBUTE", "DELETE_ATTRIBUTE" }; - - public AtlasTypeAttributePatch(DefaultMetadataService metadataService, TypeSystem typeSystem) { - super(metadataService, typeSystem, SUPPORTED_ACTIONS); - } - - /********* SAMPLE PATCH DATA *********** - * - * - { - "patches": [ - { - "action": "ADD_ATTRIBUTE", - "typeName": "hive_column", - "applyToVersion": "1.0", - "updateToVersion": "2.0", - "params": { - "optionalParam1" : "true", - "optionalParam2" : "false" - }, - "attributeDefinitions": [ - { - "name": "position", - "dataTypeName": "string", - "multiplicity": "optional", - "isComposite": false, - "isUnique": false, - "isIndexable": false, - "reverseAttributeName": null - } - ] - } - ] - } - * - */ - - @Override - public PatchResult applyPatch(PatchData patchData) { - String typeName = patchData.getTypeName(); - String applyVersion = patchData.getApplyToVersion(); - TypesDef updatedTypeDef; - PatchResult result; - try { - // Check if type exists in type system - if (checkIfTypeExists(typeName, metadataService)) { - TypesDef typesDef = getTypeDef(typeName); - String currentVersion = getTypeVersion(typeName); - - // Check version to apply patch - if (currentVersion == null || currentVersion.equalsIgnoreCase(applyVersion) || currentVersion.startsWith(applyVersion + ".")) { - updatedTypeDef = updateTypesDef(typesDef, patchData); - - if (updatedTypeDef != null) { - metadataService.updateType(TypesSerialization.toJson(updatedTypeDef)); - LOG.info("updated " + patchData.getTypeName() + " type from version " + patchData.getApplyToVersion() + " to " + patchData.getUpdateToVersion()); - result = new PatchResult("patch applied successfully!", PatchStatus.SUCCESS); - } else { - LOG.error("failed to create updated typedef for type=" +typeName+ "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() ); - result = new PatchResult("patch failed!", PatchStatus.FAILED); - } - } else { - LOG.info("patch skipped for " + patchData.getTypeName()); - result = new PatchResult("patch skipped!", PatchStatus.SKIPPED); - } - - } else { - LOG.error("failed to apply patch (typeName=" + typeName + "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() + "): type doesn't exist"); - result = new PatchResult("patch failed!", PatchStatus.FAILED); - } - - } catch (AtlasException e) { - LOG.error("error in updating type for " + patchData.getTypeName()); - result = new PatchResult("unable to update type", PatchStatus.FAILED); - } catch (JSONException e) { - LOG.error("error in updating typedef for " + patchData.getTypeName()); - result = new PatchResult("unable to update typedef", PatchStatus.FAILED); - } - - return result; - } - - public TypesDef updateTypesDef(TypesDef typesDef, PatchData patchData) throws AtlasException, JSONException { - AttributeDefinition[] typeAttributes = getAttributesFromTypesDef(typesDef, patchData.getTypeName()); - AttributeDefinition[] patchAttributes = patchData.getAttributeDefinitions(); - AttributeDefinition[] newAttributes = new AttributeDefinition[0]; - String patchAction = patchData.getAction(); - TypesDef newTypeDef = null; - - if (patchAction != null && typeAttributes != null && patchAttributes != null) { - switch (patchAction) { - case "ADD_ATTRIBUTE": - LOG.info("adding new attribute to type {}", patchData.getTypeName()); - newAttributes = addAttributes(typeAttributes, patchAttributes); - break; - case "DELETE_ATTRIBUTE": - LOG.info("deleting attribute from type {}", patchData.getTypeName()); - newAttributes = deleteAttributes(typeAttributes, patchAttributes); - break; - case "UPDATE_ATTRIBUTE": - LOG.info("updating attribute in type {}", patchData.getTypeName()); - newAttributes = updateAttributes(typeAttributes, patchAttributes); - break; - default: - LOG.info("invalid action " + patchAction + ", supported actions: " + Arrays.toString(SUPPORTED_ACTIONS)); - break; - } - - newTypeDef = createTypeDefWithNewAttributes(typesDef, patchData.getTypeName(), newAttributes, patchData.getUpdateToVersion()); - } - - return newTypeDef; - } - - private AttributeDefinition[] addAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) { - ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>(Arrays.asList(typeAttributes)); - Map<String, AttributeDefinition> typeAttrs = getAttributeNamesFromDefinition(typeAttributes); - - for (AttributeDefinition attr : patchAttributes) { - if (!typeAttrs.containsKey(attr.name)) - newAttrList.add(attr); - } - - return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]); - } - - private AttributeDefinition[] deleteAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) { - ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>(); - Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes); - - for (AttributeDefinition attr : typeAttributes) { - if (!patchAttrs.containsKey(attr.name)) - newAttrList.add(attr); - } - - return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]); - } - - private AttributeDefinition[] updateAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) { - ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>(); - Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes); - AttributeDefinition newAttr; - - for (AttributeDefinition attr : typeAttributes) { - newAttr = patchAttrs.get(attr.name); - if (patchAttrs.containsKey(attr.name) && checkIfAttributeUpdatable(attr, newAttr)) { - newAttrList.add(newAttr); - } else { - newAttrList.add(attr); - } - } - - return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]); - } - - private TypesDef createTypeDefWithNewAttributes(TypesDef typesDef, String typeName, AttributeDefinition[] newAttributes, String newVersion) throws AtlasException { - ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder(); - ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder(); - ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder(); - ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder(); - String dataType = getDataType(typeName).toUpperCase(); - switch (dataType) { - case STRUCT_TYPE: - StructTypeDefinition structType = typesDef.structTypesAsJavaList().get(0); - structs.add(new StructTypeDefinition(structType.typeName, structType.typeDescription, newVersion, newAttributes)); - break; - case CLASS_TYPE: - HierarchicalTypeDefinition<ClassType> classType = typesDef.classTypesAsJavaList().get(0); - classTypes.add(new HierarchicalTypeDefinition(ClassType.class, classType.typeName, classType.typeDescription, newVersion, classType.superTypes, newAttributes)); - break; - case TRAIT_TYPE: - HierarchicalTypeDefinition<TraitType> traitType = typesDef.traitTypesAsJavaList().get(0); - traits.add(new HierarchicalTypeDefinition(TraitType.class, traitType.typeName, traitType.typeDescription, newVersion, traitType.superTypes, newAttributes)); - break; - default: - LOG.error("unhandled datatype {} when creating new typedef", dataType); - } - - return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build()); - } - - private AttributeDefinition[] getAttributesFromTypesDef(TypesDef typesDef, String typeName) throws AtlasException { - AttributeDefinition[] typeAttributes = null; - String dataType = getDataType(typeName).toUpperCase(); - switch (dataType) { - case STRUCT_TYPE: - typeAttributes = typesDef.structTypesAsJavaList().get(0).attributeDefinitions; - break; - case CLASS_TYPE: - typeAttributes = typesDef.classTypesAsJavaList().get(0).attributeDefinitions; - break; - case TRAIT_TYPE: - typeAttributes = typesDef.traitTypesAsJavaList().get(0).attributeDefinitions; - break; - default: - LOG.error("unhandled datatype {}", dataType); - } - - return typeAttributes; - } - - private Map<String, AttributeDefinition> getAttributeNamesFromDefinition(AttributeDefinition[] attribDef) { - Map<String, AttributeDefinition> attrsMap = new HashMap<String, AttributeDefinition>(); - for (AttributeDefinition attr : attribDef) { - attrsMap.put(attr.name, attr); - } - - return attrsMap; - } - - private boolean checkIfAttributeUpdatable(AttributeDefinition attr, AttributeDefinition newAttr) { - boolean result = false; - if (!attr.equals(newAttr) && (attr.multiplicity == Multiplicity.REQUIRED - && newAttr.multiplicity == Multiplicity.OPTIONAL)) { - result = true; - } - - return result; - } - - // Returns the datatype the typename belongs to - PRIMITIVE, ENUM, ARRAY, MAP, STRUCT, TRAIT, CLASS - private String getDataType(String typeName) throws AtlasException { - IDataType dataType = typeSystem.getDataType(IDataType.class, typeName); - return dataType.getTypeCategory().toString(); - } - - private String getTypeVersion(String typeName) throws AtlasException { - return typeSystem.getDataType(IDataType.class, typeName).getVersion(); - } - - private TypesDef getTypeDef(String typeName) throws AtlasException { - return TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName)); - } - - private static boolean checkIfTypeExists(String typeName, DefaultMetadataService metadataService) { - boolean result = true; - try { - metadataService.getTypeDefinition(typeName); - } catch (AtlasException e) { - result = false; - } - - return result; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java b/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java deleted file mode 100644 index 0698efb..0000000 --- a/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.services; - - -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.TypeSystem; - -import java.util.Map; - - -public abstract class AtlasTypePatch { - protected final TypeSystem typeSystem; - protected final DefaultMetadataService metadataService; - protected final String[] supportedActions; - - protected AtlasTypePatch(DefaultMetadataService metadataService, TypeSystem typeSystem, String[] supportedActions) { - this.metadataService = metadataService; - this.typeSystem = typeSystem; - this.supportedActions = supportedActions; - } - - public final String[] getSupportedActions() { return supportedActions; } - - public abstract PatchResult applyPatch(PatchData patch); - - public enum PatchStatus { SUCCESS, FAILED, SKIPPED } - - public class PatchResult { - private String message; - private PatchStatus status; - - public PatchResult(String message, PatchStatus status) { - this.message = message; - this.status = status; - } - - public String getMessage() { return message; } - - public void setMessage(String message) { this.message = message; } - - public PatchStatus getStatus() { return status; } - - public void setStatus(PatchStatus status) { this.status = status; } - - } - - /** - * A class to capture patch content. - */ - public class PatchContent { - private PatchData[] patches; - - public PatchData[] getPatches() { - return patches; - } - } - - public static class PatchData { - private String action; - private String typeName; - private String applyToVersion; - private String updateToVersion; - private Map<String, String> params; - private AttributeDefinition[] attributeDefinitions; - - public PatchData(String action, String typeName, String applyToVersion, String updateToVersion, Map<String, String> params, AttributeDefinition[] attributeDefinitions) { - this.action = action; - this.typeName = typeName; - this.applyToVersion = applyToVersion; - this.updateToVersion = updateToVersion; - this.params = params; - this.attributeDefinitions = attributeDefinitions; - } - - public String getAction() { return action; } - - public String getTypeName() { return typeName; } - - public String getApplyToVersion() { return applyToVersion; } - - public String getUpdateToVersion() { return updateToVersion; } - - public Map<String, String> getParams() { return params; } - - public AttributeDefinition[] getAttributeDefinitions() { return attributeDefinitions; } - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 95c3dd9..69e8d12 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -20,7 +20,6 @@ package org.apache.atlas.services; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.inject.Provider; import org.apache.atlas.ApplicationProperties; @@ -29,7 +28,6 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.RequestContext; -import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; @@ -54,7 +52,6 @@ import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.persistence.ReferenceableInstance; -import org.apache.atlas.typesystem.types.AttributeDefinition; import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; @@ -83,9 +80,6 @@ import java.util.Map; import javax.inject.Inject; import javax.inject.Singleton; -import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS; -import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS; - /** @@ -103,35 +97,29 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang private final TypeSystem typeSystem; private final MetadataRepository repository; private final ITypeStore typeStore; - private IBootstrapTypesRegistrar typesRegistrar; private final Collection<TypesChangeListener> typeChangeListeners = new LinkedHashSet<>(); private final Collection<EntityChangeListener> entityChangeListeners = new LinkedHashSet<>(); - private boolean wasInitialized = false; - @Inject private EntityAuditRepository auditRepository; @Inject DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, - final IBootstrapTypesRegistrar typesRegistrar, final Collection<Provider<TypesChangeListener>> typeListenerProviders, final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache) throws AtlasException { - this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders, + this(repository, typeStore, typeListenerProviders, entityListenerProviders, TypeSystem.getInstance(), ApplicationProperties.get(), typeCache); } //for testing only public DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, - final IBootstrapTypesRegistrar typesRegistrar, final Collection<Provider<TypesChangeListener>> typeListenerProviders, final Collection<Provider<EntityChangeListener>> entityListenerProviders, final TypeSystem typeSystem, final Configuration configuration, TypeCache typeCache) throws AtlasException { this.typeStore = typeStore; - this.typesRegistrar = typesRegistrar; this.typeSystem = typeSystem; /** * Ideally a TypeCache implementation should have been injected in the TypeSystemProvider, @@ -163,70 +151,20 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang private void restoreTypeSystem() throws AtlasException { LOG.info("Restoring type system from the store"); + TypesDef typesDef = typeStore.restore(); - if (!wasInitialized) { - LOG.info("Initializing type system for the first time."); - typeSystem.defineTypes(typesDef); - - // restore types before creating super types - createSuperTypes(); - typesRegistrar.registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, this); - wasInitialized = true; - } else { - LOG.info("Type system was already initialized, refreshing cache."); - refreshCache(typesDef); - } + + refreshCache(typesDef); + LOG.info("Restored type system from the store"); } private void refreshCache(TypesDef typesDef) throws AtlasException { - TypeSystem.TransientTypeSystem transientTypeSystem - = typeSystem.createTransientTypeSystem(typesDef, true); - Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded(); - LOG.info("Number of types got from transient type system: " + typesAdded.size()); - typeSystem.commitTypes(typesAdded); - } - - @InterfaceAudience.Private - private void createSuperTypes() throws AtlasException { - HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil - .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(), - new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null)); - createType(referenceableType); - - HierarchicalTypeDefinition<ClassType> assetType = TypesUtil - .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(), - new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), - TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE), - new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)); - createType(assetType); - - HierarchicalTypeDefinition<ClassType> infraType = TypesUtil - .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE)); - createType(infraType); - - HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil - .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE)); - createType(datasetType); - - HierarchicalTypeDefinition<ClassType> processType = TypesUtil - .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE), - new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE), - Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE), - Multiplicity.OPTIONAL, false, null)); - createType(processType); - } - - private void createType(HierarchicalTypeDefinition<ClassType> type) throws AtlasException { - if (!typeSystem.isRegistered(type.typeName)) { - TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.of(type)); - createType(TypesSerialization.toJson(typesDef)); + if (typesDef != null && !typesDef.isEmpty()) { + TypeSystem.TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(typesDef, true); + Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded(); + LOG.info("Number of types got from transient type system: " + typesAdded.size()); + typeSystem.commitTypes(typesAdded); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java deleted file mode 100644 index fce5cb3..0000000 --- a/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.services; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.TypeSystem; - -public interface IBootstrapTypesRegistrar { - void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService) - throws AtlasException; -}
