Repository: incubator-atlas Updated Branches: refs/heads/master 710c17f55 -> 95083cb01
ATLAS-1242: update TypesResource API implementation to use new v2 TypesREST API Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/95083cb0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/95083cb0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/95083cb0 Branch: refs/heads/master Commit: 95083cb01fd2daeded1d55f6a2ce4791a933c974 Parents: 710c17f Author: Sarath Subramanian <[email protected]> Authored: Thu Nov 17 14:12:07 2016 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Nov 18 00:23:07 2016 -0800 ---------------------------------------------------------------------- .../java/org/apache/atlas/AtlasErrorCode.java | 1 + .../org/apache/atlas/model/SearchFilter.java | 2 +- release-log.txt | 1 + .../atlas/typesystem/types/utils/TypesUtil.java | 2 +- .../java/org/apache/atlas/util/RestUtils.java | 520 +++++++++++++++++++ .../atlas/web/resources/TypesResource.java | 117 +++-- .../web/resources/TypesJerseyResourceIT.java | 2 +- 7 files changed, 600 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 6a35d2b..8ee3458 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -54,6 +54,7 @@ public enum AtlasErrorCode { TYPE_ALREADY_EXISTS(409, "ATLAS4091E", "Given type {0} already exists"), TYPE_HAS_REFERENCES(409, "ATLAS4092E", "Given type {0} has references"), TYPE_MATCH_FAILED(409, "ATLAS4093E", "Given type {0} doesn't match {1}"), + INVALID_TYPE_DEFINITION(409, "ATLAS4094E", "Invalid type definition {0}"), INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"), INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/intg/src/main/java/org/apache/atlas/model/SearchFilter.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/SearchFilter.java b/intg/src/main/java/org/apache/atlas/model/SearchFilter.java index 64800a5..b5f6c3a 100644 --- a/intg/src/main/java/org/apache/atlas/model/SearchFilter.java +++ b/intg/src/main/java/org/apache/atlas/model/SearchFilter.java @@ -41,7 +41,7 @@ import javax.xml.bind.annotation.XmlRootElement; public class SearchFilter { public static final String PARAM_TYPE = "type"; public static final String PARAM_SUPERTYPE = "supertype"; - public static final String PARAM_NOT_SUPERTYPE = "notSupertype"; + public static final String PARAM_NOT_SUPERTYPE = "notsupertype"; /** * to specify whether the result should be sorted? If yes, whether asc or desc. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a3f4f20..6a17548 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1242 update TypesResource API implementation to use new v2 TypesREST API ATLAS-1306 bootstrap type-load ignores model file contents if a type in the file already exists ATLAS-1299 The project org.apache.atlas:atlas-hbase-client-shaded - build error (shwethags) ATLAS-1246 Upgrade versions of dependencies (shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java index 77cbd39..7923a36 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java @@ -82,7 +82,7 @@ public class TypesUtil { public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description, String version, ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { - return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs); + return new HierarchicalTypeDefinition<>(TraitType.class, name, description, version, superTypes, attrDefs); } public static StructTypeDefinition createStructTypeDef(String name, AttributeDefinition... attrDefs) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/webapp/src/main/java/org/apache/atlas/util/RestUtils.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/util/RestUtils.java b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java new file mode 100644 index 0000000..bc937e4 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java @@ -0,0 +1,520 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.util; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; +import org.apache.atlas.model.typedef.AtlasTypeDefHeader; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE; +import static org.apache.atlas.type.AtlasTypeUtil.isArrayType; + + +public final class RestUtils { + private RestUtils() {} + private static final Logger LOG = LoggerFactory.getLogger(RestUtils.class); + + public static TypesDef toTypesDef(AtlasEnumDef enumDef) { + TypesDef ret = null; + + if (enumDef != null) { + String enumName = enumDef.getName(); + String enumDesc = enumDef.getDescription(); + String enumVersion = enumDef.getTypeVersion(); + EnumValue[] enumValues = getEnumValues(enumDef.getElementDefs()); + + if (enumName != null && enumValues != null && enumValues.length > 0) { + EnumTypeDefinition enumTypeDef = new EnumTypeDefinition(enumName, enumDesc, enumVersion, enumValues); + + ret = TypesUtil.getTypesDef(ImmutableList.of(enumTypeDef), + ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); + } + } + + return ret; + } + + public static TypesDef toTypesDef(AtlasStructDef structDef, AtlasTypeRegistry registry) throws AtlasBaseException { + String typeName = structDef.getName(); + String typeDesc = structDef.getDescription(); + AttributeDefinition[] attributes = getAttributes(structDef, registry); + StructTypeDefinition structType = TypesUtil.createStructTypeDef(typeName, typeDesc, attributes); + + TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), + ImmutableList.of(structType), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); + + return ret; + } + + public static TypesDef toTypesDef(AtlasEntityDef entityDef, AtlasTypeRegistry registry) throws AtlasBaseException { + String typeName = entityDef.getName(); + String typeDesc = entityDef.getDescription(); + String typeVersion = entityDef.getTypeVersion(); + ImmutableSet superTypes = ImmutableSet.copyOf(entityDef.getSuperTypes()); + AttributeDefinition[] attributes = getAttributes(entityDef, registry); + + HierarchicalTypeDefinition<ClassType> classType = TypesUtil.createClassTypeDef(typeName, typeDesc, typeVersion, + superTypes, attributes); + TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), + ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(classType)); + + return ret; + } + + public static TypesDef toTypesDef(AtlasClassificationDef classifDef, AtlasTypeRegistry registry) throws AtlasBaseException { + String typeName = classifDef.getName(); + String typeDesc = classifDef.getDescription(); + String typeVersion = classifDef.getTypeVersion(); + ImmutableSet superTypes = ImmutableSet.copyOf(classifDef.getSuperTypes()); + AttributeDefinition[] attributes = getAttributes(classifDef, registry); + + HierarchicalTypeDefinition traitType = TypesUtil.createTraitTypeDef(typeName, typeDesc, typeVersion, superTypes, + attributes); + TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), + ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(traitType), + ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); + return ret; + } + + + + public static AtlasTypesDef toAtlasTypesDef(String typeDefinition, AtlasTypeRegistry registry) throws AtlasBaseException { + AtlasTypesDef ret = new AtlasTypesDef(); + + try { + if (StringUtils.isEmpty(typeDefinition)) { + throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition); + } + + TypesDef typesDef = TypesSerialization.fromJson(typeDefinition); + if (CollectionUtils.isNotEmpty(typesDef.enumTypesAsJavaList())) { + List<AtlasEnumDef> enumDefs = toAtlasEnumDefs(typesDef.enumTypesAsJavaList()); + ret.setEnumDefs(enumDefs); + } + + if (CollectionUtils.isNotEmpty(typesDef.structTypesAsJavaList())) { + List<AtlasStructDef> structDefs = toAtlasStructDefs(typesDef.structTypesAsJavaList()); + ret.setStructDefs(structDefs); + } + + if (CollectionUtils.isNotEmpty(typesDef.classTypesAsJavaList())) { + List<AtlasEntityDef> entityDefs = toAtlasEntityDefs(typesDef.classTypesAsJavaList(), registry); + ret.setEntityDefs(entityDefs); + } + + if (CollectionUtils.isNotEmpty(typesDef.traitTypesAsJavaList())) { + List<AtlasClassificationDef> classificationDefs = toAtlasClassificationDefs(typesDef.traitTypesAsJavaList()); + ret.setClassificationDefs(classificationDefs); + } + + } catch (Exception e) { + LOG.error("Invalid type definition = {}", typeDefinition, e); + throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition); + } + + return ret; + } + + public static ImmutableList<String> getTypeNames(List<AtlasTypeDefHeader> atlasTypesDefs) { + List<String> ret = new ArrayList<String>(); + if (CollectionUtils.isNotEmpty(atlasTypesDefs)) { + for (AtlasTypeDefHeader atlasTypesDef : atlasTypesDefs) { + ret.add(atlasTypesDef.getName()); + } + } + + return ImmutableList.copyOf(ret); + } + + public static List<String> getTypeNames(AtlasTypesDef typesDef) { + List<AtlasTypeDefHeader> atlasTypesDefs = AtlasTypeUtil.toTypeDefHeader(typesDef); + return getTypeNames(atlasTypesDefs); + } + + private static List<AtlasEnumDef> toAtlasEnumDefs(List<EnumTypeDefinition> enumTypeDefinitions) { + List<AtlasEnumDef> ret = new ArrayList<AtlasEnumDef>(); + + for (EnumTypeDefinition enumType : enumTypeDefinitions) { + AtlasEnumDef enumDef = new AtlasEnumDef(); + enumDef.setName(enumType.name); + enumDef.setDescription(enumType.description); + enumDef.setTypeVersion(enumType.version); + enumDef.setElementDefs(getAtlasEnumElementDefs(enumType.enumValues)); + + ret.add(enumDef); + } + + return ret; + } + + private static List<AtlasStructDef> toAtlasStructDefs(List<StructTypeDefinition> structTypeDefinitions) + throws AtlasBaseException { + List<AtlasStructDef> ret = new ArrayList<AtlasStructDef>(); + + for (StructTypeDefinition structType : structTypeDefinitions) { + AtlasStructDef structDef = new AtlasStructDef(); + List<AtlasAttributeDef> attrDefs = new ArrayList<AtlasAttributeDef>(); + + structDef.setName(structType.typeName); + structDef.setDescription(structType.typeDescription); + structDef.setTypeVersion(structType.typeVersion); + + AttributeDefinition[] attrDefinitions = structType.attributeDefinitions; + for (AttributeDefinition attrDefinition : attrDefinitions) { + attrDefs.add(toAtlasAttributeDef(attrDefinition)); + } + + structDef.setAttributeDefs(attrDefs); + ret.add(structDef); + } + + return ret; + } + + private static List<AtlasClassificationDef> toAtlasClassificationDefs(List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions) + throws AtlasBaseException { + List<AtlasClassificationDef> ret = new ArrayList<AtlasClassificationDef>(); + + for (HierarchicalTypeDefinition<TraitType> traitType : traitTypeDefinitions) { + AtlasClassificationDef classifDef = new AtlasClassificationDef(); + List<AtlasAttributeDef> attrDefs = new ArrayList<AtlasAttributeDef>(); + + classifDef.setName(traitType.typeName); + classifDef.setDescription(traitType.typeDescription); + classifDef.setTypeVersion(traitType.typeVersion); + classifDef.setSuperTypes(traitType.superTypes); + + AttributeDefinition[] attrDefinitions = traitType.attributeDefinitions; + for (AttributeDefinition attrDefinition : attrDefinitions) { + attrDefs.add(toAtlasAttributeDef(attrDefinition)); + } + + classifDef.setAttributeDefs(attrDefs); + ret.add(classifDef); + } + + return ret; + } + + private static List<AtlasEntityDef> toAtlasEntityDefs(List<HierarchicalTypeDefinition<ClassType>> classTypeDefinitions, + AtlasTypeRegistry registry) throws AtlasBaseException { + List<AtlasEntityDef> atlasEntityDefs = new ArrayList<AtlasEntityDef>(); + + for (HierarchicalTypeDefinition<ClassType> classType : classTypeDefinitions) { + List<AtlasAttributeDef> attrDefs = new ArrayList<AtlasAttributeDef>(); + AtlasEntityDef atlasEntityDef = new AtlasEntityDef(); + String classTypeDefName = classType.typeName; + + atlasEntityDef.setName(classTypeDefName); + atlasEntityDef.setDescription(classType.typeDescription); + atlasEntityDef.setTypeVersion(classType.typeVersion); + atlasEntityDef.setSuperTypes(classType.superTypes); + + AttributeDefinition[] attrDefinitions = classType.attributeDefinitions; + for (AttributeDefinition oldAttr : attrDefinitions) { + AtlasAttributeDef newAttr = toAtlasAttributeDef(oldAttr); + + // isComposite and reverseAttributeName applicable only for entities/classes. + if (oldAttr.isComposite) { + String attrType = oldAttr.dataTypeName; + attrType = isArrayType(attrType) ? getArrayTypeName(attrType) : attrType; + + if (!AtlasTypeUtil.isBuiltInType(attrType)) { + String refAttrName = null; + + // 1. Check if attribute datatype is present in payload definition, if present get the typeDefinition, + // check all its attributes and find attribute that matches with classTypeDefName and reverseAttributeName + HierarchicalTypeDefinition<ClassType> refType = findClassType(classTypeDefinitions, attrType); + if (refType != null) { + for (AttributeDefinition refAttr : refType.attributeDefinitions) { + String refAttrDataTypeName = refAttr.dataTypeName; + String refAttrRevAttrName = refAttr.reverseAttributeName; + + if (StringUtils.equals(refAttrDataTypeName, classTypeDefName) && + StringUtils.equals(refAttrRevAttrName, oldAttr.name)) { + refAttrName = refAttr.name; + break; + } + } + } + + // 2. Check if attribute present in typeRegistry. If present fetch all attributes associated with the type and + // check revAttrName equals base type attr name AND classTypeDefName equals attribute name + else { + if (registry.isRegisteredType(attrType)) { + AtlasType atlasType = registry.getType(attrType); + + if (isEntity(atlasType)) { + AtlasEntityType entityType = (AtlasEntityType) atlasType; + List<AtlasAttributeDef> atlasAttrDefs = entityType.getEntityDef().getAttributeDefs(); + + for (AtlasAttributeDef attrDef : atlasAttrDefs) { + boolean isForeignKey = entityType.isForeignKeyAttribute(attrDef.getName()); + + if (isForeignKey) { + AtlasType attribType = entityType.getAttributeType(attrDef.getName()); + + if (attribType.getTypeCategory() == TypeCategory.ARRAY) { + attribType = ((AtlasArrayType) attribType).getElementType(); + } + + if (attribType.getTypeCategory() == TypeCategory.ENTITY) { + String revAttrName = ((AtlasEntityType) attribType). + getMappedFromRefAttribute(entityType.getTypeName(), attrDef.getName()); + + if (StringUtils.equals(classTypeDefName , attrDef.getTypeName()) && + StringUtils.equals(oldAttr.name, revAttrName)) { + refAttrName = attrDef.getName(); + } + } + } + + } + } + } + } + + if (StringUtils.isNotBlank(refAttrName)) { // ex: hive_table.columns, hive_column.table + Map<String, Object> params = new HashMap<>(); + params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, refAttrName); + + newAttr.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_MAPPED_FROM_REF, params)); + } else { // ex: hive_table.partitionKeys, with no reverseAttribute-reference + newAttr.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY)); + } + } + + } else if (StringUtils.isNotEmpty(oldAttr.reverseAttributeName)) { + Map<String, Object> params = new HashMap<>(); + params.put(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE); + + newAttr.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, params)); + } + attrDefs.add(newAttr); + } + + atlasEntityDef.setAttributeDefs(attrDefs); + atlasEntityDefs.add(atlasEntityDef); + } + + return atlasEntityDefs; + } + + private static String getArrayTypeName(String attrType) { + String ret = null; + if (isArrayType(attrType)) { + Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(attrType); + if (typeNames.size() > 0) { + ret = typeNames.iterator().next(); + } + } + + return ret; + } + + private static List<AtlasEnumElementDef> getAtlasEnumElementDefs(EnumValue[] enums) { + List<AtlasEnumElementDef> ret = new ArrayList<AtlasEnumElementDef>(); + + for (EnumValue enumElem : enums) { + ret.add(new AtlasEnumElementDef(enumElem.value, null, enumElem.ordinal)); + } + + return ret; + } + + private static EnumValue[] getEnumValues(List<AtlasEnumElementDef> enumDefs) { + List<EnumValue> ret = new ArrayList<EnumValue>(); + + if (CollectionUtils.isNotEmpty(enumDefs)) { + for (AtlasEnumElementDef enumDef : enumDefs) { + if (enumDef != null) { + ret.add(new EnumValue(enumDef.getValue(), enumDef.getOrdinal())); + } + } + } + + return ret.toArray(new EnumValue[ret.size()]); + } + + private static AtlasAttributeDef toAtlasAttributeDef(AttributeDefinition attrDefinition) { + AtlasAttributeDef ret = new AtlasAttributeDef(); + + ret.setName(attrDefinition.name); + ret.setTypeName(attrDefinition.dataTypeName); + ret.setIsIndexable(attrDefinition.isIndexable); + ret.setIsUnique(attrDefinition.isUnique); + + // Multiplicity attribute mapping + Multiplicity multiplicity = attrDefinition.multiplicity; + int minCount = multiplicity.lower; + int maxCount = multiplicity.upper; + boolean isUnique = multiplicity.isUnique; + + if (minCount == 0) { + ret.setIsOptional(true); + ret.setValuesMinCount(0); + } else { + ret.setIsOptional(false); + ret.setValuesMinCount(minCount); + } + + if (maxCount < 2) { + ret.setCardinality(Cardinality.SINGLE); + ret.setValuesMaxCount(1); + } else { + if (!isUnique) { + ret.setCardinality(Cardinality.LIST); + } else { + ret.setCardinality(Cardinality.SET); + } + + ret.setValuesMaxCount(maxCount); + } + + return ret; + } + + private static AttributeDefinition[] getAttributes(AtlasStructDef structDef, AtlasTypeRegistry registry) throws AtlasBaseException { + List<AttributeDefinition> ret = new ArrayList<>(); + List<AtlasAttributeDef> attrDefs = structDef.getAttributeDefs(); + + if (CollectionUtils.isNotEmpty(attrDefs)) { + + for (AtlasAttributeDef attrDef : attrDefs) { + String name = attrDef.getName(); + String dataTypeName = attrDef.getTypeName(); + Boolean isUnique = attrDef.getIsUnique(); + Boolean isIndexable = attrDef.getIsIndexable(); + String reverseAttribName = null; + boolean isComposite; + + // Multiplicity mapping + final int lower; + final int upper; + + if (attrDef.getCardinality() == Cardinality.SINGLE) { + lower = attrDef.getIsOptional() ? 0 : 1; + upper = 1; + } else { + if(attrDef.getIsOptional()) { + lower = 0; + } else { + lower = attrDef.getValuesMinCount() < 1 ? 1 : attrDef.getValuesMinCount(); + } + + upper = attrDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attrDef.getValuesMaxCount(); + } + + Multiplicity multiplicity = new Multiplicity(lower, upper, Cardinality.SET.equals(attrDef.getCardinality())); + + // Constraint checks: + // 1. [ mappedFromRef -> isComposite ] + // 2. [ foreignKey(onDelete=cascade) -> reverseAttribute ] + AtlasStructType structType = (AtlasStructType) registry.getType(structDef.getName()); + boolean isForeignKey = structType.isForeignKeyAttribute(attrDef.getName()); + boolean isMappedFromRef = structType.isMappedFromRefAttribute(attrDef.getName()); + AtlasType attrType = structType.getAttributeType(attrDef.getName()); + + if (attrType != null && isForeignKey) { + if (attrType.getTypeCategory() == TypeCategory.ARRAY) { + attrType = ((AtlasArrayType) attrType).getElementType(); + } + + if (attrType.getTypeCategory() == TypeCategory.ENTITY) { + reverseAttribName = ((AtlasStructType) attrType). + getMappedFromRefAttribute(structType.getTypeName(), attrDef.getName()); + } + } + isComposite = isMappedFromRef || (isForeignKey && StringUtils.isBlank(reverseAttribName)); + ret.add(new AttributeDefinition(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable, reverseAttribName)); + } + } + + return ret.toArray(new AttributeDefinition[ret.size()]); + } + + private static HierarchicalTypeDefinition<ClassType> findClassType(List<HierarchicalTypeDefinition<ClassType>> classDefs, + String typeName) { + HierarchicalTypeDefinition<ClassType> ret = null; + + if (CollectionUtils.isNotEmpty(classDefs)) { + for (HierarchicalTypeDefinition<ClassType> classType : classDefs) { + if (classType.typeName.equalsIgnoreCase(typeName)) { + ret = classType; + } + } + } + + return ret; + } + + private static boolean isEntity(AtlasType type) { + return type.getTypeCategory() == TypeCategory.ENTITY; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java index 1eafc7d..e899fcf 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java @@ -19,15 +19,26 @@ package org.apache.atlas.web.resources; import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.core.ResourceContext; import org.apache.atlas.AtlasClient; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; -import org.apache.atlas.services.MetadataService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.exception.TypeExistsException; -import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.util.RestUtils; import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.rest.TypesREST; import org.apache.atlas.web.util.Servlets; -import org.apache.commons.lang.StringUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -49,9 +60,7 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * This class provides RESTful API for Types. @@ -64,17 +73,18 @@ import java.util.Map; @Path("types") @Singleton public class TypesResource { - private static final Logger LOG = LoggerFactory.getLogger(TypesResource.class); private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.TypesResource"); - - private final MetadataService metadataService; + private static AtlasTypeRegistry typeRegistry; @Inject - public TypesResource(MetadataService metadataService) { - this.metadataService = metadataService; + public TypesResource(AtlasTypeRegistry typeRegistry) { + this.typeRegistry = typeRegistry; } + @Context + private ResourceContext resourceContext; + /** * Submits a type definition corresponding to a given type representing a meta model of a * domain. Could represent things like Hive Database, Hive Table, etc. @@ -83,7 +93,10 @@ public class TypesResource { @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response submit(@Context HttpServletRequest request) { + TypesREST typesRest = resourceContext.getResource(TypesREST.class); AtlasPerfTracer perf = null; + JSONArray typesResponse = new JSONArray(); + try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.submit()"); @@ -92,12 +105,12 @@ public class TypesResource { final String typeDefinition = Servlets.getRequestPayload(request); LOG.info("Creating type with definition {} ", typeDefinition); - JSONObject typesJson = metadataService.createType(typeDefinition); - final JSONArray typesJsonArray = typesJson.getJSONArray(AtlasClient.TYPES); + AtlasTypesDef createTypesDef = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry); + AtlasTypesDef createdTypesDef = typesRest.createAtlasTypeDefs(createTypesDef); + List<String> typeNames = RestUtils.getTypeNames(createdTypesDef); - JSONArray typesResponse = new JSONArray(); - for (int i = 0; i < typesJsonArray.length(); i++) { - final String name = typesJsonArray.getString(i); + for (int i = 0; i < typeNames.size(); i++) { + final String name = typeNames.get(i); typesResponse.put(new JSONObject() {{ put(AtlasClient.NAME, name); }}); @@ -107,10 +120,10 @@ public class TypesResource { response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); response.put(AtlasClient.TYPES, typesResponse); return Response.status(ClientResponse.Status.CREATED).entity(response).build(); - } catch (TypeExistsException e) { + } catch (AtlasBaseException e) { LOG.error("Type already exists", e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT)); - } catch (AtlasException | IllegalArgumentException e) { + } catch (IllegalArgumentException e) { LOG.error("Unable to persist types", e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); } catch (Throwable e) { @@ -134,7 +147,9 @@ public class TypesResource { @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response update(@Context HttpServletRequest request) { + TypesREST typesRest = resourceContext.getResource(TypesREST.class); AtlasPerfTracer perf = null; + JSONArray typesResponse = new JSONArray(); try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.update()"); @@ -143,12 +158,12 @@ public class TypesResource { final String typeDefinition = Servlets.getRequestPayload(request); LOG.info("Updating type with definition {} ", typeDefinition); - JSONObject typesJson = metadataService.updateType(typeDefinition); - final JSONArray typesJsonArray = typesJson.getJSONArray(AtlasClient.TYPES); + AtlasTypesDef updateTypesDef = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry); + AtlasTypesDef updatedTypesDef = typesRest.updateAtlasTypeDefs(updateTypesDef); + List<String> typeNames = RestUtils.getTypeNames(updatedTypesDef); - JSONArray typesResponse = new JSONArray(); - for (int i = 0; i < typesJsonArray.length(); i++) { - final String name = typesJsonArray.getString(i); + for (int i = 0; i < typeNames.size(); i++) { + final String name = typeNames.get(i); typesResponse.put(new JSONObject() {{ put(AtlasClient.NAME, name); }}); @@ -181,21 +196,49 @@ public class TypesResource { @Path("{typeName}") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getDefinition(@Context HttpServletRequest request, @PathParam("typeName") String typeName) { + TypesREST typesRest = resourceContext.getResource(TypesREST.class); + JSONObject response = new JSONObject(); AtlasPerfTracer perf = null; + try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getDefinition(" + typeName + ")"); } - final String typeDefinition = metadataService.getTypeDefinition(typeName); + TypeCategory typeCategory = typeRegistry.getType(typeName).getTypeCategory(); + TypesDef typesDef = null; - JSONObject response = new JSONObject(); + if (typeCategory != null) { + switch (typeCategory) { + case ENUM: + AtlasEnumDef enumDef = typesRest.getEnumDefByName(typeName); + typesDef = RestUtils.toTypesDef(enumDef); + break; + case STRUCT: + AtlasStructDef structDef = typesRest.getStructDefByName(typeName); + typesDef = RestUtils.toTypesDef(structDef, typeRegistry); + break; + case ENTITY: + AtlasEntityDef entityDef = typesRest.getEntityDefByName(typeName); + typesDef = RestUtils.toTypesDef(entityDef, typeRegistry); + break; + case CLASSIFICATION: + AtlasClassificationDef classificationDef = typesRest.getClassificationDefByName(typeName); + typesDef = RestUtils.toTypesDef(classificationDef, typeRegistry); + break; + default: + typesDef = new TypesDef(); + break; + } + } + + final String typeDefinition = TypesSerialization.toJson(typesDef); response.put(AtlasClient.TYPENAME, typeName); response.put(AtlasClient.DEFINITION, new JSONObject(typeDefinition)); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); return Response.ok(response).build(); - } catch (AtlasException e) { + } catch (AtlasBaseException e) { LOG.error("Unable to get type definition for type {}", typeName, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); } catch (JSONException | IllegalArgumentException e) { @@ -225,29 +268,26 @@ public class TypesResource { @Produces(Servlets.JSON_MEDIA_TYPE) public Response getTypesByFilter(@Context HttpServletRequest request, @QueryParam("type") String typeCategory, @QueryParam("supertype") String supertype, - @QueryParam("notsupertype") String notsupertype) { + @QueryParam("notsupertype") String notsupertype) throws AtlasBaseException { + TypesREST typesRest = resourceContext.getResource(TypesREST.class); + JSONObject response = new JSONObject(); AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getTypesByFilter(" + typeCategory + ")"); } - Map<TypeCache.TYPE_FILTER, String> filterMap = new HashMap<>(); - addToFilterIfNotEmpty(filterMap, TypeCache.TYPE_FILTER.CATEGORY, typeCategory); - addToFilterIfNotEmpty(filterMap, TypeCache.TYPE_FILTER.SUPERTYPE, supertype); - addToFilterIfNotEmpty(filterMap, TypeCache.TYPE_FILTER.NOT_SUPERTYPE, notsupertype); - List<String> result = metadataService.getTypeNames(filterMap); + List<String> result = RestUtils.getTypeNames(typesRest.getTypeDefHeaders()); - JSONObject response = new JSONObject(); response.put(AtlasClient.RESULTS, new JSONArray(result)); response.put(AtlasClient.COUNT, result.size()); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); return Response.ok(response).build(); - } catch (IllegalArgumentException | AtlasException ie) { - LOG.error("Unsupported typeName while retrieving type list {}", typeCategory); + } catch (AtlasBaseException e) { + LOG.error("Given search filter did not yield any results"); throw new WebApplicationException( - Servlets.getErrorResponse(new Exception("Unsupported type " + typeCategory, ie), Response.Status.BAD_REQUEST)); + Servlets.getErrorResponse(new Exception("Given search filter did not yield any results "), Response.Status.BAD_REQUEST)); } catch (Throwable e) { LOG.error("Unable to get types list", e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); @@ -255,11 +295,4 @@ public class TypesResource { AtlasPerfTracer.log(perf); } } - - private void addToFilterIfNotEmpty(Map<TypeCache.TYPE_FILTER, String> filterMap, TypeCache.TYPE_FILTER filterType, - String filterValue) { - if (StringUtils.isNotEmpty(filterValue)) { - filterMap.put(filterType, filterValue); - } - } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java index 7d03689..b6dae4d 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java @@ -78,7 +78,7 @@ public class TypesJerseyResourceIT extends BaseResourceIT { @Test public void testSubmit() throws Exception { for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) { - String typesAsJSON = TypesSerialization.toJson(typeDefinition); + String typesAsJSON = TypesSerialization.toJson(typeDefinition, false); System.out.println("typesAsJSON = " + typesAsJSON); WebResource resource = service.path("api/atlas/types");
