http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java deleted file mode 100755 index 036d18d..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.persistence.Id; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; - -/** - * Given a IReferenceableInstance, a Walker will traverse the Object Graph - * reachable form the instance. It will invoke the process call on the provided NodeProcessor - * for each non-primitive attribute (Structs, Traits, References, Arrays of Non-Primitives, Maps - * of Non-Primitives) - */ -public class ObjectGraphWalker { - - final Queue<IReferenceableInstance> queue; - final TypeSystem typeSystem; - final NodeProcessor nodeProcessor; - Set<Id> processedIds; - - public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor) throws AtlasException { - this(typeSystem, nodeProcessor, (IReferenceableInstance) null); - } - - public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor, IReferenceableInstance start) - throws AtlasException { - this.typeSystem = typeSystem; - this.nodeProcessor = nodeProcessor; - queue = new LinkedList<>(); - processedIds = new HashSet<>(); - if (start != null) { - visitReferenceableInstance(start); - } - } - - public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor, - List<? extends IReferenceableInstance> roots) throws AtlasException { - this.typeSystem = typeSystem; - this.nodeProcessor = nodeProcessor; - queue = new LinkedList<>(); - processedIds = new HashSet<>(); - for (IReferenceableInstance r : roots) { - visitReferenceableInstance(r); - } - } - - public void walk() throws AtlasException { - while (!queue.isEmpty()) { - IReferenceableInstance r = queue.poll(); - if(r != null) { - processReferenceableInstance(r); - } - } - } - - public void addRoot(IReferenceableInstance root) { - visitReferenceableInstance(root); - } - - void traverseValue(IDataType dT, Object val) throws AtlasException { - if (val != null) { - if (dT.getTypeCategory() == DataTypes.TypeCategory.ARRAY) { - IDataType elemType = ((DataTypes.ArrayType) dT).getElemType(); - visitCollection(elemType, val); - } else if (dT.getTypeCategory() == DataTypes.TypeCategory.MAP) { - IDataType keyType = ((DataTypes.MapType) dT).getKeyType(); - IDataType valueType = ((DataTypes.MapType) dT).getValueType(); - visitMap(keyType, valueType, val); - } else if (dT.getTypeCategory() == DataTypes.TypeCategory.STRUCT - || dT.getTypeCategory() == DataTypes.TypeCategory.TRAIT) { - visitStruct(val); - } else if (dT.getTypeCategory() == DataTypes.TypeCategory.CLASS) { - visitReferenceableInstance(val); - } - } - } - - void visitMap(IDataType keyType, IDataType valueType, Object val) throws AtlasException { - if (keyType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE - && valueType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) { - return; - } - - if (val != null) { - Iterator<Map.Entry> it = null; - if (Map.class.isAssignableFrom(val.getClass())) { - it = ((Map) val).entrySet().iterator(); - ImmutableMap.Builder b = ImmutableMap.builder(); - while (it.hasNext()) { - Map.Entry e = it.next(); - traverseValue(keyType, e.getKey()); - traverseValue(valueType, e.getValue()); - } - } - } - } - - void visitCollection(IDataType elemType, Object val) throws AtlasException { - - if (elemType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) { - return; - } - - if (val != null) { - Iterator it = null; - if (val instanceof Collection) { - it = ((Collection) val).iterator(); - } else if (val instanceof Iterable) { - it = ((Iterable) val).iterator(); - } else if (val instanceof Iterator) { - it = (Iterator) val; - } - if (it != null) { - DataTypes.TypeCategory elemCategory = elemType.getTypeCategory(); - while (it.hasNext()) { - Object elem = it.next(); - traverseValue(elemType, elem); - } - } - } - } - - void visitStruct(Object val) throws AtlasException { - - if (val == null || !(val instanceof IStruct)) { - return; - } - - IStruct i = (IStruct) val; - - IConstructableType type = typeSystem.getDataType(IConstructableType.class, i.getTypeName()); - - for (Map.Entry<String, AttributeInfo> e : type.fieldMapping().fields.entrySet()) { - AttributeInfo aInfo = e.getValue(); - String attrName = e.getKey(); - if (aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE) { - Object aVal = i.get(attrName); - nodeProcessor.processNode(new Node(i, attrName, aInfo, aVal)); - traverseValue(aInfo.dataType(), aVal); - } - } - } - - void visitReferenceableInstance(Object val) { - - if (val == null || !(val instanceof IReferenceableInstance)) { - return; - } - - IReferenceableInstance ref = (IReferenceableInstance) val; - - if (!processedIds.contains(ref.getId())) { - processedIds.add(ref.getId()); - if (!(ref instanceof Id)) { - queue.add(ref); - } - } - } - - void processReferenceableInstance(IReferenceableInstance ref) throws AtlasException { - - nodeProcessor.processNode(new Node(ref, null, null, null)); - visitStruct(ref); - ImmutableList<String> traits = ref.getTraits(); - for (String trait : traits) { - visitStruct(ref.getTrait(trait)); - } - } - - public interface NodeProcessor { - - void processNode(Node nd) throws AtlasException; - } - - /** - * Represents a non-primitive value of an instance. - */ - public static class Node { - public final IStruct instance; - public final String attributeName; - public final AttributeInfo aInfo; - public final Object value; - - public Node(IStruct instance, String attributeName, AttributeInfo aInfo, Object value) { - this.instance = instance; - this.attributeName = attributeName; - this.aInfo = aInfo; - this.value = value; - } - - @Override - public String toString(){ - StringBuilder string = new StringBuilder().append(instance).append(aInfo).append(value); - return string.toString(); - } - } -}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java deleted file mode 100755 index 57f2517..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java +++ /dev/null @@ -1,280 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.security.MessageDigest; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.atlas.AtlasConstants; -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedStruct; - -public class StructType extends AbstractDataType<IStruct> implements IConstructableType<IStruct, ITypedStruct> { - - public final TypeSystem typeSystem; - public final FieldMapping fieldMapping; - public final Map<AttributeInfo, List<String>> infoToNameMap; - public final int numFields; - private final TypedStructHandler handler; - - protected StructType(TypeSystem typeSystem, String name, String description, int numFields) { - this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, numFields); - } - - protected StructType(TypeSystem typeSystem, String name, String description, String version, int numFields) { - super(name, description, version); - this.typeSystem = typeSystem; - this.fieldMapping = null; - infoToNameMap = null; - this.numFields = numFields; - this.handler = null; - } - - protected StructType(TypeSystem typeSystem, String name, String description, AttributeInfo... fields) - throws AtlasException { - this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, fields); - } - - protected StructType(TypeSystem typeSystem, String name, String description, String version, AttributeInfo... fields) - throws AtlasException { - super(name, description, version); - this.typeSystem = typeSystem; - this.fieldMapping = constructFieldMapping(fields); - infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping); - this.numFields = this.fieldMapping.fields.size(); - this.handler = new TypedStructHandler(this); - } - - public FieldMapping fieldMapping() { - return fieldMapping; - } - - /** - * Validate that current definition can be updated with the new definition - * @param newType - * @return true if the current definition can be updated with the new definition, else false - */ - @Override - public void validateUpdate(IDataType newType) throws TypeUpdateException { - super.validateUpdate(newType); - - StructType newStructType = (StructType) newType; - try { - TypeUtils.validateUpdate(fieldMapping, newStructType.fieldMapping); - } catch (TypeUpdateException e) { - throw new TypeUpdateException(newType, e); - } - } - - protected FieldMapping constructFieldMapping(AttributeInfo... fields) - throws AtlasException { - - Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<>(); - Map<String, Integer> fieldPos = new HashMap<>(); - Map<String, Integer> fieldNullPos = new HashMap<>(); - int numBools = 0; - int numBytes = 0; - int numShorts = 0; - int numInts = 0; - int numLongs = 0; - int numFloats = 0; - int numDoubles = 0; - int numBigInts = 0; - int numBigDecimals = 0; - int numDates = 0; - int numStrings = 0; - int numArrays = 0; - int numMaps = 0; - int numStructs = 0; - int numReferenceables = 0; - - for (AttributeInfo i : fields) { - if (fieldsMap.containsKey(i.name)) { - throw new AtlasException( - String.format("Struct defintion cannot contain multiple fields with the same " + "name %s", - i.name)); - } - fieldsMap.put(i.name, i); - fieldNullPos.put(i.name, fieldNullPos.size()); - if (i.dataType() == DataTypes.BOOLEAN_TYPE) { - fieldPos.put(i.name, numBools); - numBools++; - } else if (i.dataType() == DataTypes.BYTE_TYPE) { - fieldPos.put(i.name, numBytes); - numBytes++; - } else if (i.dataType() == DataTypes.SHORT_TYPE) { - fieldPos.put(i.name, numShorts); - numShorts++; - } else if (i.dataType() == DataTypes.INT_TYPE) { - fieldPos.put(i.name, numInts); - numInts++; - } else if (i.dataType() == DataTypes.LONG_TYPE) { - fieldPos.put(i.name, numLongs); - numLongs++; - } else if (i.dataType() == DataTypes.FLOAT_TYPE) { - fieldPos.put(i.name, numFloats); - numFloats++; - } else if (i.dataType() == DataTypes.DOUBLE_TYPE) { - fieldPos.put(i.name, numDoubles); - numDoubles++; - } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) { - fieldPos.put(i.name, numBigInts); - numBigInts++; - } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) { - fieldPos.put(i.name, numBigDecimals); - numBigDecimals++; - } else if (i.dataType() == DataTypes.DATE_TYPE) { - fieldPos.put(i.name, numDates); - numDates++; - } else if (i.dataType() == DataTypes.STRING_TYPE) { - fieldPos.put(i.name, numStrings); - numStrings++; - } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) { - fieldPos.put(i.name, numInts); - numInts++; - } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) { - fieldPos.put(i.name, numArrays); - numArrays++; - } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) { - fieldPos.put(i.name, numMaps); - numMaps++; - } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT - || i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) { - fieldPos.put(i.name, numStructs); - numStructs++; - } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) { - fieldPos.put(i.name, numReferenceables); - numReferenceables++; - } else { - throw new AtlasException(String.format("Unknown datatype %s", i.dataType())); - } - } - - return new FieldMapping(fieldsMap, fieldPos, fieldNullPos, numBools, numBytes, numShorts, numInts, numLongs, - numFloats, numDoubles, numBigInts, numBigDecimals, numDates, numStrings, numArrays, numMaps, numStructs, - numReferenceables); - } - - - @Override - public DataTypes.TypeCategory getTypeCategory() { - return DataTypes.TypeCategory.STRUCT; - } - - @Override - public ITypedStruct convert(Object val, Multiplicity m) throws AtlasException { - return handler.convert(val, m); - } - - public ITypedStruct createInstance() { - return handler.createInstance(); - } - - @Override - public void output(IStruct s, Appendable buf, String prefix, Set<IStruct> inProcess) throws AtlasException { - handler.output(s, buf, prefix, inProcess); - } - - @Override - public String toString() { - StringBuilder buf = new StringBuilder(); - try { - output(buf, new HashSet<String>()); - } - catch (AtlasException e) { - throw new RuntimeException(e); - } - return buf.toString(); - } - - @Override - public void output(Appendable buf, Set<String> typesInProcess) throws AtlasException { - - if (typesInProcess == null) { - typesInProcess = new HashSet<>(); - } - else if (typesInProcess.contains(name)) { - // Avoid infinite recursion on bi-directional reference attributes. - try { - buf.append(name); - } catch (IOException e) { - throw new AtlasException(e); - } - return; - } - - typesInProcess.add(name); - try { - buf.append(getClass().getSimpleName()); - buf.append("{name=").append(name); - buf.append(", description=").append(description); - buf.append(", fieldMapping.fields=["); - Iterator<AttributeInfo> it = fieldMapping.fields.values().iterator(); - while (it.hasNext()) { - AttributeInfo attrInfo = it.next(); - attrInfo.output(buf, typesInProcess); - if (it.hasNext()) { - buf.append(", "); - } - else { - buf.append(']'); - } - } - buf.append("}"); - } - catch(IOException e) { - throw new AtlasException(e); - } - finally { - typesInProcess.remove(name); - } - } - - @Override - public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { - if( !(val instanceof ITypedStruct)) { - throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct"); - } - digester.update(getName().getBytes(Charset.forName("UTF-8"))); - - if(fieldMapping.fields != null && val != null) { - IStruct typedValue = (IStruct) val; - for (AttributeInfo aInfo : fieldMapping.fields.values()) { - Object attrVal = typedValue.get(aInfo.name); - if(attrVal != null) { - aInfo.dataType().updateSignatureHash(digester, attrVal); - } - } - } - } - - public List<String> getNames(AttributeInfo info) { - return infoToNameMap.get(info); - } - -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java deleted file mode 100755 index 4f8695b..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import org.apache.atlas.AtlasConstants; -import org.apache.atlas.utils.ParamChecker; - -import java.util.Arrays; -import java.util.Objects; - -public class StructTypeDefinition { - - public final String typeName; - public final String typeDescription;//optional field - public final String typeVersion; - public final AttributeDefinition[] attributeDefinitions; - - protected StructTypeDefinition(String typeName, String typeDescription, boolean validate, - AttributeDefinition... attributeDefinitions) { - this(typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, validate, attributeDefinitions); - } - - protected StructTypeDefinition(String typeName, String typeDescription, String typeVersion, boolean validate, - AttributeDefinition... attributeDefinitions) { - this.typeName = ParamChecker.notEmpty(typeName, "Struct type name"); - this.typeDescription = typeDescription; - if (validate) { - ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions"); - } - this.attributeDefinitions = attributeDefinitions; - this.typeVersion = typeVersion; - } - - public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) { - this(typeName, null, AtlasConstants.DEFAULT_TYPE_VERSION, attributeDefinitions); - } - - public StructTypeDefinition(String typeName, String typeDescription, - AttributeDefinition[] attributeDefinitions) { - - this(typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, attributeDefinitions); - } - - public StructTypeDefinition(String typeName, String typeDescription, String typeVersion, - AttributeDefinition[] attributeDefinitions) { - this.typeName = ParamChecker.notEmpty(typeName, "Struct type name"); - this.typeDescription = typeDescription; - this.typeVersion = typeVersion; - this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions"); - } - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - StructTypeDefinition that = (StructTypeDefinition) o; - return Objects.equals(typeName, that.typeName) && - Objects.equals(typeDescription, that.typeDescription) && - Objects.equals(typeVersion, that.typeVersion) && - Arrays.equals(attributeDefinitions, that.attributeDefinitions); - } - - @Override - public int hashCode() { - return Objects.hash(typeName, typeDescription, typeVersion, attributeDefinitions); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java deleted file mode 100755 index bbb845a..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import com.google.common.collect.ImmutableSet; - -import org.apache.atlas.AtlasConstants; -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedStruct; - -import java.nio.charset.Charset; -import java.security.MessageDigest; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class TraitType extends HierarchicalType<TraitType, IStruct> - implements IConstructableType<IStruct, ITypedStruct> { - - public final Map<AttributeInfo, List<String>> infoToNameMap; - private final TypedStructHandler handler; - - TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, int numFields) { - this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTraits, numFields); - } - - TraitType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTraits, int numFields) { - super(typeSystem, TraitType.class, name, description, version, superTraits, numFields); - handler = null; - infoToNameMap = null; - } - - TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, AttributeInfo... fields) - throws AtlasException { - this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTraits, fields); - } - - TraitType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTraits, AttributeInfo... fields) - throws AtlasException { - super(typeSystem, TraitType.class, name, description, version, superTraits, fields); - handler = new TypedStructHandler(this); - infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping); - } - - @Override - public DataTypes.TypeCategory getTypeCategory() { - return DataTypes.TypeCategory.TRAIT; - } - - @Override - public ITypedStruct convert(Object val, Multiplicity m) throws AtlasException { - return handler.convert(val, m); - } - - public ITypedStruct createInstance() { - return handler.createInstance(); - } - - @Override - public void output(IStruct s, Appendable buf, String prefix, Set<IStruct> inProcess) throws AtlasException { - handler.output(s, buf, prefix, inProcess); - } - - @Override - public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { - if( !(val instanceof ITypedStruct)) { - throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct"); - } - digester.update(getName().getBytes(Charset.forName("UTF-8"))); - - if(fieldMapping.fields != null && val != null) { - IStruct typedValue = (IStruct) val; - for (AttributeInfo aInfo : fieldMapping.fields.values()) { - Object attrVal = typedValue.get(aInfo.name); - if(attrVal != null) { - aInfo.dataType().updateSignatureHash(digester, attrVal); - } - } - } - } - - @Override - public List<String> getNames(AttributeInfo info) { - return infoToNameMap.get(info); - } - -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java deleted file mode 100755 index 262f784..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ /dev/null @@ -1,821 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.AtlasException; -import org.apache.atlas.classification.InterfaceAudience; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.exception.TypeExistsException; -import org.apache.atlas.typesystem.exception.TypeNotFoundException; -import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; -import org.apache.atlas.typesystem.types.cache.TypeCache; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Singleton; -import java.lang.reflect.Constructor; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; - -@Singleton -@InterfaceAudience.Private -@Deprecated -public class TypeSystem { - private static final Logger LOG = LoggerFactory.getLogger(TypeSystem.class); - - private static final TypeSystem INSTANCE = new TypeSystem(); - private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() { - @Override - public SimpleDateFormat initialValue() { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - return dateFormat; - } - }; - - private TypeCache typeCache = new DefaultTypeCache(); - private IdType idType; - private Map<String, IDataType> coreTypes; - - public TypeSystem() { - initialize(); - } - - public static TypeSystem getInstance() { - return INSTANCE; - } - - /** - * This is only used for testing purposes. Not intended for public use. - */ - @InterfaceAudience.Private - public TypeSystem reset() { - - typeCache.clear(); // clear all entries in cache - initialize(); - - return this; - } - - public void setTypeCache(TypeCache typeCache) { - this.typeCache = typeCache; - } - - private void initialize() { - - coreTypes = new ConcurrentHashMap<>(); - - registerPrimitiveTypes(); - registerCoreTypes(); - } - - public ImmutableList<String> getCoreTypes() { - return ImmutableList.copyOf(coreTypes.keySet()); - } - - public ImmutableList<String> getTypeNames() throws AtlasException { - List<String> typeNames = new ArrayList<>(typeCache.getAllTypeNames()); - return ImmutableList.copyOf(typeNames); - } - - public ImmutableList<String> getTypeNamesByCategory(final DataTypes.TypeCategory typeCategory) throws AtlasException { - return getTypeNames(new HashMap<TypeCache.TYPE_FILTER, String>() {{ - put(TypeCache.TYPE_FILTER.CATEGORY, typeCategory.name()); - }}); - } - - public ImmutableList<String> getTypeNames(Map<TypeCache.TYPE_FILTER, String> filterMap) throws AtlasException { - return ImmutableList.copyOf(typeCache.getTypeNames(filterMap)); - } - - private void registerPrimitiveTypes() { - coreTypes.put(DataTypes.BOOLEAN_TYPE.getName(), DataTypes.BOOLEAN_TYPE); - coreTypes.put(DataTypes.BYTE_TYPE.getName(), DataTypes.BYTE_TYPE); - coreTypes.put(DataTypes.SHORT_TYPE.getName(), DataTypes.SHORT_TYPE); - coreTypes.put(DataTypes.INT_TYPE.getName(), DataTypes.INT_TYPE); - coreTypes.put(DataTypes.LONG_TYPE.getName(), DataTypes.LONG_TYPE); - coreTypes.put(DataTypes.FLOAT_TYPE.getName(), DataTypes.FLOAT_TYPE); - coreTypes.put(DataTypes.DOUBLE_TYPE.getName(), DataTypes.DOUBLE_TYPE); - coreTypes.put(DataTypes.BIGINTEGER_TYPE.getName(), DataTypes.BIGINTEGER_TYPE); - coreTypes.put(DataTypes.BIGDECIMAL_TYPE.getName(), DataTypes.BIGDECIMAL_TYPE); - coreTypes.put(DataTypes.DATE_TYPE.getName(), DataTypes.DATE_TYPE); - coreTypes.put(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE); - } - - /* - * The only core OOB type we will define is the Struct to represent the Identity of an Instance. - */ - private void registerCoreTypes() { - - idType = new IdType(); - coreTypes.put(idType.getStructType().getName(), idType.getStructType()); - } - - public IdType getIdType() { - return idType; - } - - public boolean isRegistered(String typeName) throws AtlasException { - return isCoreType(typeName) || typeCache.has(typeName); - } - - protected boolean isCoreType(String typeName) { - - return coreTypes.containsKey(typeName); - } - - public IDataType getDataType(String name) throws AtlasException { - if (isCoreType(name)) { - return coreTypes.get(name); - } - - if (typeCache.has(name)) { - return typeCache.get(name); - } - - /* - * is this an Array Type? - */ - String arrElemType = TypeUtils.parseAsArrayType(name); - if (arrElemType != null) { - IDataType dT = defineArrayType(getDataType(arrElemType)); - return dT; - } - - /* - * is this a Map Type? - */ - String[] mapType = TypeUtils.parseAsMapType(name); - if (mapType != null) { - IDataType dT = - defineMapType(getDataType(mapType[0]), getDataType(mapType[1])); - return dT; - } - - /* - * Invoke cache callback to possibly obtain type from other storage. - */ - IDataType dT = typeCache.onTypeFault(name); - if (dT != null) { - return dT; - } - - throw new TypeNotFoundException(String.format("Unknown datatype: %s", name)); - } - - public <T extends IDataType> T getDataType(Class<T> cls, String name) throws AtlasException { - try { - IDataType dt = getDataType(name); - return cls.cast(dt); - } catch (ClassCastException cce) { - throw new AtlasException(cce); - } - - } - - public StructType defineStructType(String name, boolean errorIfExists, AttributeDefinition... attrDefs) - throws AtlasException { - return defineStructType(name, null, errorIfExists, attrDefs); - } - - public StructType defineStructType(String name, String description, boolean errorIfExists, AttributeDefinition... attrDefs) - throws AtlasException { - StructTypeDefinition structDef = new StructTypeDefinition(name, description, attrDefs); - defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); - return getDataType(StructType.class, structDef.typeName); - } - - /** - * construct a temporary StructType for a Query Result. This is not registered in the - * typeSystem. - * The attributes in the typeDefinition can only reference permanent types. - * @param name struct type name - * @param attrDefs struct type definition - * @return temporary struct type - * @throws AtlasException - */ - public StructType defineQueryResultType(String name, Map<String, IDataType> tempTypes, - AttributeDefinition... attrDefs) throws AtlasException { - - AttributeInfo[] infos = new AttributeInfo[attrDefs.length]; - for (int i = 0; i < attrDefs.length; i++) { - infos[i] = new AttributeInfo(this, attrDefs[i], tempTypes); - } - - return new StructType(this, name, null, infos); - } - - public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef) throws AtlasException { - defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.of(traitDef), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); - return getDataType(TraitType.class, traitDef.typeName); - } - - public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef) throws AtlasException { - defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(classDef)); - return getDataType(ClassType.class, classDef.typeName); - } - - public Map<String, IDataType> defineTraitTypes(HierarchicalTypeDefinition<TraitType>... traitDefs) - throws AtlasException { - TransientTypeSystem transientTypes = - new TransientTypeSystem(ImmutableList.<EnumTypeDefinition>of(), - ImmutableList.<StructTypeDefinition>of(), ImmutableList.copyOf(traitDefs), - ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); - return transientTypes.defineTypes(false); - } - - public Map<String, IDataType> defineClassTypes(HierarchicalTypeDefinition<ClassType>... classDefs) - throws AtlasException { - TransientTypeSystem transientTypes = new TransientTypeSystem(ImmutableList.<EnumTypeDefinition>of(), - ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.copyOf(classDefs)); - return transientTypes.defineTypes(false); - } - - public Map<String, IDataType> updateTypes(TypesDef typesDef) throws AtlasException { - ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList()); - ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList()); - ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs = - ImmutableList.copyOf(typesDef.traitTypesAsJavaList()); - ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs = - ImmutableList.copyOf(typesDef.classTypesAsJavaList()); - - TransientTypeSystem transientTypes = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs); - return transientTypes.defineTypes(true); - } - - public Map<String, IDataType> defineTypes(TypesDef typesDef) throws AtlasException { - ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList()); - ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList()); - ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs = - ImmutableList.copyOf(typesDef.traitTypesAsJavaList()); - ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs = - ImmutableList.copyOf(typesDef.classTypesAsJavaList()); - - return defineTypes(enumDefs, structDefs, traitDefs, classDefs); - } - - public Map<String, IDataType> defineTypes(ImmutableList<EnumTypeDefinition> enumDefs, - ImmutableList<StructTypeDefinition> structDefs, - ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs, - ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) throws AtlasException { - TransientTypeSystem transientTypes = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs); - return transientTypes.defineTypes(false); - } - - public DataTypes.ArrayType defineArrayType(IDataType elemType) throws AtlasException { - assert elemType != null; - DataTypes.ArrayType dT = new DataTypes.ArrayType(elemType); - return dT; - } - - public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType) throws AtlasException { - assert keyType != null; - assert valueType != null; - DataTypes.MapType dT = new DataTypes.MapType(keyType, valueType); - return dT; - } - - public EnumType defineEnumType(String name, EnumValue... values) throws AtlasException { - return defineEnumType(new EnumTypeDefinition(name, values)); - } - - public EnumType defineEnumType(String name, String description, EnumValue... values) throws AtlasException { - return defineEnumType(new EnumTypeDefinition(name, description, values)); - } - - public EnumType defineEnumType(EnumTypeDefinition eDef) throws AtlasException { - assert eDef.name != null; - if (isRegistered(eDef.name)) { - throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name)); - } - - EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.version, eDef.enumValues); - typeCache.put(eT); - return eT; - } - - public SimpleDateFormat getDateFormat() { - return dateFormat.get(); - } - - public boolean allowNullsInCollections() { - return false; - } - - /** - * Create an instance of {@link TransientTypeSystem} with the types defined in the {@link TypesDef}. - * - * As part of this, a set of verifications are run on the types defined. - * @param typesDef The new list of types to be created or updated. - * @param isUpdate True, if types are updated, false otherwise. - * @return {@link TransientTypeSystem} that holds the newly added types. - * @throws AtlasException - */ - public TransientTypeSystem createTransientTypeSystem(TypesDef typesDef, boolean isUpdate) throws AtlasException { - ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList()); - ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList()); - ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs = - ImmutableList.copyOf(typesDef.traitTypesAsJavaList()); - ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs = - ImmutableList.copyOf(typesDef.classTypesAsJavaList()); - TransientTypeSystem transientTypeSystem = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs); - transientTypeSystem.verifyTypes(isUpdate); - return transientTypeSystem; - } - - /** - * Commit the given types to this {@link TypeSystem} instance. - * - * This step should be called only after the types have been committed to the backend stores successfully. - * @param typesAdded newly added types. - * @throws AtlasException - */ - public void commitTypes(Map<String, IDataType> typesAdded) throws AtlasException { - for (Map.Entry<String, IDataType> typeEntry : typesAdded.entrySet()) { - IDataType type = typeEntry.getValue(); - //Add/replace the new type in the typesystem - typeCache.put(type); - } - } - - public class TransientTypeSystem extends TypeSystem { - - final ImmutableList<StructTypeDefinition> structDefs; - final ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs; - final ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs; - private final ImmutableList<EnumTypeDefinition> enumDefs; - - Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<>(); - Map<String, HierarchicalTypeDefinition<TraitType>> traitNameToDefMap = new HashMap<>(); - Map<String, HierarchicalTypeDefinition<ClassType>> classNameToDefMap = new HashMap<>(); - - Map<String, IDataType> transientTypes = null; - - List<AttributeInfo> recursiveRefs = new ArrayList<>(); - List<DataTypes.ArrayType> recursiveArrayTypes = new ArrayList<>(); - List<DataTypes.MapType> recursiveMapTypes = new ArrayList<>(); - - - TransientTypeSystem(ImmutableList<EnumTypeDefinition> enumDefs, ImmutableList<StructTypeDefinition> structDefs, - ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs, - ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) { - this.enumDefs = enumDefs; - this.structDefs = structDefs; - this.traitDefs = traitDefs; - this.classDefs = classDefs; - transientTypes = new HashMap<>(); - } - - private IDataType dataType(String name) throws AtlasException { - if (transientTypes.containsKey(name)) { - return transientTypes.get(name); - } - - return TypeSystem.this.getDataType(IDataType.class, name); - } - - /* - * Step 1: - * - validate cannot redefine types - * - setup shallow Type instances to facilitate recursive type graphs - */ - private void validateAndSetupShallowTypes(boolean update) throws AtlasException { - for (EnumTypeDefinition eDef : enumDefs) { - assert eDef.name != null; - if (!update) { - if (TypeSystem.this.isRegistered(eDef.name)) { - throw new TypeExistsException(String.format("Redefinition of type %s is not supported", eDef.name)); - } else if (transientTypes.containsKey(eDef.name)) { - LOG.warn("Found duplicate definition of type {}. Ignoring..", eDef.name); - continue; - } - } - - EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.version, eDef.enumValues); - transientTypes.put(eDef.name, eT); - } - - for (StructTypeDefinition sDef : structDefs) { - assert sDef.typeName != null; - if (!update) { - if (TypeSystem.this.isRegistered(sDef.typeName)) { - throw new TypeExistsException(String.format("Redefinition of type %s is not supported", sDef.typeName)); - } else if (transientTypes.containsKey(sDef.typeName)) { - LOG.warn("Found duplicate definition of type {}. Ignoring..", sDef.typeName); - continue; - } - } - - StructType sT = new StructType(this, sDef.typeName, sDef.typeDescription, sDef.typeVersion, sDef.attributeDefinitions.length); - structNameToDefMap.put(sDef.typeName, sDef); - transientTypes.put(sDef.typeName, sT); - } - - for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) { - assert traitDef.typeName != null; - if (!update) { - if (TypeSystem.this.isRegistered(traitDef.typeName)) { - throw new TypeExistsException(String.format("Redefinition of type %s is not supported", traitDef.typeName)); - } else if (transientTypes.containsKey(traitDef.typeName)) { - LOG.warn("Found duplicate definition of type {}. Ignoring..", traitDef.typeName); - continue; - } - } - - TraitType tT = new TraitType(this, traitDef.typeName, traitDef.typeDescription, traitDef.typeVersion, traitDef.superTypes, - traitDef.attributeDefinitions.length); - traitNameToDefMap.put(traitDef.typeName, traitDef); - transientTypes.put(traitDef.typeName, tT); - } - - for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) { - assert classDef.typeName != null; - if (!update) { - if (TypeSystem.this.isRegistered(classDef.typeName)) { - throw new TypeExistsException(String.format("Redefinition of type %s is not supported", classDef.typeName)); - } else if (transientTypes.containsKey(classDef.typeName)) { - LOG.warn("Found duplicate definition of type {}. Ignoring..", classDef.typeName); - continue; - } - } - - ClassType cT = new ClassType(this, classDef.typeName, classDef.typeDescription, classDef.typeVersion, classDef.superTypes, - classDef.attributeDefinitions.length); - classNameToDefMap.put(classDef.typeName, classDef); - transientTypes.put(classDef.typeName, cT); - } - } - - @Override - public boolean isRegistered(String typeName) throws AtlasException { - return transientTypes.containsKey(typeName) || TypeSystem.this.isRegistered(typeName); - } - - private <U extends HierarchicalType> void validateSuperTypes(Class<U> cls, HierarchicalTypeDefinition<U> def) - throws AtlasException { - for (String superTypeName : def.superTypes) { - - IDataType dT = dataType(superTypeName); - - if (dT == null) { - throw new AtlasException( - String.format("Unknown superType %s in definition of type %s", superTypeName, - def.typeName)); - } - - if (!cls.isAssignableFrom(dT.getClass())) { - throw new AtlasException( - String.format("SuperType %s must be a %s, in definition of type %s", superTypeName, - cls.getName(), def.typeName)); - } - } - } - - /* - * Step 2: - * - for Hierarchical Types, validate SuperTypes. - * - for each Hierarchical Type setup their SuperTypes Graph - */ - private void validateAndSetupSuperTypes() throws AtlasException { - for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) { - validateSuperTypes(TraitType.class, traitDef); - TraitType traitType = getDataType(TraitType.class, traitDef.typeName); - traitType.setupSuperTypesGraph(); - } - - for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) { - validateSuperTypes(ClassType.class, classDef); - ClassType classType = getDataType(ClassType.class, classDef.typeName); - classType.setupSuperTypesGraph(); - } - } - - private AttributeInfo constructAttributeInfo(AttributeDefinition attrDef) throws AtlasException { - AttributeInfo info = new AttributeInfo(this, attrDef, null); - if (transientTypes.keySet().contains(attrDef.dataTypeName)) { - recursiveRefs.add(info); - } - if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) { - DataTypes.ArrayType arrType = (DataTypes.ArrayType) info.dataType(); - if (transientTypes.keySet().contains(arrType.getElemType().getName())) { - recursiveArrayTypes.add(arrType); - } - } - if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) { - DataTypes.MapType mapType = (DataTypes.MapType) info.dataType(); - if (transientTypes.keySet().contains(mapType.getKeyType().getName())) { - recursiveMapTypes.add(mapType); - } else if (transientTypes.keySet().contains(mapType.getValueType().getName())) { - recursiveMapTypes.add(mapType); - } - } - - if (info.multiplicity.upper > 1 && !(info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP - || info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY)) { - throw new AtlasException( - String.format("A multiplicty of more than one requires a collection type for attribute '%s'", - info.name)); - } - - return info; - } - - private StructType constructStructureType(StructTypeDefinition def) throws AtlasException { - AttributeInfo[] infos = new AttributeInfo[def.attributeDefinitions.length]; - for (int i = 0; i < def.attributeDefinitions.length; i++) { - infos[i] = constructAttributeInfo(def.attributeDefinitions[i]); - } - - StructType type = new StructType(this, def.typeName, def.typeDescription, def.typeVersion, infos); - transientTypes.put(def.typeName, type); - return type; - } - - private <U extends HierarchicalType> U constructHierarchicalType(Class<U> cls, - HierarchicalTypeDefinition<U> def) throws AtlasException { - AttributeInfo[] infos = new AttributeInfo[def.attributeDefinitions.length]; - for (int i = 0; i < def.attributeDefinitions.length; i++) { - infos[i] = constructAttributeInfo(def.attributeDefinitions[i]); - } - - try { - Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, String.class, ImmutableSet.class, - AttributeInfo[].class); - U type = cons.newInstance(this, def.typeName, def.typeDescription, def.typeVersion, def.superTypes, infos); - transientTypes.put(def.typeName, type); - return type; - } catch (Exception e) { - e.printStackTrace(); - throw new AtlasException(String.format("Cannot construct Type of MetaType %s - %s", cls.getName(), def.typeName), e); - } - } - - /* - * Step 3: - * - Order Hierarchical Types in order of SuperType before SubType. - * - Construct all the Types - */ - private void orderAndConstructTypes() throws AtlasException { - - List<TraitType> traitTypes = new ArrayList<>(); - for (String traitTypeName : traitNameToDefMap.keySet()) { - traitTypes.add(getDataType(TraitType.class, traitTypeName)); - } - traitTypes = HierarchicalTypeDependencySorter.sortTypes(traitTypes); - - List<ClassType> classTypes = new ArrayList<>(); - for (String classTypeName : classNameToDefMap.keySet()) { - classTypes.add(getDataType(ClassType.class, classTypeName)); - } - classTypes = HierarchicalTypeDependencySorter.sortTypes(classTypes); - - for (StructTypeDefinition structDef : structDefs) { - constructStructureType(structDef); - } - - for (TraitType traitType : traitTypes) { - constructHierarchicalType(TraitType.class, traitNameToDefMap.get(traitType.getName())); - } - - for (ClassType classType : classTypes) { - constructHierarchicalType(ClassType.class, classNameToDefMap.get(classType.getName())); - } - } - - /* - * Step 4: - * - fix up references in recursive AttrInfo and recursive Collection Types. - */ - private void setupRecursiveTypes() throws AtlasException { - for (AttributeInfo info : recursiveRefs) { - info.setDataType(dataType(info.dataType().getName())); - } - for (DataTypes.ArrayType arrType : recursiveArrayTypes) { - arrType.setElemType(dataType(arrType.getElemType().getName())); - } - for (DataTypes.MapType mapType : recursiveMapTypes) { - mapType.setKeyType(dataType(mapType.getKeyType().getName())); - mapType.setValueType(dataType(mapType.getValueType().getName())); - } - } - - /** - * Step 5: - * - Validate that the update can be done - */ - private void validateUpdateIsPossible() throws TypeUpdateException, AtlasException { - //If the type is modified, validate that update can be done - for (IDataType newType : transientTypes.values()) { - IDataType oldType = null; - try { - oldType = TypeSystem.this.getDataType(IDataType.class, newType.getName()); - } catch (TypeNotFoundException e) { - LOG.debug(String.format("No existing type %s found - update OK", newType.getName())); - } - if (oldType != null) { - oldType.validateUpdate(newType); - } - } - } - - Map<String, IDataType> defineTypes(boolean update) throws AtlasException { - verifyTypes(update); - Map<String, IDataType> typesAdded = getTypesAdded(); - commitTypes(typesAdded); - return typesAdded; - } - - @Override - public ImmutableList<String> getTypeNames() throws AtlasException { - Set<String> typeNames = transientTypes.keySet(); - typeNames.addAll(TypeSystem.this.getTypeNames()); - return ImmutableList.copyOf(typeNames); - } - - //get from transient types. Else, from main type system - @Override - public IDataType getDataType(String name) throws AtlasException { - - if (transientTypes != null) { - if (transientTypes.containsKey(name)) { - return transientTypes.get(name); - } - - /* - * is this an Array Type? - */ - String arrElemType = TypeUtils.parseAsArrayType(name); - if (arrElemType != null) { - IDataType dT = defineArrayType(getDataType(IDataType.class, arrElemType)); - return dT; - } - - /* - * is this a Map Type? - */ - String[] mapType = TypeUtils.parseAsMapType(name); - if (mapType != null) { - IDataType dT = - defineMapType(getDataType(IDataType.class, mapType[0]), getDataType(IDataType.class, mapType[1])); - return dT; - } - } - - return TypeSystem.this.getDataType(name); - } - - @Override - public StructType defineStructType(String name, boolean errorIfExists, AttributeDefinition... attrDefs) - throws AtlasException { - throw new AtlasException("Internal Error: define type called on TransientTypeSystem"); - } - - @Override - public TraitType defineTraitType(HierarchicalTypeDefinition traitDef) throws AtlasException { - throw new AtlasException("Internal Error: define type called on TransientTypeSystem"); - } - - @Override - public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef) throws AtlasException { - throw new AtlasException("Internal Error: define type called on TransientTypeSystem"); - } - - @Override - public Map<String, IDataType> defineTypes(ImmutableList<EnumTypeDefinition> enumDefs, - ImmutableList<StructTypeDefinition> structDefs, - ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs, - ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) throws AtlasException { - throw new AtlasException("Internal Error: define type called on TransientTypeSystem"); - } - - @Override - public DataTypes.ArrayType defineArrayType(IDataType elemType) throws AtlasException { - return super.defineArrayType(elemType); - } - - @Override - public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType) throws AtlasException { - return super.defineMapType(keyType, valueType); - } - - void verifyTypes(boolean isUpdate) throws AtlasException { - validateAndSetupShallowTypes(isUpdate); - validateAndSetupSuperTypes(); - orderAndConstructTypes(); - setupRecursiveTypes(); - if (isUpdate) { - validateUpdateIsPossible(); - } - } - - @Override - public void commitTypes(Map<String, IDataType> typesAdded) throws AtlasException { - TypeSystem.this.commitTypes(typesAdded); - } - - public Map<String, IDataType> getTypesAdded() { - return new HashMap<>(transientTypes); - } - - /** - * The core types do not change and they are registered - * once in the main type system. - */ - @Override - public ImmutableList<String> getCoreTypes() { - return TypeSystem.this.getCoreTypes(); - } - } - - public class IdType { - private static final String ID_ATTRNAME = "guid"; - private static final String TYPENAME_ATTRNAME = "typeName"; - private static final String STATE_ATTRNAME = "state"; - private static final String VERSION_ATTRNAME = "version"; - private static final String TYP_NAME = "__IdType"; - - private StructType type; - - private IdType() { - AttributeDefinition idAttr = - new AttributeDefinition(ID_ATTRNAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null); - AttributeDefinition typNmAttr = - new AttributeDefinition(TYPENAME_ATTRNAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, - false, null); - AttributeDefinition stateAttr = - new AttributeDefinition(STATE_ATTRNAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, - false, null); - AttributeDefinition versionAttr = - new AttributeDefinition(VERSION_ATTRNAME, DataTypes.INT_TYPE.getName(), Multiplicity.REQUIRED, - false, null); - try { - AttributeInfo[] infos = new AttributeInfo[4]; - infos[0] = new AttributeInfo(TypeSystem.this, idAttr, null); - infos[1] = new AttributeInfo(TypeSystem.this, typNmAttr, null); - infos[2] = new AttributeInfo(TypeSystem.this, stateAttr, null); - infos[3] = new AttributeInfo(TypeSystem.this, versionAttr, null); - - type = new StructType(TypeSystem.this, TYP_NAME, null, infos); - } catch (AtlasException me) { - throw new RuntimeException(me); - } - } - - public StructType getStructType() { - return type; - } - - public String getName() { - return TYP_NAME; - } - - public String idAttrName() { - return ID_ATTRNAME; - } - - public String typeNameAttrName() { - return TYPENAME_ATTRNAME; - } - - public String stateAttrName() { - return STATE_ATTRNAME; - } - - public String versionAttrName() { - return VERSION_ATTRNAME; - } - } - - public static final String ID_STRUCT_ID_ATTRNAME = IdType.ID_ATTRNAME; - public static final String ID_STRUCT_TYP_NAME = IdType.TYP_NAME; -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java deleted file mode 100644 index 33d1cb5..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import org.apache.atlas.AtlasException; - -public class TypeUpdateException extends AtlasException { - public TypeUpdateException(IDataType newType) { - super(newType.getName() + " can't be updated"); - } - - public TypeUpdateException(IDataType newType, Exception e) { - super(newType.getName() + " can't be updated - " + e.getMessage(), e); - } - - public TypeUpdateException(String message) { - super(message); - } - - public TypeUpdateException(IDataType newType, String message) { - super(newType.getName() + " can't be updated - " + message); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java deleted file mode 100755 index 6a14dc4..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import com.google.common.collect.ImmutableList; -import org.apache.atlas.AtlasException; - -import java.io.IOException; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class TypeUtils { - - public static final String NAME_REGEX = "[a-zA-z][a-zA-Z0-9_]*"; - public static final Pattern NAME_PATTERN = Pattern.compile(NAME_REGEX); - public static final Pattern ARRAY_TYPE_NAME_PATTERN = Pattern.compile(String.format("array<(%s)>", NAME_REGEX)); - public static final Pattern MAP_TYPE_NAME_PATTERN = - Pattern.compile(String.format("map<(%s),(%s)>", NAME_REGEX, NAME_REGEX)); - - public static void outputVal(String val, Appendable buf, String prefix) throws AtlasException { - try { - buf.append(prefix).append(val); - } catch (IOException ie) { - throw new AtlasException(ie); - } - } - - public static String parseAsArrayType(String typeName) { - Matcher m = ARRAY_TYPE_NAME_PATTERN.matcher(typeName); - return m.matches() ? m.group(1) : null; - } - - public static String[] parseAsMapType(String typeName) { - Matcher m = MAP_TYPE_NAME_PATTERN.matcher(typeName); - return m.matches() ? new String[]{m.group(1), m.group(2)} : null; - } - - public static Map<AttributeInfo, List<String>> buildAttrInfoToNameMap(FieldMapping f) { - Map<AttributeInfo, List<String>> b = new HashMap(); - for (Map.Entry<String, AttributeInfo> e : f.fields.entrySet()) { - List<String> names = b.get(e.getValue()); - if (names == null) { - names = new ArrayList<>(); - b.put(e.getValue(), names); - } - names.add(e.getKey()); - } - return b; - } - - public static class Pair<L, R> { - public L left; - public R right; - - public Pair(L left, R right) { - this.left = left; - this.right = right; - } - - public static <L, R> Pair<L, R> of(L left, R right) { - return new Pair<>(left, right); - } - - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - Pair p = (Pair)o; - - return Objects.equals(left, p.left) && Objects.equals(right, p.right); - } - - public int hashCode() { return Objects.hash(left, right); } - } - - /** - * Validates that the old field mapping can be replaced with new field mapping - * @param oldFieldMapping - * @param newFieldMapping - */ - public static void validateUpdate(FieldMapping oldFieldMapping, FieldMapping newFieldMapping) - throws TypeUpdateException { - Map<String, AttributeInfo> newFields = newFieldMapping.fields; - for (AttributeInfo attribute : oldFieldMapping.fields.values()) { - if (newFields.containsKey(attribute.name)) { - AttributeInfo newAttribute = newFields.get(attribute.name); - //If old attribute is also in new definition, only allowed change is multiplicity change from REQUIRED to OPTIONAL - if (!newAttribute.equals(attribute)) { - if (attribute.multiplicity == Multiplicity.REQUIRED - && newAttribute.multiplicity == Multiplicity.OPTIONAL) { - continue; - } else { - throw new TypeUpdateException("Attribute " + attribute.name + " can't be updated"); - } - } - - } else { - //If old attribute is missing in new definition, return false as attributes can't be deleted - throw new TypeUpdateException("Old Attribute " + attribute.name + " is missing"); - } - } - - //Only new attributes - Set<String> newAttributes = new HashSet<>(ImmutableList.copyOf(newFields.keySet())); - newAttributes.removeAll(oldFieldMapping.fields.keySet()); - for (String attributeName : newAttributes) { - AttributeInfo newAttribute = newFields.get(attributeName); - //New required attribute can't be added - if (newAttribute.multiplicity == Multiplicity.REQUIRED) { - throw new TypeUpdateException("Can't add required attribute " + attributeName); - } - } - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java deleted file mode 100755 index 9afa873..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedStruct; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.persistence.ReferenceableInstance; -import org.apache.atlas.typesystem.persistence.StructInstance; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.Date; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -public class TypedStructHandler { - - private final IConstructableType<IStruct, ITypedStruct> structType; - private final FieldMapping fieldMapping; - - public TypedStructHandler(IConstructableType<IStruct, ITypedStruct> structType) { - this.structType = structType; - fieldMapping = structType.fieldMapping(); - } - - public ITypedStruct convert(Object val, Multiplicity m) throws AtlasException { - if (val != null) { - if (val instanceof ITypedStruct) { - ITypedStruct ts = (ITypedStruct) val; - if (!Objects.equals(ts.getTypeName(), structType.getName())) { - throw new ValueConversionException(structType, val); - } - return ts; - } else if (val instanceof Struct) { - Struct s = (Struct) val; - if (!s.typeName.equals(structType.getName())) { - throw new ValueConversionException(structType, val); - } - ITypedStruct ts = createInstance(); - for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) { - String attrKey = e.getKey(); - AttributeInfo i = e.getValue(); - Object aVal = s.get(attrKey); - try { - ts.set(attrKey, aVal); - } catch (ValueConversionException ve) { - throw new ValueConversionException(structType, val, ve); - } - } - return ts; - } else if (val instanceof StructInstance && Objects.equals(((StructInstance) val).getTypeName(), structType.getName())) { - return (StructInstance) val; - } else { - throw new ValueConversionException(structType, val); - } - } - if (!m.nullAllowed()) { - throw new ValueConversionException.NullConversionException(m); - } - return null; - } - - public DataTypes.TypeCategory getTypeCategory() { - return DataTypes.TypeCategory.STRUCT; - } - - public ITypedStruct createInstance() { - return new StructInstance(structType.getName(), fieldMapping, new boolean[fieldMapping.fields.size()], - new boolean[fieldMapping.fields.size()], - fieldMapping.numBools == 0 ? null : new boolean[fieldMapping.numBools], - fieldMapping.numBytes == 0 ? null : new byte[fieldMapping.numBytes], - fieldMapping.numShorts == 0 ? null : new short[fieldMapping.numShorts], - fieldMapping.numInts == 0 ? null : new int[fieldMapping.numInts], - fieldMapping.numLongs == 0 ? null : new long[fieldMapping.numLongs], - fieldMapping.numFloats == 0 ? null : new float[fieldMapping.numFloats], - fieldMapping.numDoubles == 0 ? null : new double[fieldMapping.numDoubles], - fieldMapping.numBigDecimals == 0 ? null : new BigDecimal[fieldMapping.numBigDecimals], - fieldMapping.numBigInts == 0 ? null : new BigInteger[fieldMapping.numBigInts], - fieldMapping.numDates == 0 ? null : new Date[fieldMapping.numDates], - fieldMapping.numStrings == 0 ? null : new String[fieldMapping.numStrings], - fieldMapping.numArrays == 0 ? null : new ImmutableList[fieldMapping.numArrays], - fieldMapping.numMaps == 0 ? null : new ImmutableMap[fieldMapping.numMaps], - fieldMapping.numStructs == 0 ? null : new StructInstance[fieldMapping.numStructs], - fieldMapping.numReferenceables == 0 ? null : new ReferenceableInstance[fieldMapping.numReferenceables], - fieldMapping.numReferenceables == 0 ? null : new Id[fieldMapping.numReferenceables]); - } - - public void output(IStruct s, Appendable buf, String prefix, Set<IStruct> inProcess) throws AtlasException { - fieldMapping.output(s, buf, prefix, inProcess); - } - -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java deleted file mode 100755 index f756135..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import org.apache.atlas.AtlasException; - -public class ValueConversionException extends AtlasException { - - public ValueConversionException(IDataType typ, Object val) { - this(typ, val, (Throwable) null); - } - - public ValueConversionException(IDataType typ, Object val, Throwable t) { - super(String.format("Cannot convert value '%s' to datatype %s", val.toString(), typ.getName()), t); - } - - public ValueConversionException(IDataType typ, Object val, String msg) { - super(String - .format("Cannot convert value '%s' to datatype %s because: %s", val.toString(), typ.getName(), msg)); - } - - public ValueConversionException(String typeName, Object val, String msg) { - super(String.format("Cannot convert value '%s' to datatype %s because: %s", val.toString(), typeName, msg)); - } - - protected ValueConversionException(String msg) { - super(msg); - } - - protected ValueConversionException(String msg, Exception e) { - super(msg, e); - } - - public static class NullConversionException extends ValueConversionException { - public NullConversionException(Multiplicity m) { - super(String.format("Null value not allowed for multiplicty %s", m)); - } - - public NullConversionException(Multiplicity m, String msg){ - super(String.format("Null value not allowed for multiplicty %s . Message %s", m, msg)); - } - - public NullConversionException(String msg, Exception e) { - super(msg, e); - } - } -}
