[3/5] atlas git commit: ATLAS-2251: relocated legacy classes in intg module
http://git-wip-us.apache.org/repos/asf/atlas/blob/eacf8513/intg/src/main/java/org/apache/atlas/v1/model/instance/AtlasSystemAttributes.java -- diff --git a/intg/src/main/java/org/apache/atlas/v1/model/instance/AtlasSystemAttributes.java b/intg/src/main/java/org/apache/atlas/v1/model/instance/AtlasSystemAttributes.java new file mode 100644 index 000..43eca0b --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/v1/model/instance/AtlasSystemAttributes.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.v1.model.instance; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Date; +import java.util.Objects; + +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; + + +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasSystemAttributes implements Serializable { +private static final long serialVersionUID = 1L; + +private String createdBy; +private String modifiedBy; +private Date createdTime; +private Date modifiedTime; + + +public AtlasSystemAttributes() { +} + +public AtlasSystemAttributes(AtlasSystemAttributes that) { +if (that != null) { +this.createdBy= that.createdBy; +this.modifiedBy = that.modifiedBy; +this.createdTime = that.createdTime; +this.modifiedTime = that.modifiedTime; +} +} + +public AtlasSystemAttributes(String createdBy, String modifiedBy, Date createdTime, Date modifiedTime){ +this.createdBy= createdBy; +this.modifiedBy = modifiedBy; +this.createdTime = createdTime; +this.modifiedTime = modifiedTime; +} + +public String getCreatedBy(){ +return createdBy; +} + +public void setCreatedBy(String createdBy) { +this.createdBy = createdBy; +} + +public String getModifiedBy(){ +return modifiedBy; +} + +public void setModifiedBy(String modifiedBy) { +this.modifiedBy = modifiedBy; +} + +public Date getCreatedTime(){ +return createdTime; +} + +public void setCreatedTime(Date createdTime) { +this.createdTime = createdTime; +} + +public Date getModifiedTime(){ +return modifiedTime; +} + +public void setModifiedTime(Date modifiedTime) { +this.modifiedTime = modifiedTime; +} + + +@Override +public boolean equals(Object o) { +if (this == o) { +return true; +} +if (o == null || getClass() != o.getClass()) { +return false; +} + +AtlasSystemAttributes obj = (AtlasSystemAttributes) o; + +return Objects.equals(createdBy, obj.createdBy) && + Objects.equals(modifiedBy, obj.modifiedBy) && + Objects.equals(createdTime, obj.createdTime) && + Objects.equals(modifiedTime, obj.modifiedTime); +} + +@Override +public int hashCode() { +return Objects.hash(createdBy, modifiedBy, createdTime, modifiedTime); +} +} http://git-wip-us.apache.org/repos/asf/atlas/blob/eacf8513/intg/src/main/java/org/apache/atlas/v1/model/instance/Id.java -- diff --git a/intg/src/main/java/org/apache/atlas/v1/model/instance/Id.java b/intg/src/main/java/org/apache/atlas/v1/model/instance/Id.java new file mode 100644 index 000..1b250f0 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/v1/model/instance/Id.java @@ -0,0
[5/5] atlas git commit: Merge branch 'ATLAS-2251' of https://git-wip-us.apache.org/repos/asf/atlas into ATLAS-2251
Merge branch 'ATLAS-2251' of https://git-wip-us.apache.org/repos/asf/atlas into ATLAS-2251 Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/3f44770d Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/3f44770d Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/3f44770d Branch: refs/heads/ATLAS-2251 Commit: 3f44770da188f270330230594344e9a263b5a5d4 Parents: eacf851 10c19ea Author: Madhan NeethirajAuthored: Sun Nov 5 12:41:54 2017 -0800 Committer: Madhan Neethiraj Committed: Sun Nov 5 12:41:54 2017 -0800 -- --
[16/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java -- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java deleted file mode 100644 index ef8bd2f..000 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java +++ /dev/null @@ -1,1049 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.store.graph.v1; - -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.AtlasException; -import org.apache.atlas.TestModules; -import org.apache.atlas.RequestContextV1; -import org.apache.atlas.TestUtils; -import org.apache.atlas.TestUtilsV2; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasClassification; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.model.instance.EntityMutations; -import org.apache.atlas.model.instance.EntityMutations.EntityOperation; -import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasEntityDef; -import org.apache.atlas.model.typedef.AtlasStructDef; -import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; -import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.store.AtlasTypeDefStore; -import org.apache.atlas.type.AtlasArrayType; -import org.apache.atlas.type.AtlasMapType; -import org.apache.atlas.type.AtlasStructType; -import org.apache.atlas.type.AtlasType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; -import static org.apache.atlas.TestUtils.COLUMN_TYPE; -import static org.apache.atlas.TestUtils.NAME; -import static org.apache.atlas.TestUtils.randomString; -import static org.apache.atlas.TestUtilsV2.STORAGE_DESC_TYPE; -import static org.apache.atlas.TestUtilsV2.TABLE_TYPE; -import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -@Guice(modules = TestModules.TestOnlyModule.class) -public class AtlasEntityStoreV1Test { -private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV1Test.class); - -@Inject -AtlasTypeRegistry typeRegistry; - -@Inject -AtlasTypeDefStore typeDefStore; - -AtlasEntityStore entityStore; - -@Inject -MetadataService metadataService; - -@Inject -
[36/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java -- diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java deleted file mode 100644 index b936695..000 --- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java +++ /dev/null @@ -1,485 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.gremlin; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.groovy.AbstractFunctionExpression; -import org.apache.atlas.groovy.CastExpression; -import org.apache.atlas.groovy.ClosureExpression; -import org.apache.atlas.groovy.ComparisonExpression; -import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator; -import org.apache.atlas.groovy.ComparisonOperatorExpression; -import org.apache.atlas.groovy.FieldExpression; -import org.apache.atlas.groovy.FunctionCallExpression; -import org.apache.atlas.groovy.GroovyExpression; -import org.apache.atlas.groovy.IdentifierExpression; -import org.apache.atlas.groovy.LiteralExpression; -import org.apache.atlas.groovy.LogicalExpression; -import org.apache.atlas.groovy.LogicalExpression.LogicalOperator; -import org.apache.atlas.groovy.TernaryOperatorExpression; -import org.apache.atlas.groovy.TraversalStepType; -import org.apache.atlas.groovy.TypeCoersionExpression; -import org.apache.atlas.query.GraphPersistenceStrategies; -import org.apache.atlas.query.TypeUtils.FieldInfo; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.IDataType; - -/** - * Generates gremlin query expressions using Gremlin 3 syntax. - * - */ -public class Gremlin3ExpressionFactory extends GremlinExpressionFactory { - - - -private static final String VERTEX_LIST_CLASS = "List"; -private static final String VERTEX_ARRAY_CLASS = "Vertex[]"; -private static final String OBJECT_ARRAY_CLASS = "Object[]"; -private static final String VERTEX_CLASS = "Vertex"; -private static final String FUNCTION_CLASS = "Function"; - -private static final String VALUE_METHOD = "value"; -private static final String IS_PRESENT_METHOD = "isPresent"; -private static final String MAP_METHOD = "map"; -private static final String VALUES_METHOD = "values"; -private static final String GET_METHOD = "get"; -private static final String OR_ELSE_METHOD = "orElse"; -private static final String PROPERTY_METHOD = "property"; -private static final String BY_METHOD = "by"; -private static final String EQ_METHOD = "eq"; -private static final String EMIT_METHOD = "emit"; -private static final String TIMES_METHOD = "times"; -private static final String REPEAT_METHOD = "repeat"; -private static final String RANGE_METHOD = "range"; -private static final String LAST_METHOD = "last"; -private static final String TO_STRING_METHOD = "toString"; - -private static final GroovyExpression EMPTY_STRING_EXPRESSION = new LiteralExpression(""); - -@Override -public GroovyExpression generateLogicalExpression(GroovyExpression parent, String operator, - List operands) { -return new FunctionCallExpression(TraversalStepType.FILTER, parent, operator, operands); -} - -@Override -public GroovyExpression generateBackReferenceExpression(GroovyExpression parent, boolean inSelect, String alias) { -if (inSelect) { -return getFieldInSelect(); -} else { -return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, SELECT_METHOD, new LiteralExpression(alias)); -} -} - -@Override -public GroovyExpression typeTestExpression(GraphPersistenceStrategies s, String typeName,
[32/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java -- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java deleted file mode 100755 index 74886b5..000 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ /dev/null @@ -1,505 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.graph; - -import com.google.common.base.Preconditions; -import org.apache.atlas.AtlasException; -import org.apache.atlas.CreateUpdateEntitiesResult; -import org.apache.atlas.GraphTransactionInterceptor; -import org.apache.atlas.RequestContext; -import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.model.instance.GuidMapping; -import org.apache.atlas.model.legacy.EntityResult; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasGraphQuery; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.ITypedStruct; -import org.apache.atlas.typesystem.exception.EntityExistsException; -import org.apache.atlas.typesystem.exception.EntityNotFoundException; -import org.apache.atlas.typesystem.exception.TraitNotFoundException; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.*; - -/** - * An implementation backed by a Graph database provided - * as a Graph Service. - */ -@Singleton -@Component -@Deprecated -public class GraphBackedMetadataRepository implements MetadataRepository { - -private static final Logger LOG = LoggerFactory.getLogger(GraphBackedMetadataRepository.class); - -private static TypeSystem typeSystem = TypeSystem.getInstance(); - -private static final GraphHelper graphHelper = GraphHelper.getInstance(); - -private DeleteHandler deleteHandler; - -private final AtlasGraph atlasGraph; -private final GraphToTypedInstanceMapper graphToInstanceMapper; - -@Inject -public GraphBackedMetadataRepository(DeleteHandler deleteHandler, AtlasGraph atlasGraph) { -this.atlasGraph = atlasGraph; -this.graphToInstanceMapper = new GraphToTypedInstanceMapper(atlasGraph); -this.deleteHandler = deleteHandler; -} - -public GraphToTypedInstanceMapper getGraphToInstanceMapper() { -return graphToInstanceMapper; -} - -@Override -public String getTypeAttributeName() { -return Constants.ENTITY_TYPE_PROPERTY_KEY; -} - -@Override -public String getStateAttributeName() { -return Constants.STATE_PROPERTY_KEY; -} - -/** - * Returns the property key used to store super type names. - * - * @return property key used to store super type names. - */ -@Override -public String getSuperTypeAttributeName() { -return Constants.SUPER_TYPES_PROPERTY_KEY; -} - -public String getIdAttributeName() { -return Constants.GUID_PROPERTY_KEY; -} - -@Override -public String getVersionAttributeName() { -return Constants.VERSION_PROPERTY_KEY; -} - -@Override -public String getTraitLabel(IDataType dataType, String traitName) { -return
[20/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java -- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java deleted file mode 100644 index e64c2c8..000 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.graph; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.TestModules; -import org.apache.atlas.TestUtils; -import org.apache.atlas.repository.graph.GraphHelper.VertexInfo; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.exception.TypeNotFoundException; -import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.codehaus.jettison.json.JSONArray; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.testng.Assert.*; - -@Guice(modules = TestModules.TestOnlyModule.class) -public class GraphHelperTest { - - -@DataProvider(name = "encodeDecodeTestData") -private Object[][] createTestData() { -return new Object[][]{ -{"hivedb$", "hivedb_d"}, -{"hivedb", "hivedb"}, -{"{hivedb}", "_ohivedb_c"}, -{"%hivedb}", "_phivedb_c"}, -{"\"hivedb\"", "_qhivedb_q"}, -{"\"$%{}", "_q_d_p_o_c"}, -{"", ""}, -{" ", " "}, -{"\n\r", "\n\r"}, -{null, null} -}; -} - -@Inject -private MetadataService metadataService; - -@Inject -private GraphBackedMetadataRepository repositoryService; - -private TypeSystem typeSystem; - -@Inject -private AtlasTypeRegistry typeRegistry; - -@BeforeClass -public void setUp() throws Exception { -typeSystem = TypeSystem.getInstance(); -typeSystem.reset(); - -new GraphBackedSearchIndexer(typeRegistry); -TypesDef typesDef = TestUtils.defineHiveTypes(); -try { -metadataService.getTypeDefinition(TestUtils.TABLE_TYPE); -} catch (TypeNotFoundException e) { -metadataService.createType(TypesSerialization.toJson(typesDef)); -} -TestUtils.defineDeptEmployeeTypes(typeSystem); -} - -@AfterClass -public void tearDown() { -//AtlasGraphProvider.cleanup(); -} - -@Test -public void testGetInstancesByUniqueAttributes() throws Exception { - -GraphHelper helper = GraphHelper.getInstance(); -List instances = new ArrayList<>(); -List guids = new ArrayList<>(); -TypeSystem ts = TypeSystem.getInstance(); -ClassType dbType = ts.getDataType(ClassType.class, TestUtils.DATABASE_TYPE); - -for(int i = 0; i < 10; i++) { -Referenceable db = TestUtils.createDBEntity(); -String guid = createInstance(db); -ITypedReferenceableInstance
[10/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java -- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java b/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java deleted file mode 100755 index e0f8576..000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem; - -/** - * A marker interface for StructType and TraitType instances. - */ -public interface IStruct extends IInstance { - -} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java -- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java deleted file mode 100755 index c3bb9af..000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.FieldMapping; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.security.MessageDigest; -import java.util.Date; - -/** - * An instance whose structure is associated with a IDataType. - * This is obtained by a call to 'createInstance' or the result of a Query. - * A ITypedInstance can only contain information on attributes of the associated Type. - * Instance can still be invalid because of missing required fields or incorrect multiplicity. - * But user can only get/set on a known field of the associated type. Type values have to match - * the IDataType of the associated attribute. - */ -public interface ITypedInstance extends IInstance { - -FieldMapping fieldMapping(); - -boolean getBoolean(String attrName) throws AtlasException; - -byte getByte(String attrName) throws AtlasException; - -short getShort(String attrName) throws AtlasException; - -int getInt(String attrName) throws AtlasException; - -long getLong(String attrName) throws AtlasException; - -float getFloat(String attrName) throws AtlasException; - -double getDouble(String attrName) throws AtlasException; - -BigInteger getBigInt(String attrName) throws AtlasException; - -BigDecimal getBigDecimal(String attrName) throws AtlasException; - -Date getDate(String attrName) throws AtlasException; - -String getString(String attrName) throws AtlasException; - -void setBoolean(String attrName, boolean val) throws AtlasException; - -void setByte(String attrName, byte val) throws AtlasException; - -void setShort(String attrName, short val) throws AtlasException; - -void setInt(String attrName, int val) throws AtlasException; - -void setLong(String attrName, long val) throws AtlasException; - -void setFloat(String attrName, float val) throws AtlasException; - -void setDouble(String attrName, double val) throws AtlasException; - -void setBigInt(String attrName, BigInteger val) throws AtlasException; - -void setBigDecimal(String attrName, BigDecimal
[34/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/query/Expressions.java -- diff --git a/repository/src/main/java/org/apache/atlas/query/Expressions.java b/repository/src/main/java/org/apache/atlas/query/Expressions.java new file mode 100644 index 000..9e93ce4 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/query/Expressions.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.query; + + +import java.util.List; + +public class Expressions { +public static class Expression { + +} + +public static class AliasExpression { +public String alias() { +String ret = null; + +return ret; +} + +} + +public static class SelectExpression { +public List toJavaList() { +List ret = null; + +return ret; +} + +} +} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/query/GremlinQuery.java -- diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinQuery.java b/repository/src/main/java/org/apache/atlas/query/GremlinQuery.java new file mode 100644 index 000..fcb1f48 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/query/GremlinQuery.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.query; + +import org.apache.atlas.query.Expressions.Expression; + + +public class GremlinQuery { + +public boolean hasSelectList() { +boolean ret = false; + +return ret; +} + +public String queryStr() { +String ret = null; + +return ret; +} + +public Expression expr() { +Expression ret = null; + +return ret; +} +} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java -- diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java b/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java new file mode 100644 index 000..5395ddd --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.query; + +import org.apache.atlas.query.Expressions.Expression; + +public class GremlinTranslator { +private Expression expression; +
[13/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java -- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java deleted file mode 100644 index e885b8c..000 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ /dev/null @@ -1,1381 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.service; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasException; -import org.apache.atlas.CreateUpdateEntitiesResult; -import org.apache.atlas.EntityAuditEvent; -import org.apache.atlas.RequestContext; -import org.apache.atlas.TestModules; -import org.apache.atlas.TestUtils; -import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.listener.ChangedTypeDefs; -import org.apache.atlas.listener.EntityChangeListener; -import org.apache.atlas.listener.TypeDefChangeListener; -import org.apache.atlas.model.legacy.EntityResult; -import org.apache.atlas.query.QueryParams; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.audit.EntityAuditRepository; -import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; -import org.apache.atlas.repository.audit.HBaseTestUtils; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.services.DefaultMetadataService; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.exception.EntityNotFoundException; -import org.apache.atlas.typesystem.exception.TypeNotFoundException; -import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumValue; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.ValueConversionException; -import org.apache.atlas.typesystem.types.cache.TypeCache; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.apache.atlas.utils.ParamChecker; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.testng.Assert; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import static org.apache.atlas.TestUtils.*; -import static
[24/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java -- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java deleted file mode 100755 index bd730e4..000 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ /dev/null @@ -1,1334 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.discovery; - -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.AtlasException; -import org.apache.atlas.BaseRepositoryTest; -import org.apache.atlas.RequestContext; -import org.apache.atlas.TestModules; -import org.apache.atlas.TestUtils; -import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; -import org.apache.atlas.query.QueryParams; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.repository.graphdb.GremlinVersion; -import org.apache.atlas.util.AtlasGremlinQueryProvider; -import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import javax.inject.Inject; -import java.text.SimpleDateFormat; -import java.util.*; - -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; - -@Guice(modules = TestModules.TestOnlyModule.class) -public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { - -@Inject -private MetadataRepository repositoryService; - -@Inject -private GraphBackedDiscoveryService discoveryService; -private QueryParams queryParams = new QueryParams(40, 0); -private static final String idType = "idType"; -@Override -@BeforeClass -public void setUp() throws Exception { -super.setUp(); - -repositoryService = TestUtils.addTransactionWrapper(repositoryService); -final TypeSystem typeSystem = TypeSystem.getInstance(); -Collection oldTypeNames = new HashSet<>(); -oldTypeNames.addAll(typeSystem.getTypeNames()); - -TestUtils.defineDeptEmployeeTypes(typeSystem); - -addIndexesForNewTypes(oldTypeNames, typeSystem); - -ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem); -repositoryService.createEntities(hrDept); - -ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Manager", "name", "Jane"); -Id janeGuid = jane.getId(); -ClassType personType = typeSystem.getDataType(ClassType.class, "Person"); -ITypedReferenceableInstance instance = personType.createInstance(janeGuid); -
[18/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java -- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java deleted file mode 100644 index 1c1c68f..000 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.impexp; - -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.testng.Assert; -import org.testng.ITestContext; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.io.ByteArrayInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.List; - -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.AssertJUnit.assertTrue; - -public class ZipSourceTest { -@DataProvider(name = "zipFileStocks") -public static Object[][] getDataFromZipFile() throws IOException { -FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream("stocks.zip"); - -return new Object[][] {{ new ZipSource(fs) }}; -} - -@DataProvider(name = "zipFileStocksFloat") -public static Object[][] getDataFromZipFileWithLongFloats() throws IOException { -FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream("stocks-float.zip"); - -return new Object[][] {{ new ZipSource(fs) }}; -} - -@DataProvider(name = "sales") -public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException { -return getZipSource("sales-v1-full.zip"); -} - -@Test -public void improperInit_ReturnsNullCreationOrder() throws IOException, AtlasBaseException { -byte bytes[] = new byte[10]; -ByteArrayInputStream bais = new ByteArrayInputStream(bytes); -ZipSource zs = new ZipSource(bais); -List s = zs.getCreationOrder(); -Assert.assertNull(s); -} - -@Test(dataProvider = "zipFileStocks") -public void examineContents_BehavesAsExpected(ZipSource zipSource) throws IOException, AtlasBaseException { -List creationOrder = zipSource.getCreationOrder(); - -assertNotNull(creationOrder); -assertEquals(creationOrder.size(), 4); - -AtlasTypesDef typesDef = zipSource.getTypesDef(); -assertNotNull(typesDef); -assertEquals(typesDef.getEntityDefs().size(), 6); - -useCreationOrderToFetchEntitiesWithExtInfo(zipSource, creationOrder); -useCreationOrderToFetchEntities(zipSource, creationOrder); -attemptToFetchNonExistentGuid_ReturnsNull(zipSource, "non-existent-guid"); -verifyGuidRemovalOnImportComplete(zipSource, creationOrder.get(0)); -} - -private void useCreationOrderToFetchEntities(ZipSource zipSource, List creationOrder) { -for (String guid : creationOrder) { -AtlasEntity e = zipSource.getByGuid(guid); -assertNotNull(e); -} -} - -private void verifyGuidRemovalOnImportComplete(ZipSource zipSource, String guid) { -AtlasEntity e = zipSource.getByGuid(guid); -assertNotNull(e); - -zipSource.onImportComplete(guid); - -e = zipSource.getByGuid(guid); -Assert.assertNull(e); -} - -private void attemptToFetchNonExistentGuid_ReturnsNull(ZipSource zipSource, String guid) { -AtlasEntity e = zipSource.getByGuid(guid); -Assert.assertNull(e); -} - -private void useCreationOrderToFetchEntitiesWithExtInfo(ZipSource zipSource, List creationOrder) throws AtlasBaseException { -for (String guid : creationOrder) { -
[26/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java -- diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java deleted file mode 100644 index 7c6b60b..000 --- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java +++ /dev/null @@ -1,428 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.*; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.testng.annotations.Guice; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS; -import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS; - -/** - * Base Class to set up hive types and instances for tests - */ -@Guice(modules = TestModules.TestOnlyModule.class) -public class BaseRepositoryTest { - -@Inject -protected MetadataService metadataService; - -@Inject -protected MetadataRepository repository; - - -protected void setUp() throws Exception { -//force graph initialization / built in type registration -TestUtils.getGraph(); -setUpDefaultTypes(); -setUpTypes(); -TestUtils.getGraph().commit(); -new GraphBackedSearchIndexer(new AtlasTypeRegistry()); -TestUtils.resetRequestContext(); -setupInstances(); -TestUtils.getGraph().commit(); -TestUtils.dumpGraph(TestUtils.getGraph()); -} - -protected void tearDown() throws Exception { -TypeSystem.getInstance().reset(); -} - -private void setUpTypes() throws Exception { -TypesDef typesDef = createTypeDefinitions(); -String typesAsJSON = TypesSerialization.toJson(typesDef); -metadataService.createType(typesAsJSON); -} - -protected static final String DATABASE_TYPE = "hive_db"; -protected static final String HIVE_TABLE_TYPE = "hive_table"; -private static final String COLUMN_TYPE = "hive_column"; -private static final String HIVE_PROCESS_TYPE = "hive_process"; -private static final String STORAGE_DESC_TYPE = "StorageDesc"; -private static final String VIEW_TYPE = "View"; -private static final String PARTITION_TYPE = "hive_partition"; -protected static final String DATASET_SUBTYPE = "dataset_subtype"; - -TypesDef createTypeDefinitions() { -HierarchicalTypeDefinition dbClsDef = TypesUtil -.createClassTypeDef(DATABASE_TYPE, null, -TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), -attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), -attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE)); - -HierarchicalTypeDefinition columnClsDef = TypesUtil -.createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), -attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); - -HierarchicalTypeDefinition storageDescClsDef = TypesUtil -.createClassTypeDef(STORAGE_DESC_TYPE, null, -attrDef("location", DataTypes.STRING_TYPE), -attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat",
[33/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java -- diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java index 6b6ee01..b03eda3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java +++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java @@ -18,21 +18,24 @@ package org.apache.atlas.repository.converters; import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.AtlasException; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.model.v1.instance.Struct; +import org.apache.atlas.type.*; +import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; -import org.apache.atlas.type.AtlasType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.Struct; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter { @@ -64,19 +67,12 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter { } else { ret = new AtlasStruct(type.getTypeName()); } -} else if (v1Obj instanceof IStruct) { -IStruct struct= (IStruct) v1Obj; -Mapv1Attribs = null; - -try { -v1Attribs = struct.getValuesMap(); -} catch (AtlasException excp) { -LOG.error("IStruct.getValuesMap() failed", excp); -} +} else if (v1Obj instanceof Struct) { +Struct struct = (Struct) v1Obj; -ret = new AtlasStruct(type.getTypeName(), fromV1ToV2(structType, v1Attribs, converterContext)); +ret = new AtlasStruct(type.getTypeName(), fromV1ToV2(structType, struct.getValues(), converterContext)); } else { -throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "Map or IStruct", v1Obj.getClass().getCanonicalName()); +throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "Map or Struct", v1Obj.getClass().getCanonicalName()); } } @@ -118,7 +114,8 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter { } protected Map fromV2ToV1(AtlasStructType structType, Map attributes, ConverterContext context) throws AtlasBaseException { -Map ret = null; +Map ret = null; +boolean isEntityType = structType instanceof AtlasEntityType; if (MapUtils.isNotEmpty(attributes)) { ret = new HashMap<>(); @@ -132,13 +129,80 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter { continue; } -AtlasType attrType = attr.getAttributeType(); +AtlasTypeattrType = attr.getAttributeType(); +AtlasFormatConverter attrConverter = converterRegistry.getConverter(attrType.getTypeCategory()); +Object v2Value = attributes.get(attr.getName()); + +if (v2Value != null && isEntityType && attr.isOwnedRef()) { +if (LOG.isDebugEnabled()) { +LOG.debug("{}: is ownedRef, attrType={}", attr.getQualifiedName(), attrType.getTypeName()); +} + +if (attrType instanceof AtlasArrayType) { +AtlasArrayType arrayType = (AtlasArrayType) attrType; +AtlasType elemType = arrayType.getElementType(); +String elemTypeName; + +if (elemType instanceof AtlasObjectIdType) { +elemTypeName = ((AtlasObjectIdType) elemType).getObjectType(); +} else { +elemTypeName =
[19/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java -- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java deleted file mode 100644 index b70b181..000 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.impexp; - -import org.apache.atlas.model.impexp.AtlasImportRequest; -import org.apache.atlas.type.AtlasType; -import org.testng.annotations.Test; - -import static org.testng.Assert.*; - -public class AtlasImportRequestTest { -@Test -public void serializeAtlasImportRequstFromJsonWithEmptyOptions() { -String jsonData = "{ \"options\": {} }"; - -AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); - -assertNotNull(request); -assertNotNull(request.getOptions()); - assertNull(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY)); - -ImportTransforms tr = ImportTransforms.fromJson(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY)); - -assertNull(tr); -} - -@Test -public void serializeOptions_VerifyAccessors() { -String guid = "\"abcd\""; -String pos = "\"1\""; -String trueVal = "\"true\""; - -String jsonData = "{ \"options\": " + -" {" + -"\"startGuid\":" + guid + "," + -"\"startPosition\":" + pos + "," + -"\"updateTypeDefinition\":" + trueVal + -"}" + -"}"; - -AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); - -assertNotNull(request); -assertNotNull(request.getStartGuid()); -assertNotNull(request.getStartPosition()); -assertNotNull(request.getUpdateTypeDefs()); - -assertEquals(request.getStartGuid(), guid.replace("\"", "")); -assertEquals(request.getStartPosition(), pos.replace("\"", "")); -assertEquals(request.getUpdateTypeDefs(), trueVal.replace("\"", "")); -} - -@Test -public void optionsDefaultsTest() { -String jsonData = "{ \"options\": " + -" {" + -"}" + -"}"; - -AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); - -assertNotNull(request); -assertNull(request.getStartGuid()); -assertNull(request.getStartPosition()); -assertNull(request.getUpdateTypeDefs()); -} - -@Test -public void serializeAtlasImportRequstFromJsonWithEmptyTransforms() { -String jsonData = "{ \"options\": { \"transforms\": \"{ }\" } }"; - -AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); - -assertNotNull(request); -assertNotNull(request.getOptions()); - assertNotNull(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY)); - -ImportTransforms tr = ImportTransforms.fromJson(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY)); - -assertNotNull(tr); -assertNotNull(tr.getTransforms()); -assertEquals(tr.getTransforms().size(), 0); -} - -@Test -public void serializeAtlasImportRequstFromJsonWith1Transform() { -String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] } }\" } }"; - -AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); - -assertNotNull(request); -assertNotNull(request.getOptions()); - assertNotNull(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY)); - -ImportTransforms tr = ImportTransforms.fromJson(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY)); - -
[35/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpandOrsOptimization.java -- diff --git a/repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpandOrsOptimization.java b/repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpandOrsOptimization.java deleted file mode 100644 index a48a007..000 --- a/repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpandOrsOptimization.java +++ /dev/null @@ -1,588 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.gremlin.optimizer; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -import org.apache.atlas.gremlin.GremlinExpressionFactory; -import org.apache.atlas.groovy.AbstractFunctionExpression; -import org.apache.atlas.groovy.ClosureExpression; -import org.apache.atlas.groovy.FunctionCallExpression; -import org.apache.atlas.groovy.GroovyExpression; -import org.apache.atlas.groovy.LiteralExpression; -import org.apache.atlas.groovy.StatementListExpression; -import org.apache.atlas.groovy.TraversalStepType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - - - -/** - * Optimization that removes 'or' expressions from a graph traversal when possible - * and replaces them with separate calls that are combined using a logical union operation. - * Unfortunately, Titan does not use indices when executing the child graph traversals associated - * with an 'or' call. In order to make the index be used, we split queries with - * or expressions into multiple queries. These queries are executed individually, - * using indices, and then the results are combined back together. Here is a - * simple example to illustrate this: - * - * Original Query - * - * - * g.V().or(has('name','Fred'),has('age','17')) - * - * - *Optimized Query - * - * - * def r = [] as Set; - * g.V().has('name','Fred').fill(r); - * g.V().has('age','17').fill(r); - * r; - * - * - * Here, we introduce an intermediate variable "r" which is declared as a Set. The Set is performing - * the union for us. If there are vertices that happen to both have "Fred" as the name and "17" as the age, - * the Set will prevent the second query execution from adding a duplicate vertex to the result. Recall that - * in Groovy scripts, the last expression is the one that will be returned back to the caller. We refer to - * that expression is the "result expression". For this example, the result expression is simply "r", which - * contains the vertices that matched the query. - * - * If the query does any kind of transformation of the vertices to produce the query result, that needs - * to be done in the result expression. To understand why that is, let's take a look at another example: - * - * Original Query - * - * - * g.V().or(has('name','Fred'),has('age','17')).as('person').select('person').by('gender') - * - * - * Incorrect Optimized Query - * - * - * def r = [] as Set; - * g.V().has('name','Fred').as('person').select('person').by('gender').fill(r) - * g.V().has('age','17').as('person').select('person').by('gender').fill(r) - * r; - * - * - * The problem with this query is that now 'r' contains Strings (the gender of the person). Suppose - * that there is one person named Fred and there are 3 people whose age is 17 (let's say Fred's age is 16). - * The original query would have produced 4 rows, one corresponding to each of those people. The new - * query would produce at most 2 rows - one for 'male' and one for 'female'. This is happening because - * we are now performing the union on the Strings, not on the vertices. To fix this, we need to split - * the original query and put the end portion into the result expression: - * - * Correct Optimized Query - * - * - * def r = [] as Set; - * g.V().has('name','Fred').fill(r) - * g.V().has('age','17').fill(r) - * __.inject(r as Object[]).as('person').select('person').by('gender') - * - * - * The logic for doing this splitting is described in more detail in - * {@link
[17/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java -- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java deleted file mode 100644 index 718538a..000 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java +++ /dev/null @@ -1,1141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.store.graph.v1; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContextV1; -import org.apache.atlas.TestUtils; -import org.apache.atlas.TestUtilsV2; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.model.instance.EntityMutations; -import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasEntityDef; -import org.apache.atlas.model.typedef.AtlasEnumDef; -import org.apache.atlas.model.typedef.AtlasStructDef; -import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.graph.AtlasEdgeLabel; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.store.AtlasTypeDefStore; -import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.ITypedStruct; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.exception.EntityNotFoundException; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.atlas.TestUtils.*; -import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -public abstract class AtlasDeleteHandlerV1Test { - -@Inject -AtlasTypeRegistry typeRegistry; - -@Inject -AtlasTypeDefStore typeDefStore; - -@Inject -AtlasEntityStore entityStore; - -@Inject -MetadataService metadataService; - -private AtlasEntityType compositeMapOwnerType; - -private AtlasEntityType compositeMapValueType; - -private TypeSystem typeSystem = TypeSystem.getInstance(); - -@BeforeClass -public void setUp() throws Exception { -RequestContextV1.clear(); -RequestContextV1.get().setUser(TestUtilsV2.TEST_USER); - -metadataService =
[30/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java -- diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java deleted file mode 100755 index 429730c..000 --- a/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.memory; - -import com.google.common.collect.ImmutableBiMap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.persistence.ReferenceableInstance; -import org.apache.atlas.typesystem.persistence.StructInstance; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.HierarchicalType; -import org.apache.atlas.typesystem.types.IConstructableType; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -@Deprecated -public abstract class HierarchicalTypeStore { - -final MemRepository repository; -final IConstructableType hierarchicalType; -final ArrayList typeNameList; -final ImmutableMapattrStores; -final ImmutableList superTypeStores; - - -/** - * Map Id to position in storage lists. - */ -Map idPosMap; - -List freePositions; - -int nextPos; - -/** - * Lock for each Class/Trait. - */ -ReentrantReadWriteLock lock; - -HierarchicalTypeStore(MemRepository repository, HierarchicalType hierarchicalType) throws RepositoryException { -this.hierarchicalType = (IConstructableType) hierarchicalType; -this.repository = repository; -ImmutableMap.Builder b = -new ImmutableBiMap.Builder<>(); -typeNameList = Lists.newArrayList((String) null); -ImmutableList l = hierarchicalType.immediateAttrs; -for (AttributeInfo i : l) { -b.put(i, AttributeStores.createStore(i)); -} -attrStores = b.build(); - -ImmutableList.Builder b1 = new ImmutableList.Builder<>(); -Set allSuperTypeNames = hierarchicalType.getAllSuperTypeNames(); -for (String s : allSuperTypeNames) { -b1.add(repository.getStore(s)); -} -superTypeStores = b1.build(); - -nextPos = 0; -idPosMap = new HashMap<>(); -freePositions = new ArrayList<>(); - -lock = new ReentrantReadWriteLock(); -} - -/** - * Assign a storage position to an Id. - * - try to assign from freePositions - * - ensure storage capacity. - * - add entry in idPosMap. - * @param id - * @return - * @throws RepositoryException - */ -int assignPosition(Id id) throws RepositoryException { - -int pos = -1; -if (!freePositions.isEmpty()) { -pos = freePositions.remove(0); -} else { -pos = nextPos++; -ensureCapacity(pos); -} - -idPosMap.put(id, pos); - -for (HierarchicalTypeStore s : superTypeStores) { -s.assignPosition(id); -} - -return pos; -} - -/** - * - remove from idPosMap - * - add to freePositions. - * @throws RepositoryException - */ -void releaseId(Id id) { - -Integer pos = idPosMap.get(id); -if (pos != null) { -idPosMap.remove(id); -freePositions.add(pos); - -for (HierarchicalTypeStore s : superTypeStores) { -s.releaseId(id); -} -} -} - -void acquireReadLock() { -lock.readLock().lock(); -
[27/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala -- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala deleted file mode 100644 index 37015d8..000 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala +++ /dev/null @@ -1,806 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.query - -import java.lang.Boolean -import java.lang.Byte -import java.lang.Double -import java.lang.Float -import java.lang.Integer -import java.lang.Long -import java.lang.Short -import java.util.ArrayList - -import scala.collection.JavaConversions.asScalaBuffer -import scala.collection.JavaConversions.bufferAsJavaList -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer - - -import org.apache.atlas.gremlin.GremlinExpressionFactory -import org.apache.atlas.gremlin.optimizer.GremlinQueryOptimizer -import org.apache.atlas.groovy.CastExpression -import org.apache.atlas.groovy.ClosureExpression -import org.apache.atlas.groovy.LabeledExpression -import org.apache.atlas.groovy.FunctionCallExpression -import org.apache.atlas.groovy.GroovyExpression -import org.apache.atlas.groovy.GroovyGenerationContext -import org.apache.atlas.groovy.IdentifierExpression -import org.apache.atlas.groovy.ListExpression -import org.apache.atlas.groovy.LiteralExpression -import org.apache.atlas.groovy.TraversalStepType -import org.apache.atlas.query.Expressions.AliasExpression -import org.apache.atlas.query.Expressions.ArithmeticExpression -import org.apache.atlas.query.Expressions.BackReference -import org.apache.atlas.query.Expressions.ClassExpression -import org.apache.atlas.query.Expressions.ComparisonExpression -import org.apache.atlas.query.Expressions.Expression -import org.apache.atlas.query.Expressions.ExpressionException -import org.apache.atlas.query.Expressions.FieldExpression -import org.apache.atlas.query.Expressions.FilterExpression -import org.apache.atlas.query.Expressions.InstanceExpression -import org.apache.atlas.query.Expressions.LimitExpression -import org.apache.atlas.query.Expressions.ListLiteral -import org.apache.atlas.query.Expressions.Literal -import org.apache.atlas.query.Expressions.LogicalExpression -import org.apache.atlas.query.Expressions.LoopExpression -import org.apache.atlas.query.Expressions.OrderExpression -import org.apache.atlas.query.Expressions.PathExpression -import org.apache.atlas.query.Expressions.SelectExpression -import org.apache.atlas.query.Expressions.TraitExpression -import org.apache.atlas.query.Expressions.TraitInstanceExpression -import org.apache.atlas.query.Expressions.hasFieldLeafExpression -import org.apache.atlas.query.Expressions.hasFieldUnaryExpression -import org.apache.atlas.query.Expressions.id -import org.apache.atlas.query.Expressions.isTraitLeafExpression -import org.apache.atlas.query.Expressions.isTraitUnaryExpression -import org.apache.atlas.repository.RepositoryException -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection -import org.apache.atlas.typesystem.types.DataTypes -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory -import org.apache.atlas.typesystem.types.IDataType -import org.apache.atlas.typesystem.types.TypeSystem -import org.joda.time.format.ISODateTimeFormat -import org.apache.atlas.query.Expressions.GroupByExpression -import org.apache.atlas.query.Expressions.MaxExpression -import org.apache.atlas.query.Expressions.MinExpression -import org.apache.atlas.query.Expressions.SumExpression -import org.apache.atlas.query.Expressions.CountExpression - -import org.apache.atlas.util.AtlasRepositoryConfiguration -import java.util.HashSet - -trait IntSequence { -def next: Int -} - -case class GremlinQuery(expr: Expression, queryStr: String, resultMaping: Map[String, (String, Int)]) { - -def hasSelectList = resultMaping != null - -def isPathExpression = expr.isInstanceOf[PathExpression] - -def isGroupBy = expr.isInstanceOf[GroupByExpression] -} - - -trait
[29/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java -- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java deleted file mode 100644 index 32a9a19..000 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.typestore; - -import java.util.Objects; - -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; - -/** - * Records the information needed to create a particular type vertex. - */ -public class TypeVertexInfo { - -private DataTypes.TypeCategory category; -private String typeName; -private String typeDescription; - -public TypeVertexInfo(TypeCategory category, String typeName, String typeDescription) { -super(); -this.category = category; -this.typeName = typeName; -this.typeDescription = typeDescription; -} - -public DataTypes.TypeCategory getCategory() { -return category; -} - -public void setCategory(DataTypes.TypeCategory category) { -this.category = category; -} - -public String getTypeName() { -return typeName; -} - -public void setTypeName(String typeName) { -this.typeName = typeName; -} - -public String getTypeDescription() { -return typeDescription; -} - -public void setTypeDescription(String typeDescription) { -this.typeDescription = typeDescription; -} - -@Override -public int hashCode() { -return Objects.hash(category, typeName); -} - -@Override -public boolean equals(Object obj) { - -if (this == obj) { -return true; -} - -if (getClass() != obj.getClass()) { -return false; -} - -TypeVertexInfo other = (TypeVertexInfo)obj; -if(! Objects.equals(category, other.category)) { -return false; -} - -if(! Objects.equals(typeName, other.typeName)) { -return false; -} - -return true; -} - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java -- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java deleted file mode 100644 index a6e353c..000 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.typestore; - -import java.util.List; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; -import org.apache.atlas.typesystem.types.EnumType; -import
[08/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java -- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java deleted file mode 100755 index ab63fea..000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types; - -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.AtlasConstants; - -import java.util.Objects; - -public class HierarchicalTypeDefinition extends StructTypeDefinition { - -public final ImmutableSet superTypes; -public final String hierarchicalMetaTypeName; - -public HierarchicalTypeDefinition(Class hierarchicalMetaType, String typeName, String typeDescription, ImmutableSet superTypes, -AttributeDefinition[] attributeDefinitions) { -this(hierarchicalMetaType, typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, -attributeDefinitions); -} - -// Used only for de-serializing JSON String to typedef. -public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String typeVersion, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException { -this((Class) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, typeVersion, ImmutableSet.copyOf(superTypes), attributeDefinitions); -} -// Used only for de-serializing JSON String to typedef (no typeVersion). -public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException { -this((Class) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, ImmutableSet.copyOf(superTypes), attributeDefinitions); -} -// Used only for serializing typedef to JSON String. -public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String typeVersion, ImmutableSet superTypes, AttributeDefinition[] attributeDefinitions, String typeDef) throws ClassNotFoundException { -this((Class) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, typeVersion, superTypes, attributeDefinitions); -} -// Used only for serializing typedef to JSON String (no typeVersion). -public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, ImmutableSet superTypes, AttributeDefinition[] attributeDefinitions, String typeDef) throws ClassNotFoundException { -this((Class) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attributeDefinitions); -} - -public HierarchicalTypeDefinition(Class hierarchicalMetaType, String typeName, String typeDescription, String typeVersion, ImmutableSet superTypes, AttributeDefinition[] attributeDefinitions) { -super(typeName, typeDescription, typeVersion, false, attributeDefinitions); -this.hierarchicalMetaTypeName = hierarchicalMetaType.getName(); -this.superTypes = superTypes == null ? ImmutableSet.of() : superTypes; -} - -@Override -public boolean equals(Object o) { -if (this == o) return true; -if (o == null || getClass() != o.getClass()) return false; -if (!super.equals(o)) return false; -HierarchicalTypeDefinition that = (HierarchicalTypeDefinition) o; -return Objects.equals(superTypes, that.superTypes) && -Objects.equals(hierarchicalMetaTypeName, that.hierarchicalMetaTypeName); -} - -@Override -public int hashCode() { -return Objects.hash(super.hashCode(), superTypes,
[37/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java -- diff --git a/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java b/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java index 49b877b..3e1c3dd 100644 --- a/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java @@ -18,12 +18,13 @@ package org.apache.atlas.notification.hook; +import org.apache.atlas.model.v1.instance.Referenceable; +import org.apache.atlas.model.v1.instance.Struct; import org.apache.atlas.notification.AbstractNotification; import org.apache.atlas.notification.entity.EntityNotificationImplTest; import org.apache.atlas.notification.hook.HookNotification.EntityUpdateRequest; import org.apache.atlas.notification.hook.HookNotification.HookNotificationMessage; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.type.AtlasType; import org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.Test; @@ -61,7 +62,7 @@ public class HookMessageDeserializerTest { Referenceable entity = generateEntityWithTrait(); EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); -String jsonMsg = AbstractNotification.GSON.toJson(message); +String jsonMsg = AtlasType.toV1Json(message); HookNotificationMessage deserializedMessage = deserializer.deserialize(jsonMsg); assertEqualMessage(deserializedMessage, message); @@ -79,7 +80,7 @@ public class HookMessageDeserializerTest { assertTrue(jsonMsgList.size() == 1); String compressedMsg = jsonMsgList.get(0); -String uncompressedMsg = AbstractNotification.GSON.toJson(message); +String uncompressedMsg = AtlasType.toV1Json(message); assertTrue(compressedMsg.length() < uncompressedMsg.length(), "Compressed message (" + compressedMsg.length() + ") should be shorter than uncompressed message (" + uncompressedMsg.length() + ")"); @@ -134,7 +135,7 @@ public class HookMessageDeserializerTest { EntityUpdateRequest deserializedEntityUpdateRequest = (EntityUpdateRequest) deserializedMessage; Referenceable deserializedEntity = deserializedEntityUpdateRequest.getEntities().get(0); Referenceable entity = message.getEntities().get(0); -String traitName = entity.getTraits().get(0); +String traitName = entity.getTraitNames().get(0); assertEquals(deserializedEntity.getId(), entity.getId()); assertEquals(deserializedEntity.getTypeName(), entity.getTypeName()); http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java -- diff --git a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java index dd3257e..786fbfe 100644 --- a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java @@ -17,8 +17,8 @@ */ package org.apache.atlas.notification.hook; -import org.apache.atlas.notification.AbstractNotification; -import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.model.v1.instance.Referenceable; +import org.apache.atlas.type.AtlasType; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; @@ -37,7 +37,7 @@ public class HookNotificationTest { String user = "user"; HookNotification.EntityCreateRequest request = new HookNotification.EntityCreateRequest(user, entity1, entity2); -String notificationJson = AbstractNotification.GSON.toJson(request); +String notificationJson = AtlasType.toV1Json(request); HookNotification.HookNotificationMessage actualNotification = HOOK_MESSAGE_DESERIALIZER.deserialize(notificationJson); @@ -60,7 +60,7 @@ public class HookNotificationTest { entity.set("attr", "value"); HookNotification.EntityCreateRequest request = new HookNotification.EntityCreateRequest(null, entity); -String notificationJsonFromCode = AbstractNotification.GSON.toJson(request); +String notificationJsonFromCode =
[25/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java -- diff --git a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java deleted file mode 100644 index 3db58fe..000 --- a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java +++ /dev/null @@ -1,497 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.discovery; - -import com.google.common.collect.ImmutableList; -import org.apache.atlas.AtlasException; -import org.apache.atlas.BaseRepositoryTest; -import org.apache.atlas.TestModules; -import org.apache.atlas.TestUtils; -import org.apache.atlas.model.legacy.EntityResult; -import org.apache.atlas.query.QueryParams; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.exception.EntityNotFoundException; -import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.commons.collections.ArrayStack; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import javax.inject.Inject; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -/** - * Unit tests for Hive LineageService. - */ -@Guice(modules = TestModules.TestOnlyModule.class) -public class DataSetLineageServiceTest extends BaseRepositoryTest { - -@Inject -private DiscoveryService discoveryService; - -@Inject -private DataSetLineageService lineageService; - -@BeforeClass -public void setUp() throws Exception { -super.setUp(); -} - -@AfterClass -public void tearDown() throws Exception { -super.tearDown(); -} - -@DataProvider(name = "dslQueriesProvider") -private Object[][] createDSLQueries() { -return new String[][]{ -// joins -{"hive_table where name=\"sales_fact\", columns"}, -{"hive_table where name=\"sales_fact\", columns select name, dataType, comment"}, -{"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"}, -//{"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name, -// table.name"}, -{"from hive_db"}, {"hive_db"}, {"hive_db where hive_db.name=\"Reporting\""}, -{"hive_db hive_db.name = \"Reporting\""}, -{"hive_db where hive_db.name=\"Reporting\" select name, owner"}, {"hive_db has name"}, -//{"hive_db, hive_table"}, -//{"hive_db, hive_process has name"}, -//{"hive_db as db1, hive_table where db1.name = \"Reporting\""}, -//{"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System -// .currentTimeMillis()}, -{"from hive_table"}, {"hive_table"}, {"hive_table is Dimension"}, -{"hive_column where hive_column isa PII"}, -//{"hive_column where hive_column isa PII select hive_column.name"}, -{"hive_column select hive_column.name"}, {"hive_column select name"}, -{"hive_column where hive_column.name=\"customer_id\""}, {"from hive_table select hive_table.name"}, -{"hive_db where (name = \"Reporting\")"}, -{"hive_db where (name = \"Reporting\") select name as
[03/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java -- diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java index 53acf56..308f18d 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java +++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java @@ -21,31 +21,20 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.listener.EntityChangeListener; +import org.apache.atlas.model.v1.instance.Referenceable; +import org.apache.atlas.model.v1.instance.Struct; import org.apache.atlas.notification.entity.EntityNotification; import org.apache.atlas.notification.entity.EntityNotificationImpl; import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.types.FieldMapping; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.type.AtlasClassificationType; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.configuration.Configuration; import org.springframework.stereotype.Component; import javax.inject.Inject; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * Listen to the repository for entity changes and produce entity change notifications. @@ -54,7 +43,7 @@ import java.util.Set; public class NotificationEntityChangeListener implements EntityChangeListener { private final NotificationInterface notificationInterface; -private final TypeSystem typeSystem; +private final AtlasTypeRegistrytypeRegistry; private MapnotificationAttributesCache = new HashMap<>(); private static final String ATLAS_ENTITY_NOTIFICATION_PROPERTY = "atlas.notification.entity"; @@ -68,44 +57,44 @@ public class NotificationEntityChangeListener implements EntityChangeListener { * Construct a NotificationEntityChangeListener. * * @param notificationInterface the notification framework interface - * @param typeSystem the Atlas type system + * @param typeRegistry the Atlas type system */ @Inject -public NotificationEntityChangeListener(NotificationInterface notificationInterface, TypeSystem typeSystem) { +public NotificationEntityChangeListener(NotificationInterface notificationInterface, AtlasTypeRegistry typeRegistry) { this.notificationInterface = notificationInterface; -this.typeSystem = typeSystem; +this.typeRegistry = typeRegistry; } // - EntityChangeListener -- @Override -public void onEntitiesAdded(Collection entities, boolean isImport) throws AtlasException { +public void onEntitiesAdded(Collection entities, boolean isImport) throws AtlasException { notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_CREATE); } @Override -public void onEntitiesUpdated(Collection entities, boolean isImport) throws AtlasException { +public void onEntitiesUpdated(Collection entities, boolean isImport) throws AtlasException { notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_UPDATE); } @Override -public void onTraitsAdded(ITypedReferenceableInstance entity, Collection traits) throws AtlasException { +public void onTraitsAdded(Referenceable entity, Collection traits) throws AtlasException { notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.TRAIT_ADD); } @Override -public void onTraitsDeleted(ITypedReferenceableInstance entity, Collection traitNames) throws AtlasException { +public void onTraitsDeleted(Referenceable entity, Collection traitNames) throws AtlasException { notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.TRAIT_DELETE); } @Override -public void onTraitsUpdated(ITypedReferenceableInstance entity, Collection traits) throws
[40/40] atlas git commit: misc
misc Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/10c19eac Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/10c19eac Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/10c19eac Branch: refs/heads/ATLAS-2251 Commit: 10c19eac7d3e8ee57122dc5b26dcfbd8d8548250 Parents: 0877e47 Author: Madhan NeethirajAuthored: Sun Nov 5 12:29:56 2017 -0800 Committer: Madhan Neethiraj Committed: Sun Nov 5 12:29:56 2017 -0800 -- .../web/resources/DataSetLineageResource.java | 185 + .../atlas/web/resources/LineageResource.java| 207 ++ .../resources/MetadataDiscoveryResource.java| 405 +++ 3 files changed, 797 insertions(+) -- http://git-wip-us.apache.org/repos/asf/atlas/blob/10c19eac/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java -- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java new file mode 100644 index 000..278d7ca --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import org.apache.atlas.AtlasClient; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.util.Servlets; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; + +/** + * Jersey Resource for Hive Table Lineage. + */ +@Path("lineage/hive") +@Singleton +@Service +@Deprecated +public class DataSetLineageResource { + +private static final Logger LOG = LoggerFactory.getLogger(DataSetLineageResource.class); +private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.DataSetLineageResource"); + +/** + * Created by the Guice ServletModule and injected with the + * configured LineageService. + * + */ +@Inject +public DataSetLineageResource() { +} + +/** + * Returns the inputs graph for a given entity. + * + * @param tableName table name + */ +@GET +@Path("table/{tableName}/inputs/graph") +@Consumes(Servlets.JSON_MEDIA_TYPE) +@Produces(Servlets.JSON_MEDIA_TYPE) +public Response inputsGraph(@Context HttpServletRequest request, @PathParam("tableName") String tableName) { +if (LOG.isDebugEnabled()) { +LOG.debug("==> DataSetLineageResource.inputsGraph({})", tableName); +} + +AtlasPerfTracer perf = null; + +try { +if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { +perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataSetLineageResource.inputsGraph(tableName=" + tableName + ")"); +} + +final String jsonResult = ""; // TODO-typeSystem-removal: lineageService.getInputsGraph(tableName); + +JSONObject response = new JSONObject(); +response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); +response.put("tableName", tableName); +response.put(AtlasClient.RESULTS, new JSONObject(jsonResult)); + +return Response.ok(response).build(); +} catch (IllegalArgumentException e) { +LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); +throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); +} catch (WebApplicationException e) { +LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); +throw e; +} catch
[31/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java -- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java deleted file mode 100644 index b37fe75..000 --- a/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.graph; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContext; -import org.apache.atlas.annotation.ConditionalOnAtlasProperty; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; - -import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY; -import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; - -@Component -@ConditionalOnAtlasProperty(property = "atlas.DeleteHandler.impl", isDefault = true) -public class SoftDeleteHandler extends DeleteHandler { - -@Inject -public SoftDeleteHandler(TypeSystem typeSystem) { -super(typeSystem, false, true); -} - -@Override -protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) { -if (force) { -graphHelper.removeVertex(instanceVertex); -} else { -Id.EntityState state = GraphHelper.getState(instanceVertex); -if (state != Id.EntityState.DELETED) { -GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); -GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, -RequestContext.get().getRequestTime()); -GraphHelper.setProperty(instanceVertex, MODIFIED_BY_KEY, RequestContext.get().getUser()); -} -} -} - -@Override -protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException { -if (force) { -graphHelper.removeEdge(edge); -} else { -Id.EntityState state = GraphHelper.getState(edge); -if (state != Id.EntityState.DELETED) { -GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); -GraphHelper -.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); -GraphHelper.setProperty(edge, MODIFIED_BY_KEY, RequestContext.get().getUser()); -} -} -} -} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java -- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java deleted file mode 100644 index 1fb4ee9..000 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ /dev/null @@ -1,941 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the