date:20171109

[5/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

2017-11-09 Thread madhan

ATLAS-2251: restored bunch of deleted tests


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/042fc557
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/042fc557
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/042fc557

Branch: refs/heads/ATLAS-2251
Commit: 042fc55755e17b3b0112a7a08d30c296300b95d9
Parents: 5f40541
Author: Madhan Neethiraj 
Authored: Thu Nov 9 21:44:26 2017 -0800
Committer: Madhan Neethiraj 
Committed: Thu Nov 9 21:44:26 2017 -0800

--
 .../org/apache/atlas/query/QueryParser.java |   11 +-
 .../impexp/AtlasImportRequestTest.java  |  145 +++
 .../repository/impexp/ExportServiceTest.java|  358 ++
 .../repository/impexp/ImportServiceTest.java|  224 
 .../impexp/ImportServiceTestUtils.java  |  116 ++
 .../impexp/ImportTransformerJSONTest.java   |   47 +
 .../impexp/ImportTransformerTest.java   |  131 +++
 .../repository/impexp/ImportTransformsTest.java |  151 +++
 .../impexp/TypeAttributeDifferenceTest.java |  150 +++
 .../atlas/repository/impexp/UniqueListTest.java |   69 ++
 .../impexp/ZipFileResourceTestUtils.java|  220 
 .../atlas/repository/impexp/ZipSinkTest.java|  210 
 .../atlas/repository/impexp/ZipSourceTest.java  |  189 
 .../store/graph/AtlasTypeDefGraphStoreTest.java |  644 +++
 .../graph/v1/AtlasEntityDefStoreV1Test.java |   73 ++
 ...AtlasEntityStoreV1BulkImportPercentTest.java |  167 +++
 .../store/graph/v1/AtlasEntityStoreV1Test.java  | 1043 ++
 .../v1/AtlasRelationshipDefStoreV1Test.java |  330 ++
 .../AtlasRelationshipStoreHardDeleteV1Test.java |  112 ++
 .../AtlasRelationshipStoreSoftDeleteV1Test.java |  116 ++
 .../graph/v1/AtlasRelationshipStoreV1Test.java  |  623 +++
 .../InverseReferenceUpdateHardDeleteV1Test.java |   75 ++
 .../InverseReferenceUpdateSoftDeleteV1Test.java |   78 ++
 .../graph/v1/InverseReferenceUpdateV1Test.java  |  374 +++
 .../userprofile/UserProfileServiceTest.java |  280 +
 .../atlas/services/MetricsServiceTest.java  |  123 +++
 .../utils/ObjectUpdateSynchronizerTest.java |  218 
 .../org/apache/atlas/util/RestUtilsTest.java|  195 
 28 files changed, 6471 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/main/java/org/apache/atlas/query/QueryParser.java
--
diff --git a/repository/src/main/java/org/apache/atlas/query/QueryParser.java 
b/repository/src/main/java/org/apache/atlas/query/QueryParser.java
index be2d9b1..1e5e5ff 100644
--- a/repository/src/main/java/org/apache/atlas/query/QueryParser.java
+++ b/repository/src/main/java/org/apache/atlas/query/QueryParser.java
@@ -19,11 +19,20 @@ package org.apache.atlas.query;
 
 import org.apache.atlas.query.Expressions.Expression;
 
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 
 
 public class QueryParser {
+private static final Set RESERVED_KEYWORDS =
+new HashSet<>(Arrays.asList("[", "]", "(", ")", "=", "<", ">", 
"!=", "<=", ">=", ",", "and", "or", "+", "-",
+"*", "/", ".", "select", "from", 
"where", "groupby", "loop", "isa", "is", "has",
+"as", "times", "withPath", "limit", 
"offset", "orderby", "count", "max", "min",
+"sum", "by", "order", "like"));
+
 public static boolean isKeyword(String word) {
-return false; // TODO:
+return RESERVED_KEYWORDS.contains(word);
 }
 
 public static Expression apply(String queryStr, QueryParams params) {

http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
--
diff --git 
a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
 
b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
new file mode 100644
index 000..b70b181
--- /dev/null
+++ 
b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *

[3/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

2017-11-09 Thread madhan

http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
--
diff --git 
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
 
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
new file mode 100644
index 000..fd1b6db
--- /dev/null
+++ 
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
@@ -0,0 +1,1043 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v1;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.model.instance.EntityMutations;
+import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import 
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasMapType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME;
+import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
+import static org.apache.atlas.TestUtilsV2.NAME;
+import static org.apache.atlas.TestUtilsV2.randomString;
+import static org.apache.atlas.TestUtilsV2.STORAGE_DESC_TYPE;
+import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class AtlasEntityStoreV1Test {
+private static final Logger LOG = 
LoggerFactory.getLogger(AtlasEntityStoreV1Test.class);
+
+@Inject
+AtlasTypeRegistry typeRegistry;
+
+@Inject
+AtlasTypeDefStore typeDefStore;
+
+AtlasEntityStore entityStore;
+
+@Inject
+DeleteHandlerV1 deleteHandler;
+
+private AtlasEntitiesWithExtInfo deptEntity;
+private AtlasEntityWithExtInfo   dbEntity;
+

[1/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

2017-11-09 Thread madhan

Repository: atlas
Updated Branches:
  refs/heads/ATLAS-2251 5f40541b5 -> 042fc5575


http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java
--
diff --git a/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java 
b/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java
new file mode 100644
index 000..0d4af1e
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.util;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.*;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.converters.TypeConverterUtil;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.repository.store.graph.v1.AtlasStructDefStoreV1;
+import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.v1.model.typedef.*;
+import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Validates that conversion from V1 to legacy types (and back) is consistent. 
 This also tests
+ * that the conversion logic in AtlasStructDefStoreV1 is consistent with the 
conversion logic
+ * in RestUtils.  This tests particularly focuses on composite attributes, 
since a defect was
+ * found in that area.
+ */
+public class RestUtilsTest {
+
+@Test(enabled=false)
+// FIXME: On conversion back to V1, reverse attribute name
+// "containingDatabase"
+// in tables attribute in "database" type is lost.  See ATLAS-1528.
+public void testBidirectonalCompositeMappingConsistent() throws 
AtlasBaseException {
+
+ClassTypeDefinition dbV1Type = 
TypesUtil.createClassTypeDef("database", "", Collections.emptySet(),
+   new AttributeDefinition("tables", 
AtlasBaseTypeDef.getArrayTypeName("table"),
+   
Multiplicity.OPTIONAL, true, "containingDatabase"));
+
+ClassTypeDefinition tableV1Type = 
TypesUtil.createClassTypeDef("table", "", Collections.emptySet(),
+new 
AttributeDefinition("containingDatabase", "database",
+
Multiplicity.OPTIONAL, false, "tables"));
+
+testV1toV2toV1Conversion(Arrays.asList(dbV1Type, tableV1Type), new 
boolean[] { true, false });
+}
+
+@Test(enabled=false)
+// FIXME: On conversion back to V1, reverse attribute name
+// "containingDatabase" is lost
+// in "table" attribute in "database".  See ATLAS-1528.
+public void testBidirectonalNonCompositeMappingConsistent() throws 
AtlasBaseException {
+ClassTypeDefinition dbV1Type = 
TypesUtil.createClassTypeDef("database", "", Collections.emptySet(),
+new AttributeDefinition("tables", 
AtlasBaseTypeDef.getArrayTypeName("table"),
+
Multiplicity.OPTIONAL, false, "containingDatabase"));
+
+

[4/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

2017-11-09 Thread madhan

http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
--
diff --git 
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
 
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
new file mode 100644
index 000..1c1c68f
--- /dev/null
+++ 
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.testng.Assert;
+import org.testng.ITestContext;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.List;
+
+import static 
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class ZipSourceTest {
+@DataProvider(name = "zipFileStocks")
+public static Object[][] getDataFromZipFile() throws IOException {
+FileInputStream fs = 
ZipFileResourceTestUtils.getFileInputStream("stocks.zip");
+
+return new Object[][] {{ new ZipSource(fs) }};
+}
+
+@DataProvider(name = "zipFileStocksFloat")
+public static Object[][] getDataFromZipFileWithLongFloats() throws 
IOException {
+FileInputStream fs = 
ZipFileResourceTestUtils.getFileInputStream("stocks-float.zip");
+
+return new Object[][] {{ new ZipSource(fs) }};
+}
+
+@DataProvider(name = "sales")
+public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext 
context) throws IOException {
+return getZipSource("sales-v1-full.zip");
+}
+
+@Test
+public void improperInit_ReturnsNullCreationOrder() throws IOException, 
AtlasBaseException {
+byte bytes[] = new byte[10];
+ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ZipSource zs = new ZipSource(bais);
+List s = zs.getCreationOrder();
+Assert.assertNull(s);
+}
+
+@Test(dataProvider = "zipFileStocks")
+public void examineContents_BehavesAsExpected(ZipSource zipSource) throws 
IOException, AtlasBaseException {
+List creationOrder = zipSource.getCreationOrder();
+
+assertNotNull(creationOrder);
+assertEquals(creationOrder.size(), 4);
+
+AtlasTypesDef typesDef = zipSource.getTypesDef();
+assertNotNull(typesDef);
+assertEquals(typesDef.getEntityDefs().size(), 6);
+
+useCreationOrderToFetchEntitiesWithExtInfo(zipSource, creationOrder);
+useCreationOrderToFetchEntities(zipSource, creationOrder);
+attemptToFetchNonExistentGuid_ReturnsNull(zipSource, 
"non-existent-guid");
+verifyGuidRemovalOnImportComplete(zipSource, creationOrder.get(0));
+}
+
+private void useCreationOrderToFetchEntities(ZipSource zipSource, 
List creationOrder) {
+for (String guid : creationOrder) {
+AtlasEntity e = zipSource.getByGuid(guid);
+assertNotNull(e);
+}
+}
+
+private void verifyGuidRemovalOnImportComplete(ZipSource zipSource, String 
guid) {
+AtlasEntity e = zipSource.getByGuid(guid);
+assertNotNull(e);
+
+zipSource.onImportComplete(guid);
+
+e = zipSource.getByGuid(guid);
+Assert.assertNull(e);
+}
+
+private void attemptToFetchNonExistentGuid_ReturnsNull(ZipSource 
zipSource, String guid) {
+AtlasEntity e = zipSource.getByGuid(guid);
+Assert.assertNull(e);
+}
+
+private void useCreationOrderToFetchEntitiesWithExtInfo(ZipSource 
zipSource, List creationOrder) throws AtlasBaseException {
+for (String guid : creationOrder) {
+

[2/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

2017-11-09 Thread madhan

http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
--
diff --git 
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
 
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
new file mode 100644
index 000..d207a69
--- /dev/null
+++ 
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
@@ -0,0 +1,623 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v1;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.AtlasRelatedObjectId;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import 
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.atlas.TestRelationshipUtilsV2.EMPLOYEE_TYPE;
+import static 
org.apache.atlas.TestRelationshipUtilsV2.getDepartmentEmployeeInstances;
+import static 
org.apache.atlas.TestRelationshipUtilsV2.getDepartmentEmployeeTypes;
+import static 
org.apache.atlas.TestRelationshipUtilsV2.getInverseReferenceTestTypes;
+import static org.apache.atlas.TestUtilsV2.NAME;
+import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId;
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public abstract class AtlasRelationshipStoreV1Test {
+
+@Inject
+AtlasTypeRegistry typeRegistry;
+
+@Inject
+AtlasTypeDefStore typeDefStore;
+
+@Inject
+DeleteHandlerV1   deleteHandler;
+
+@Inject
+EntityGraphMapper graphMapper;
+
+AtlasEntityStore  entityStore;
+AtlasRelationshipStorerelationshipStore;
+AtlasEntityChangeNotifier mockChangeNotifier = 
mock(AtlasEntityChangeNotifier.class);
+
+protected Map employeeNameIdMap = new HashMap<>();
+
+@BeforeClass
+public void setUp() throws Exception {
+new GraphBackedSearchIndexer(typeRegistry);
+
+// create employee relationship types
+AtlasTypesDef employeeTypes = getDepartmentEmployeeTypes();
+typeDefStore.createTypesDef(employeeTypes);
+
+AtlasEntitiesWithExtInfo employeeInstances = 
getDepartmentEmployeeInstances();
+EntityMutationResponse response = entityStore.createOrUpdate(new 
AtlasEntityStream(employeeInstances), false);
+
+for (AtlasEntityHeader entityHeader :

atlas git commit: ATLAS-2251: restoring deleted InMemory and Noop EntityAuditRepository

2017-11-09 Thread madhan

Repository: atlas
Updated Branches:
  refs/heads/ATLAS-2251 c2b9846dd -> 5f40541b5


ATLAS-2251: restoring deleted InMemory and Noop EntityAuditRepository


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/5f40541b
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/5f40541b
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/5f40541b

Branch: refs/heads/ATLAS-2251
Commit: 5f40541b5345c94804b19d4a085e30e73c8ee83e
Parents: c2b9846
Author: Madhan Neethiraj 
Authored: Thu Nov 9 19:41:51 2017 -0800
Committer: Madhan Neethiraj 
Committed: Thu Nov 9 19:41:51 2017 -0800

--
 .../test/java/org/apache/atlas/TestUtilsV2.java |   4 +
 .../audit/InMemoryEntityAuditRepository.java|  84 +++
 .../audit/NoopEntityAuditRepository.java|  63 +++
 .../audit/AuditRepositoryTestBase.java  | 107 +++
 .../audit/InMemoryAuditRepositoryTest.java  |  28 +
 5 files changed, 286 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/5f40541b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
--
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java 
b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index 5564955..bbccf77 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -836,6 +836,10 @@ public final class TestUtilsV2 {
 return RandomStringUtils.randomAlphanumeric(10);
 }
 
+public static final String randomString(int count) {
+return RandomStringUtils.randomAlphanumeric(count);
+}
+
 public static AtlasEntity createDBEntity() {
 String dbName = RandomStringUtils.randomAlphanumeric(10);
 return createDBEntity(dbName);

http://git-wip-us.apache.org/repos/asf/atlas/blob/5f40541b/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
 
b/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
new file mode 100644
index 000..22d2a81
--- /dev/null
+++ 
b/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.audit;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.EntityAuditEvent;
+import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Singleton;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * Entity audit repository where audit events are stored in-memory. Used only 
for integration tests
+ */
+@Singleton
+@Component
+@ConditionalOnAtlasProperty(property = "atlas.EntityAuditRepository.impl")
+public class InMemoryEntityAuditRepository implements EntityAuditRepository {
+private TreeMap auditEvents = new TreeMap<>();
+
+@Override
+public void putEvents(EntityAuditEvent... events) throws AtlasException {
+putEvents(Arrays.asList(events));
+}
+
+@Override
+public synchronized void putEvents(List events) throws 
AtlasException {
+for (EntityAuditEvent event : events) {
+String rowKey = event.getEntityId() + (Long.MAX_VALUE - 
event.getTimestamp());
+event.setEventKey(rowKey);
+auditEvents.put(rowKey, event);
+}
+}
+
+//synchronized to avoid concurrent modification exception that occurs if 
events are added
+//while we are iterating through the map
+@Override
+public synchronized List listEvents(String entityId, 
String startKey, short

atlas git commit: ATLAS-2251: Added copy constructor, equals, hashCode for new model classes (schema fixes #3)

2017-11-09 Thread apoorvnaik

Repository: atlas
Updated Branches:
  refs/heads/ATLAS-2251 f3835b647 -> 2e24da428


ATLAS-2251: Added copy constructor, equals, hashCode for new model classes 
(schema fixes #3)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/2e24da42
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/2e24da42
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/2e24da42

Branch: refs/heads/ATLAS-2251
Commit: 2e24da428e72993be3b005be13c35e5afdc41ef7
Parents: f3835b6
Author: apoorvnaik 
Authored: Thu Nov 9 16:27:14 2017 -0800
Committer: apoorvnaik 
Committed: Thu Nov 9 16:27:14 2017 -0800

--
 .../model/lineage/DataSetLineageResponse.java   | 22 
 .../atlas/v1/model/lineage/LineageResponse.java | 22 
 .../atlas/v1/model/lineage/SchemaResponse.java  | 21 +++
 3 files changed, 65 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/2e24da42/intg/src/main/java/org/apache/atlas/v1/model/lineage/DataSetLineageResponse.java
--
diff --git 
a/intg/src/main/java/org/apache/atlas/v1/model/lineage/DataSetLineageResponse.java
 
b/intg/src/main/java/org/apache/atlas/v1/model/lineage/DataSetLineageResponse.java
index e6d4815..b073bac 100644
--- 
a/intg/src/main/java/org/apache/atlas/v1/model/lineage/DataSetLineageResponse.java
+++ 
b/intg/src/main/java/org/apache/atlas/v1/model/lineage/DataSetLineageResponse.java
@@ -21,6 +21,8 @@ import org.codehaus.jackson.annotate.JsonAutoDetect;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 
+import java.util.Objects;
+
 import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
 import static 
org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
 
@@ -33,6 +35,11 @@ public class DataSetLineageResponse extends LineageResponse {
 public DataSetLineageResponse() {
 }
 
+public DataSetLineageResponse(final DataSetLineageResponse other) {
+super(other);
+this.tableName = other.tableName;
+}
+
 public String getTableName() {
 return tableName;
 }
@@ -40,4 +47,19 @@ public class DataSetLineageResponse extends LineageResponse {
 public void setTableName(final String tableName) {
 this.tableName = tableName;
 }
+
+@Override
+public boolean equals(final Object o) {
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+if (!super.equals(o)) return false;
+final DataSetLineageResponse that = (DataSetLineageResponse) o;
+return Objects.equals(tableName, that.tableName);
+}
+
+@Override
+public int hashCode() {
+
+return Objects.hash(super.hashCode(), tableName);
+}
 }

http://git-wip-us.apache.org/repos/asf/atlas/blob/2e24da42/intg/src/main/java/org/apache/atlas/v1/model/lineage/LineageResponse.java
--
diff --git 
a/intg/src/main/java/org/apache/atlas/v1/model/lineage/LineageResponse.java 
b/intg/src/main/java/org/apache/atlas/v1/model/lineage/LineageResponse.java
index e86709b..aadbba1 100644
--- a/intg/src/main/java/org/apache/atlas/v1/model/lineage/LineageResponse.java
+++ b/intg/src/main/java/org/apache/atlas/v1/model/lineage/LineageResponse.java
@@ -22,6 +22,8 @@ import org.codehaus.jackson.annotate.JsonAutoDetect;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 
+import java.util.Objects;
+
 import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
 import static 
org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
 
@@ -35,6 +37,11 @@ public class LineageResponse {
 public LineageResponse() {
 }
 
+public LineageResponse(final LineageResponse other) {
+this.requestId = other.requestId;
+this.results = other.results;
+}
+
 public Struct getResults() {
 return results;
 }
@@ -50,4 +57,19 @@ public class LineageResponse {
 public void setRequestId(final String requestId) {
 this.requestId = requestId;
 }
+
+@Override
+public boolean equals(final Object o) {
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+final LineageResponse that = (LineageResponse) o;
+return Objects.equals(requestId, that.requestId) &&
+Objects.equals(results, that.results);
+}
+
+@Override
+public int hashCode() {
+
+return Objects.hash(requestId, results);
+}
 }

atlas git commit: ATLAS-2251: Lineage/DataSetLineage (schema fixes #2)

2017-11-09 Thread apoorvnaik

Repository: atlas
Updated Branches:
  refs/heads/ATLAS-2251 fe4535979 -> f3835b647


ATLAS-2251: Lineage/DataSetLineage (schema fixes #2)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f3835b64
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f3835b64
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f3835b64

Branch: refs/heads/ATLAS-2251
Commit: f3835b647d439f7fa224a32a40585e637aa553d8
Parents: fe45359
Author: apoorvnaik 
Authored: Thu Nov 9 16:17:59 2017 -0800
Committer: apoorvnaik 
Committed: Thu Nov 9 16:17:59 2017 -0800

--
 .../java/org/apache/atlas/type/AtlasTypeUtil.java| 10 ++
 .../atlas/v1/model/lineage/SchemaResponse.java   |  9 -
 pom.xml  | 15 ---
 .../apache/atlas/discovery/EntityLineageService.java |  2 --
 4 files changed, 10 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/f3835b64/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
--
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java 
b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
index 5cb74de..80c6a0c 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
@@ -17,6 +17,7 @@
  */
 package org.apache.atlas.type;
 
+import org.apache.atlas.model.instance.AtlasClassification;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
@@ -509,7 +510,7 @@ public class AtlasTypeUtil {
 if (CollectionUtils.isNotEmpty(entity.getClassifications())) {
 Map traitDetails = entity.getClassifications()
   .stream()
-  
.collect(Collectors.toMap(AtlasStruct::getTypeName, c -> 
getNestedTraitDetails(c.getTypeName(;
+  
.collect(Collectors.toMap(AtlasStruct::getTypeName, 
AtlasTypeUtil::getNestedTraitDetails));
 ret.put("$traits$", traitDetails);
 }
 
@@ -534,9 +535,10 @@ public class AtlasTypeUtil {
 return ret;
 }
 
-private static HashMap getNestedTraitDetails(final Object typeName) {
-return new HashMap() {{
-put("$typeName$", typeName);
+private static HashMap getNestedTraitDetails(final AtlasClassification 
atlasClassification) {
+return new HashMap() {{
+put("$typeName$", atlasClassification.getTypeName());
+putAll(atlasClassification.getAttributes());
 }};
 }
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/f3835b64/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
--
diff --git 
a/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java 
b/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
index 837668b..56d8226 100644
--- a/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
+++ b/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
@@ -70,21 +70,12 @@ public class SchemaResponse {
 @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public static class SchemaDetails {
-private Stringquery;  // The DSL query 
representation for obtaining schema
 private ClassTypeDefinition   dataType;   // Data type of the 
(hive) columns
 private List> rows;   // Column instances for 
the given table
 
 public SchemaDetails() {
 }
 
-public String getQuery() {
-return query;
-}
-
-public void setQuery(final String query) {
-this.query = query;
-}
-
 public ClassTypeDefinition getDataType() {
 return dataType;
 }

http://git-wip-us.apache.org/repos/asf/atlas/blob/f3835b64/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 35b0fec..fdf42a7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -800,7 +800,6 @@
 
 
 
-
 
 com.google.guava
 guava
@@ -1671,6 +1670,10 @@
 org.apache.maven.plugins
 maven-compiler-plugin
 3.7.0
+
+1.8
+

atlas git commit: ATLAS-2251: Lineage/DataSetLineage (schema fixes)

2017-11-09 Thread apoorvnaik

Repository: atlas
Updated Branches:
  refs/heads/ATLAS-2251 aa85a8ad4 -> fe4535979


ATLAS-2251: Lineage/DataSetLineage (schema fixes)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/fe453597
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/fe453597
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/fe453597

Branch: refs/heads/ATLAS-2251
Commit: fe4535979ea83c02fa5c29b038928ee8bc9b457b
Parents: aa85a8a
Author: apoorvnaik 
Authored: Thu Nov 9 14:39:14 2017 -0800
Committer: apoorvnaik 
Committed: Thu Nov 9 14:39:14 2017 -0800

--
 intg/pom.xml|   8 ++
 .../org/apache/atlas/type/AtlasTypeUtil.java| 137 ++-
 .../atlas/v1/model/lineage/LineageResponse.java |   2 +-
 .../atlas/v1/model/lineage/SchemaResponse.java  | 104 ++
 .../v1/model/typedef/AttributeDefinition.java   |  22 ++-
 .../v1/model/typedef/ClassTypeDefinition.java   |   8 +-
 .../atlas/discovery/AtlasLineageService.java|   5 +-
 .../atlas/discovery/EntityLineageService.java   |  91 ++--
 .../web/resources/DataSetLineageResource.java   |  20 +--
 .../atlas/web/resources/LineageResource.java|  16 +--
 10 files changed, 342 insertions(+), 71 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/fe453597/intg/pom.xml
--
diff --git a/intg/pom.xml b/intg/pom.xml
index 071e6c7..22895df 100644
--- a/intg/pom.xml
+++ b/intg/pom.xml
@@ -86,6 +86,14 @@
 
 
 
+
+org.apache.maven.plugins
+maven-compiler-plugin
+
+1.8
+1.8
+
+
 
 
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/fe453597/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
--
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java 
b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
index 0652855..5cb74de 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
@@ -20,13 +20,25 @@ package org.apache.atlas.type;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
-import org.apache.atlas.model.typedef.*;
+import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
 import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
 import 
org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import 
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
+import org.apache.atlas.model.typedef.AtlasTypeDefHeader;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.v1.model.typedef.AttributeDefinition;
+import org.apache.atlas.v1.model.typedef.ClassTypeDefinition;
+import org.apache.atlas.v1.model.typedef.Multiplicity;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
@@ -34,6 +46,7 @@ import org.apache.commons.lang.StringUtils;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
 
@@ -405,6 +418,128 @@ public class AtlasTypeUtil {
 return sb.toString();
 }
 
+public static ClassTypeDefinition toClassTypeDefinition(final 
AtlasEntityType entityType) {
+ClassTypeDefinition ret = null;
+
+if (entityType != null) {
+AtlasEntityDef entityDef = entityType.getEntityDef();
+ret = new ClassTypeDefinition();
+ret.setTypeName(entityDef.getName());
+ret.setTypeDescription(entityDef.getDescription());
+ret.setTypeVersion(entityDef.getTypeVersion());
+

[5/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

[3/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

[1/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

[4/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

[2/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests

atlas git commit: ATLAS-2251: restoring deleted InMemory and Noop EntityAuditRepository

atlas git commit: ATLAS-2251: Added copy constructor, equals, hashCode for new model classes (schema fixes #3)

atlas git commit: ATLAS-2251: Lineage/DataSetLineage (schema fixes #2)

atlas git commit: ATLAS-2251: Lineage/DataSetLineage (schema fixes)

9 matches

Site Navigation

Mail list logo

Footer information