Repository: incubator-atlas Updated Branches: refs/heads/master 46b1b36e9 -> 6fd04d9a8
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/GraphQueryTest.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/GraphQueryTest.java b/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/GraphQueryTest.java new file mode 100644 index 0000000..45fbbea --- /dev/null +++ b/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/GraphQueryTest.java @@ -0,0 +1,451 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graphdb.titan1; + + + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.testng.annotations.Test; + +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; + + + +/** + * Tests for Titan1GraphQuery. + */ +@Test +public class GraphQueryTest extends AbstractGraphDatabaseTest { + + + @Test + public <V, E> void testQueryThatCannotRunInMemory() throws AtlasException { + AtlasGraph<V, E> graph = getGraph(); + AtlasVertex<V, E> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + + AtlasVertex<V, E> v2 = createVertex(graph); + v2.setProperty("name", "Fred"); + + AtlasVertex<V, E> v3 = createVertex(graph); + v3.setProperty("size15", "15"); + + graph.commit(); + + AtlasVertex<V, E> v4 = createVertex(graph); + v4.setProperty("name", "Fred"); + v4.setProperty("size15", "15"); + + AtlasGraphQuery q = graph.query(); + q.has("name", ComparisionOperator.NOT_EQUAL, "George"); + q.has("size15", "15"); + graph.commit(); + pause(); //pause to let the index get updated + + assertQueryMatches(q, v1, v3, v4); + + } + + @Test + public void testCombinationOfAndsAndOrs() throws AtlasException { + Titan1Graph graph = getTitan1Graph(); + + AtlasVertex<Titan1Vertex, Titan1Edge> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + v1.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v2 = createVertex(graph); + v2.setProperty("name", "George"); + v2.setProperty("size15", "16"); + v2.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v3 = createVertex(graph); + v3.setProperty("name", "Jane"); + v3.setProperty("size15", "17"); + v3.setProperty("typeName", "Person"); + + + AtlasVertex<Titan1Vertex, Titan1Edge> v4 = createVertex(graph); + v4.setProperty("name", "Bob"); + v4.setProperty("size15", "18"); + v4.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v5 = createVertex(graph); + v5.setProperty("name", "Julia"); + v5.setProperty("size15", "19"); + v5.setProperty("typeName", "Manager"); + + + AtlasGraphQuery q = getGraphQuery(); + q.has("typeName", "Person"); + //initially match + AtlasGraphQuery inner1a = q.createChildQuery(); + AtlasGraphQuery inner1b = q.createChildQuery(); + inner1a.has("name", "Fred"); + inner1b.has("name", "Jane"); + q.or(toList(inner1a, inner1b)); + + + AtlasGraphQuery inner2a = q.createChildQuery(); + AtlasGraphQuery inner2b = q.createChildQuery(); + AtlasGraphQuery inner2c = q.createChildQuery(); + inner2a.has("size15", "18"); + inner2b.has("size15", "15"); + inner2c.has("size15", "16"); + q.or(toList(inner2a, inner2b, inner2c)); + + assertQueryMatches(q, v1); + graph.commit(); + pause(); //let the index update + assertQueryMatches(q, v1); + } + + @Test + public void testWithinStep() throws AtlasException { + Titan1Graph graph = getTitan1Graph(); + + AtlasVertex<Titan1Vertex, Titan1Edge> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + v1.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v2 = createVertex(graph); + v2.setProperty("name", "George"); + v2.setProperty("size15", "16"); + v2.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v3 = createVertex(graph); + v3.setProperty("name", "Jane"); + v3.setProperty("size15", "17"); + v3.setProperty("typeName", "Person"); + + + AtlasVertex<Titan1Vertex, Titan1Edge> v4 = createVertex(graph); + v4.setProperty("name", "Bob"); + v4.setProperty("size15", "18"); + v4.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v5 = createVertex(graph); + v5.setProperty("name", "Julia"); + v5.setProperty("size15", "19"); + v5.setProperty("typeName", "Manager"); + + + AtlasGraphQuery q = getGraphQuery(); + q.has("typeName", "Person"); + //initially match + q.in("name", toList("Fred", "Jane")); + q.in("size15", toList("18", "15", "16")); + + assertQueryMatches(q, v1); + graph.commit(); + pause(); //let the index update + assertQueryMatches(q, v1); + } + + @Test + public void testWithinStepWhereGraphIsStale() throws AtlasException { + Titan1Graph graph = getTitan1Graph(); + + AtlasVertex<Titan1Vertex, Titan1Edge> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + v1.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v2 = createVertex(graph); + v2.setProperty("name", "George"); + v2.setProperty("size15", "16"); + v2.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v3 = createVertex(graph); + v3.setProperty("name", "Jane"); + v3.setProperty("size15", "17"); + v3.setProperty("typeName", "Person"); + + + AtlasVertex<Titan1Vertex, Titan1Edge> v4 = createVertex(graph); + v4.setProperty("name", "Bob"); + v4.setProperty("size15", "18"); + v4.setProperty("typeName", "Person"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v5 = createVertex(graph); + v5.setProperty("name", "Julia"); + v5.setProperty("size15", "19"); + v5.setProperty("typeName", "Manager"); + + + AtlasGraphQuery q = getGraphQuery(); + q.has("typeName", "Person"); + //initially match + q.in("name", toList("Fred", "Jane")); + + graph.commit(); + pause(); //let the index update + assertQueryMatches(q, v1, v3); + //make v3 no longer match the query. Within step should filter out the vertex since it no longer matches. + v3.setProperty("name", "Janet"); + assertQueryMatches(q, v1); + } + + @Test + public void testSimpleOrQuery() throws AtlasException { + Titan1Graph graph = getTitan1Graph(); + + + AtlasVertex<Titan1Vertex, Titan1Edge> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v2 = createVertex(graph); + v2.setProperty("name", "Fred"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v3 = createVertex(graph); + v3.setProperty("size15", "15"); + + graph.commit(); + + AtlasVertex<Titan1Vertex, Titan1Edge> v4 = createVertex(graph); + v4.setProperty("name", "Fred"); + v4.setProperty("size15", "15"); + + AtlasVertex<Titan1Vertex, Titan1Edge> v5 = createVertex(graph); + v5.setProperty("name", "George"); + v5.setProperty("size15", "16"); + + AtlasGraphQuery q = graph.query(); + AtlasGraphQuery inner1 = q.createChildQuery().has("name", "Fred"); + AtlasGraphQuery inner2 = q.createChildQuery().has("size15", "15"); + q.or(toList(inner1, inner2)); + assertQueryMatches(q, v1, v2, v3, v4); + graph.commit(); + pause(); //pause to let the indexer get updated (this fails frequently without a pause) + assertQueryMatches(q, v1, v2, v3, v4); + } + + + + + @Test + public <V, E> void testQueryMatchesAddedVertices() throws AtlasException { + AtlasGraph<V, E> graph = getGraph(); + + AtlasVertex<V, E> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + + AtlasVertex<V, E> v2 = createVertex(graph); + v2.setProperty("name", "Fred"); + + AtlasVertex<V, E> v3 = createVertex(graph); + v3.setProperty("size15", "15"); + + graph.commit(); + + AtlasVertex<V, E> v4 = createVertex(graph); + v4.setProperty("name", "Fred"); + v4.setProperty("size15", "15"); + + AtlasGraphQuery q = getGraphQuery(); + q.has("name", "Fred"); + q.has("size15", "15"); + + assertQueryMatches(q, v1, v4); + graph.commit(); + assertQueryMatches(q, v1, v4); + + } + + + @Test + public <V, E> void testQueryDoesNotMatchRemovedVertices() throws AtlasException { + AtlasGraph<V, E> graph = getGraph(); + + AtlasVertex<V, E> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + + AtlasVertex<V, E> v2 = createVertex(graph); + v2.setProperty("name", "Fred"); + + AtlasVertex<V, E> v3 = createVertex(graph); + v3.setProperty("size15", "15"); + + AtlasVertex<V, E> v4 = createVertex(graph); + v4.setProperty("name", "Fred"); + v4.setProperty("size15", "15"); + + graph.commit(); + + graph.removeVertex(v1); + + AtlasGraphQuery q = getGraphQuery(); + q.has("name", "Fred"); + q.has("size15", "15"); + + assertQueryMatches(q, v4); + graph.commit(); + + assertQueryMatches(q, v4); + } + + @Test + public <V, E> void testQueryDoesNotMatchUncommittedAddedAndRemovedVertices() throws AtlasException { + AtlasGraph<V, E> graph = getGraph(); + + AtlasVertex<V, E> v1 = createVertex(graph); + + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + + AtlasVertex<V, E> v2 = createVertex(graph); + v2.setProperty("name", "Fred"); + + AtlasVertex<V, E> v3 = createVertex(graph); + v3.setProperty("size15", "15"); + + AtlasVertex<V, E> v4 = createVertex(graph); + v4.setProperty("name", "Fred"); + v4.setProperty("size15", "15"); + + + AtlasGraphQuery q = getGraphQuery(); + q.has("name", "Fred"); + q.has("size15", "15"); + + assertQueryMatches(q, v1, v4); + + graph.removeVertex(v1); + + + assertQueryMatches(q, v4); + graph.commit(); + + assertQueryMatches(q, v4); + } + + + @Test + public <V, E> void testQueryResultsReflectPropertyAdd() throws AtlasException { + AtlasGraph<V, E> graph = getGraph(); + + AtlasVertex<V, E> v1 = createVertex(graph); + v1.setProperty("name", "Fred"); + v1.setProperty("size15", "15"); + v1.addProperty(TRAIT_NAMES, "trait1"); + v1.addProperty(TRAIT_NAMES, "trait2"); + + AtlasVertex<V, E> v2 = createVertex(graph); + v2.setProperty("name", "Fred"); + v2.addProperty(TRAIT_NAMES, "trait1"); + + AtlasVertex<V, E> v3 = createVertex(graph); + v3.setProperty("size15", "15"); + v3.addProperty(TRAIT_NAMES, "trait2"); + + AtlasGraphQuery query = getGraphQuery(); + query.has("name", "Fred"); + query.has(TRAIT_NAMES, "trait1"); + query.has("size15", "15"); + + assertQueryMatches(query, v1); + //make v3 match the query + v3.setProperty("name", "Fred"); + v3.addProperty(TRAIT_NAMES, "trait1"); + assertQueryMatches(query, v1, v3); + v3.removeProperty(TRAIT_NAMES); + assertQueryMatches(query, v1); + v3.addProperty(TRAIT_NAMES, "trait2"); + assertQueryMatches(query, v1); + v1.removeProperty(TRAIT_NAMES); + assertQueryMatches(query); + graph.commit(); + assertQueryMatches(query); + + } + + private static <T> List<T> toList(Iterable<T> itr) { + List<T> result = new ArrayList<T>(); + for(T object : itr) { + result.add(object); + } + return result; + + } + + private <V, E> void assertQueryMatches(AtlasGraphQuery expr, AtlasVertex... expectedResults) throws AtlasException { + + //getGraph().commit(); + Collection<AtlasVertex<Titan1Vertex, Titan1Edge>> temp = toList(expr.vertices()); + //filter out vertices from previous test executions + Collection<AtlasVertex<Titan1Vertex, Titan1Edge>> result = + Collections2.filter(temp, new Predicate<AtlasVertex<Titan1Vertex, Titan1Edge>>() { + + @Override + public boolean apply(AtlasVertex<Titan1Vertex, Titan1Edge> input) { + return newVertices.contains(input); + } + + }); + String errorMessage = "Expected/found result sizes differ. Expected: " + + Arrays.asList(expectedResults).toString() +", found: " + result; + assertEquals(errorMessage, expectedResults.length, result.size()); + + for(AtlasVertex<V, E> v : expectedResults) { + assertTrue(result.contains(v)); + } + } + + private static List<Object> toList(Object...objects) { + return Arrays.asList(objects); + } + + private AtlasGraphQuery<Titan1Vertex, Titan1Edge> getGraphQuery() { + return getTitan1Graph().query(); + } + + private void pause() { + try { + Thread.sleep(5000); + } catch(InterruptedException e) { + //ignore + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/Titan1DatabaseTest.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/Titan1DatabaseTest.java b/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/Titan1DatabaseTest.java new file mode 100644 index 0000000..6db0da8 --- /dev/null +++ b/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/Titan1DatabaseTest.java @@ -0,0 +1,432 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graphdb.titan1; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasCardinality; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphManagement; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; +import org.apache.atlas.repository.graphdb.AtlasPropertyKey; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +/** + * Sanity test of basic graph operations using the Titan 0.5.4 graphdb + * abstraction layer implementation. + */ +public class Titan1DatabaseTest { + + private AtlasGraph<?, ?> atlasGraph; + + private <V, E> AtlasGraph<V, E> getGraph() { + if (atlasGraph == null) { + Titan1GraphDatabase db = new Titan1GraphDatabase(); + atlasGraph = db.getGraph(); + AtlasGraphManagement mgmt = atlasGraph.getManagementSystem(); + // create the index (which defines these properties as being mult + // many) + for (String propertyName : new String[]{"__superTypeNames", "__traitNames"}) { + AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName); + if (propertyKey == null) { + propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET); + mgmt.createExactMatchIndex(propertyName, false, Collections.singletonList(propertyKey)); + } + } + mgmt.commit(); + } + return (AtlasGraph<V, E>) atlasGraph; + } + + @AfterClass + public void cleanup() { + atlasGraph.clear(); + atlasGraph = null; + + } + + @Test + public <V, E> void testPropertyDataTypes() { + + // primitives + AtlasGraph<V, E> graph = getGraph(); + + testProperty(graph, "booleanProperty", Boolean.TRUE); + testProperty(graph, "booleanProperty", Boolean.FALSE); + testProperty(graph, "booleanProperty", new Boolean(Boolean.TRUE)); + testProperty(graph, "booleanProperty", new Boolean(Boolean.FALSE)); + + testProperty(graph, "byteProperty", Byte.MAX_VALUE); + testProperty(graph, "byteProperty", Byte.MIN_VALUE); + testProperty(graph, "byteProperty", new Byte(Byte.MAX_VALUE)); + testProperty(graph, "byteProperty", new Byte(Byte.MIN_VALUE)); + + testProperty(graph, "shortProperty", Short.MAX_VALUE); + testProperty(graph, "shortProperty", Short.MIN_VALUE); + testProperty(graph, "shortProperty", new Short(Short.MAX_VALUE)); + testProperty(graph, "shortProperty", new Short(Short.MIN_VALUE)); + + testProperty(graph, "intProperty", Integer.MAX_VALUE); + testProperty(graph, "intProperty", Integer.MIN_VALUE); + testProperty(graph, "intProperty", new Integer(Integer.MAX_VALUE)); + testProperty(graph, "intProperty", new Integer(Integer.MIN_VALUE)); + + testProperty(graph, "longProperty", Long.MIN_VALUE); + testProperty(graph, "longProperty", Long.MAX_VALUE); + testProperty(graph, "longProperty", new Long(Long.MIN_VALUE)); + testProperty(graph, "longProperty", new Long(Long.MAX_VALUE)); + + testProperty(graph, "doubleProperty", Double.MAX_VALUE); + testProperty(graph, "doubleProperty", Double.MIN_VALUE); + testProperty(graph, "doubleProperty", new Double(Double.MAX_VALUE)); + testProperty(graph, "doubleProperty", new Double(Double.MIN_VALUE)); + + testProperty(graph, "floatProperty", Float.MAX_VALUE); + testProperty(graph, "floatProperty", Float.MIN_VALUE); + testProperty(graph, "floatProperty", new Float(Float.MAX_VALUE)); + testProperty(graph, "floatProperty", new Float(Float.MIN_VALUE)); + + // enumerations - TypeCategory + testProperty(graph, "typeCategoryProperty", TypeCategory.CLASS); + + // biginteger + testProperty(graph, "bigIntegerProperty", + new BigInteger(String.valueOf(Long.MAX_VALUE)).multiply(BigInteger.TEN)); + + // bigdecimal + BigDecimal bigDecimal = new BigDecimal(Double.MAX_VALUE); + testProperty(graph, "bigDecimalProperty", bigDecimal.multiply(bigDecimal)); + } + + private <V, E> void testProperty(AtlasGraph<V, E> graph, String name, Object value) { + + AtlasVertex<V, E> vertex = graph.addVertex(); + vertex.setProperty(name, value); + assertEquals(value, vertex.getProperty(name, value.getClass())); + AtlasVertex<V, E> loaded = graph.getVertex(vertex.getId().toString()); + assertEquals(value, loaded.getProperty(name, value.getClass())); + } + + @Test + public <V, E> void testMultiplicityOnePropertySupport() { + + AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph(); + + AtlasVertex<V, E> vertex = graph.addVertex(); + vertex.setProperty("name", "Jeff"); + vertex.setProperty("location", "Littleton"); + assertEquals("Jeff", vertex.getProperty("name", String.class)); + assertEquals("Littleton", vertex.getProperty("location", String.class)); + + AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString()); + + assertEquals("Jeff", vertexCopy.getProperty("name", String.class)); + assertEquals("Littleton", vertexCopy.getProperty("location", String.class)); + + assertTrue(vertexCopy.getPropertyKeys().contains("name")); + assertTrue(vertexCopy.getPropertyKeys().contains("location")); + + assertTrue(vertexCopy.getPropertyValues("name", String.class).contains("Jeff")); + assertTrue(vertexCopy.getPropertyValues("location", String.class).contains("Littleton")); + assertTrue(vertexCopy.getPropertyValues("test", String.class).isEmpty()); + assertNull(vertexCopy.getProperty("test", String.class)); + + vertex.removeProperty("name"); + assertFalse(vertex.getPropertyKeys().contains("name")); + assertNull(vertex.getProperty("name", String.class)); + assertTrue(vertex.getPropertyValues("name", String.class).isEmpty()); + + vertexCopy = graph.getVertex(vertex.getId().toString()); + assertFalse(vertexCopy.getPropertyKeys().contains("name")); + assertNull(vertexCopy.getProperty("name", String.class)); + assertTrue(vertexCopy.getPropertyValues("name", String.class).isEmpty()); + + } + + @Test + public <V, E> void testRemoveEdge() { + + AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph(); + AtlasVertex<V, E> v1 = graph.addVertex(); + AtlasVertex<V, E> v2 = graph.addVertex(); + + AtlasEdge<V, E> edge = graph.addEdge(v1, v2, "knows"); + + // make sure the edge exists + AtlasEdge<V, E> edgeCopy = graph.getEdge(edge.getId().toString()); + assertNotNull(edgeCopy); + assertEquals(edgeCopy, edge); + + graph.removeEdge(edge); + + edgeCopy = graph.getEdge(edge.getId().toString()); + // should return null now, since edge was deleted + assertNull(edgeCopy); + + } + + @Test + public <V, E> void testRemoveVertex() { + + AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph(); + + AtlasVertex<V, E> v1 = graph.addVertex(); + + assertNotNull(graph.getVertex(v1.getId().toString())); + + graph.removeVertex(v1); + + assertNull(graph.getVertex(v1.getId().toString())); + } + + @Test + public <V, E> void testGetEdges() { + + AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph(); + AtlasVertex<V, E> v1 = graph.addVertex(); + AtlasVertex<V, E> v2 = graph.addVertex(); + AtlasVertex<V, E> v3 = graph.addVertex(); + + AtlasEdge<V, E> knows = graph.addEdge(v2, v1, "knows"); + AtlasEdge<V, E> eats = graph.addEdge(v3, v1, "eats"); + AtlasEdge<V, E> drives = graph.addEdge(v3, v2, "drives"); + AtlasEdge<V, E> sleeps = graph.addEdge(v2, v3, "sleeps"); + + assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.IN), knows, eats); + assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.OUT)); + assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.BOTH), knows, eats); + + assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.IN, "knows"), knows); + assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.OUT, "knows")); + assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.BOTH, "knows"), knows); + + assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.IN), drives); + assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.OUT), knows, sleeps); + assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.BOTH), knows, sleeps, drives); + + assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.BOTH, "delivers")); + } + + private <V, E> void assertEdgesMatch(Iterable<AtlasEdge<V, E>> edgesIt, AtlasEdge<V, E>... expected) { + List<AtlasEdge<V, E>> edges = toList(edgesIt); + assertEquals(expected.length, edges.size()); + for (AtlasEdge<V, E> edge : expected) { + assertTrue(edges.contains(edge)); + } + } + + @Test + public <V, E> void testMultiplictyManyPropertySupport() { + + AtlasGraph<V, E> graph = getGraph(); + + AtlasVertex<V, E> vertex = graph.addVertex(); + String vertexId = vertex.getId().toString(); + vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1"); + vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2"); + assertEquals(vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class).size(), 2); + vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait3"); + vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait4"); + + assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY)); + validateMultManyPropertiesInVertex(vertex); + // fetch a copy of the vertex, make sure result + // is the same + + validateMultManyPropertiesInVertex(graph.getVertex(vertexId)); + + } + + private <V, E> void validateMultManyPropertiesInVertex(AtlasVertex<V, E> vertex) { + + assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY)); + Collection<String> traitNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class); + assertTrue(traitNames.contains("trait1")); + assertTrue(traitNames.contains("trait2")); + assertTrue(traitNames.contains("trait3")); + assertTrue(traitNames.contains("trait4")); + + try { + vertex.getProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class); + fail("Expected exception not thrown"); + } catch (IllegalStateException expected) { + // multiple property values exist + } + } + + @Test + public <V, E> void testListProperties() throws AtlasException { + + AtlasGraph<V, E> graph = getGraph(); + AtlasVertex<V, E> vertex = graph.addVertex(); + List<String> colorsToSet = new ArrayList<String>(); + colorsToSet.add("red"); + colorsToSet.add("blue"); + colorsToSet.add("green"); + vertex.setListProperty("colors", colorsToSet); + List<String> colors = vertex.getListProperty("colors"); + assertTrue(colors.contains("red")); + assertTrue(colors.contains("blue")); + assertTrue(colors.contains("green")); + + AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString()); + colors = vertexCopy.getListProperty("colors"); + assertTrue(colors.contains("red")); + assertTrue(colors.contains("blue")); + assertTrue(colors.contains("green")); + + } + + @Test + public <V, E> void testRemoveProperty() { + + AtlasGraph<V, E> graph = getGraph(); + AtlasVertex<V, E> vertex = graph.addVertex(); + vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1"); + vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1"); + vertex.setProperty("name", "Jeff"); + + // remove existing property - multiplicity one + vertex.removeProperty("jeff"); + + assertFalse(vertex.getPropertyKeys().contains("jeff")); + + // remove existing property - multiplicity many + vertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY); + assertFalse(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY)); + + AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString()); + assertFalse(vertexCopy.getPropertyKeys().contains("jeff")); + assertFalse(vertexCopy.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY)); + + // remove non-existing property + vertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY); + vertex.removeProperty("jeff"); + + } + + @Test + public <V, E> void getGetGraphQueryForVertices() { + + AtlasGraph<V, E> graph = getGraph(); + + AtlasVertex<V, E> v1 = graph.addVertex(); + AtlasVertex<V, E> v2 = graph.addVertex(); + AtlasVertex<V, E> v3 = graph.addVertex(); + + v1.setProperty("name", "Jeff"); + v1.setProperty("weight", 1); + + v2.setProperty("name", "Fred"); + v2.setProperty("weight", 2); + + v3.setProperty("name", "Chris"); + v3.setProperty("weight", 3); + + AtlasEdge<V, E> knows = graph.addEdge(v2, v1, "knows"); + knows.setProperty("weight", 1); + AtlasEdge<V, E> eats = graph.addEdge(v3, v1, "eats"); + eats.setProperty("weight", 2); + AtlasEdge<V, E> drives = graph.addEdge(v3, v2, "drives"); + drives.setProperty("weight", 3); + + AtlasEdge<V, E> sleeps = graph.addEdge(v2, v3, "sleeps"); + sleeps.setProperty("weight", 4); + + testExecuteGraphQuery("name", null, "Jeff", v1); + testExecuteGraphQuery("weight", ComparisionOperator.EQUAL, 2, v2); + testExecuteGraphQuery("weight", ComparisionOperator.GREATER_THAN_EQUAL, 2, v2, v3); + testExecuteGraphQuery("weight", ComparisionOperator.LESS_THAN_EQUAL, 2, v2, v1); + + } + + private <V, E> void testExecuteGraphQuery(String property, ComparisionOperator op, Object value, + AtlasVertex<V, E>... expected) { + AtlasGraph<V, E> graph = getGraph(); + AtlasGraphQuery<V, E> query = graph.query(); + if (op != null) { + query.has(property, op, value); + } else { + query.has(property, value); + } + Iterable<? extends AtlasVertex<V, E>> result = query.vertices(); + List<AtlasVertex<V, E>> list = toList(result); + assertEquals(expected.length, list.size()); + for (AtlasVertex<V, E> vertex : expected) { + assertTrue(list.contains(vertex)); + } + } + + @Test + public <V, E> void testAddMultManyPropertyValueTwice() { + + AtlasGraph<V, E> graph = getGraph(); + String vertexId; + + AtlasVertex<V, E> vertex = graph.addVertex(); + vertexId = vertex.getId().toString(); + vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1"); + vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1"); + vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2"); + vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2"); + + validateDuplicatePropertyVertex(vertex); + + // fetch a copy of the vertex, make sure result is the same + + validateDuplicatePropertyVertex(graph.getVertex(vertexId)); + } + + private <V, E> void validateDuplicatePropertyVertex(AtlasVertex<V, E> vertex) { + assertEquals(2, vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class).size()); + assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY)); + Collection<String> traitNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class); + assertTrue(traitNames.contains("trait1")); + assertTrue(traitNames.contains("trait2")); + } + + private static <T> List<T> toList(Iterable<? extends T> iterable) { + List<T> result = new ArrayList<T>(); + for (T item : iterable) { + result.add(item); + } + return result; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/TitanGraphProviderTest.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/TitanGraphProviderTest.java b/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/TitanGraphProviderTest.java new file mode 100644 index 0000000..6b255f0 --- /dev/null +++ b/graphdb/titan1/src/test/java/org/apache/atlas/repository/graphdb/titan1/TitanGraphProviderTest.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graphdb.titan1; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.commons.configuration.Configuration; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +@Test +public class TitanGraphProviderTest { + + private Configuration configuration; + private AtlasGraph<?, ?> graph; + + @BeforeTest + public void setUp() throws AtlasException { + //First get Instance + graph = new Titan1Graph(); + configuration = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(), + Titan1GraphDatabase.GRAPH_PREFIX); + } + + @AfterClass + public void tearDown() throws Exception { + try { + graph.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + + try { + graph.clear(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testValidate() throws AtlasException { + try { + Titan1GraphDatabase.validateIndexBackend(configuration); + } catch (Exception e) { + Assert.fail("Unexpected exception ", e); + } + + //Change backend + configuration.setProperty(Titan1GraphDatabase.INDEX_BACKEND_CONF, Titan1GraphDatabase.INDEX_BACKEND_LUCENE); + try { + Titan1GraphDatabase.validateIndexBackend(configuration); + Assert.fail("Expected exception"); + } catch (Exception e) { + Assert.assertEquals(e.getMessage(), + "Configured Index Backend lucene differs from earlier configured " + + "Index Backend elasticsearch. Aborting!"); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/graphdb/titan1/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/test/resources/atlas-application.properties b/graphdb/titan1/src/test/resources/atlas-application.properties new file mode 100644 index 0000000..99fe18a --- /dev/null +++ b/graphdb/titan1/src/test/resources/atlas-application.properties @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +######### Graph Database to Use ######### +atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan1.Titan1Database + +######### Atlas Server Configs ######### +atlas.rest.address=http://localhost:31000 + +######### Graph Database Configs ######### +# Graph Storage +atlas.graph.storage.backend=${titan.storage.backend} + +# Graph Search Index Backend +atlas.graph.index.search.backend=${titan.index.backend} + +#Berkeley storage directory +atlas.graph.storage.directory=${sys:atlas.data}/berkley + +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 + +atlas.graph.storage.hostname=${titan.storage.hostname} +atlas.graph.storage.hbase.regions-per-server=1 +atlas.graph.storage.lock.wait-time=10000 + +#ElasticSearch +atlas.graph.index.search.directory=${sys:atlas.data}/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +# Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} + + +######### Hive Lineage Configs ######### +# This models reflects the base super types for Data and Process +#atlas.lineage.hive.table.type.name=DataSet +#atlas.lineage.hive.process.type.name=Process +#atlas.lineage.hive.process.inputs.name=inputs +#atlas.lineage.hive.process.outputs.name=outputs + +## Schema +atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns + +######### Notification Configs ######### +atlas.notification.embedded=true + +atlas.kafka.zookeeper.connect=localhost:19026 +atlas.kafka.bootstrap.servers=localhost:19027 +atlas.kafka.data=${sys:atlas.data}/kafka +atlas.kafka.zookeeper.session.timeout.ms=4000 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.consumer.timeout.ms=100 +atlas.kafka.auto.commit.interval.ms=100 +atlas.kafka.hook.group.id=atlas +atlas.kafka.entities.group.id=atlas_entities + +######### Entity Audit Configs ######### +atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS +atlas.audit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=localhost +atlas.audit.hbase.zookeeper.property.clientPort=19026 + +######### Security Properties ######### + +# SSL config +atlas.enableTLS=false +atlas.server.https.port=31443 + +######### Security Properties ######### + +hbase.security.authentication=simple + +atlas.hook.falcon.synchronous=true +######### High Availability Configuration ######## +atlas.server.ha.enabled=false +#atlas.server.ids=id1 +#atlas.server.address.id1=localhost:21000 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 798e965..13f6fe9 100644 --- a/pom.xml +++ b/pom.xml @@ -490,7 +490,7 @@ <titan.storage.backend>berkeleyje</titan.storage.backend> <titan.index.backend>elasticsearch</titan.index.backend> <entity.repository.impl>org.apache.atlas.repository.audit.InMemoryEntityAuditRepository</entity.repository.impl> - <graphdb.backend.impl>org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase</graphdb.backend.impl> + <graphdb.backend.impl>org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase</graphdb.backend.impl> <atlas.surefire.options></atlas.surefire.options> <aspectj.runtime.version>1.8.7</aspectj.runtime.version> @@ -533,6 +533,28 @@ <skipDocs>false</skipDocs> </properties> </profile> + <profile> + <id>titan1</id> + <properties> + <graphdb.backend.impl>org.apache.atlas.repository.graphdb.titan1.Titan1GraphDatabase</graphdb.backend.impl> + </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-graphdb-impls</artifactId> + <version>${project.version}</version> + <type>pom</type> + <exclusions> + <exclusion> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-graphdb-titan0</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </dependencyManagement> + </profile> </profiles> <modules> @@ -1251,9 +1273,14 @@ <artifactId>atlas-graphdb-impls</artifactId> <version>${project.version}</version> <type>pom</type> - <!-- exclusions should be added here for all of the non-titan0 implementations --> - <scope>test</scope> - </dependency> + <!-- exclusions for all of the non-titan0 implementations --> + <exclusions> + <exclusion> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-graphdb-titan1</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>org.apache.atlas</groupId> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b9d391f..d25e462 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-695: Add Titan 1.0.0 support to Atlas (guptaneeru via jnhagelberg) ATLAS-1552: automatic update of inverse references in V2 code path (dkantor) ATLAS-1603: fix to handle null value for object_id type attributes (mneethiraj via kevalbhatt) ATLAS 1607: notify listeners on classification addition/deletion (sarathkumarsubramanian via mneethiraj) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index e5abd77..f09aa5a 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -53,6 +53,7 @@ import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.GremlinVersion; import org.apache.atlas.repository.typestore.GraphBackedTypeStore; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.services.DefaultMetadataService; @@ -85,6 +86,7 @@ import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; import org.codehaus.jettison.json.JSONArray; import org.testng.Assert; +import org.testng.SkipException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -778,4 +780,12 @@ public final class TestUtils { checker.removeAll(actual); assertEquals(checker.size(), 0); } + + public static void skipForGremlin3EnabledGraphDb() throws SkipException { + //ATLAS-1579 Currently, some tests are skipped for titan1 backened. As these tests are hard coded to use Gremlin2. See ATLAS-1579, ATLAS-1591 once it is fixed, please remove it. + if (TestUtils.getGraph().getSupportedGremlinVersion() == GremlinVersion.THREE) { + throw new SkipException ("This test requires Gremlin2. Skipping test "); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index a90543e..120187d 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -49,6 +49,7 @@ import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; +import org.apache.atlas.repository.graphdb.GremlinVersion; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; @@ -63,6 +64,7 @@ import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.testng.Assert; +import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -275,6 +277,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { @Test public void testRawSearch1() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); // Query for all Vertices in Graph Object r = discoveryService.searchByGremlin("g.V.toList()"); Assert.assertTrue(r instanceof List); @@ -398,7 +401,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db has name", 3}, {"hive_db, hive_table", 10}, {"View is JdbcAccess", 2}, - {"hive_db as db1, hive_table where db1.name = \"Reporting\"", 0}, //Not working - ATLAS-145 + {"hive_db as db1, hive_table where db1.name = \"Reporting\"", isGremlin3() ? 4 : 0}, //Not working in with Titan 0 - ATLAS-145 // - Final working query -> discoveryService.searchByGremlin("L:{_var_0 = [] as Set;g.V().has(\"__typeName\", \"hive_db\").fill(_var_0);g.V().has(\"__superTypeNames\", \"hive_db\").fill(_var_0);_var_0._().as(\"db1\").in(\"__hive_table.db\").back(\"db1\").and(_().has(\"hive_db.name\", T.eq, \"Reporting\")).toList()}") /* {"hive_db, hive_process has name"}, //Invalid query @@ -419,7 +422,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db where hive_db is JdbcAccess", 0}, //Not supposed to work {"hive_db hive_table", 10}, {"hive_db where hive_db has name", 3}, - {"hive_db as db1 hive_table where (db1.name = \"Reporting\")", 0}, //Not working -> ATLAS-145 + {"hive_db as db1 hive_table where (db1.name = \"Reporting\")", isGremlin3() ? 4 : 0}, //Not working in Titan 0 -> ATLAS-145 {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ", 1}, {"hive_table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 ", 1}, {"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1 ", 1}, @@ -528,7 +531,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"View is JdbcAccess", 2}, {"View is JdbcAccess limit 1", 1}, {"View is JdbcAccess limit 2 offset 1", 1}, - {"hive_db as db1, hive_table where db1.name = \"Reporting\"", 0}, //Not working - ATLAS-145 + {"hive_db as db1, hive_table where db1.name = \"Reporting\"", isGremlin3() ? 4 : 0}, //Not working in Titan 0 - ATLAS-145 {"from hive_table", 10}, @@ -586,7 +589,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db where hive_db has name limit 2 offset 0", 2}, {"hive_db where hive_db has name limit 2 offset 1", 2}, - {"hive_db as db1 hive_table where (db1.name = \"Reporting\")", 0}, //Not working -> ATLAS-145 + {"hive_db as db1 hive_table where (db1.name = \"Reporting\")", isGremlin3() ? 4 : 0}, //Not working in Titan 0 -> ATLAS-145 {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 limit 10", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 limit 10 offset 1", 0}, @@ -1252,4 +1255,8 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { private FieldValueValidator makeNoResultsValidator() { return new FieldValueValidator(); } + + private boolean isGremlin3() { + return TestUtils.getGraph().getSupportedGremlinVersion() == GremlinVersion.THREE; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java index 19124d7..6a1979a 100644 --- a/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java @@ -18,18 +18,30 @@ package org.apache.atlas.lineage; -import com.google.common.collect.ImmutableList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.BaseRepositoryTest; import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.TestUtils; import org.apache.atlas.discovery.EntityLineageService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.lineage.AtlasLineageInfo; -import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; +import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.persistence.Id; import org.apache.commons.collections.ArrayStack; @@ -37,17 +49,12 @@ import org.apache.commons.lang.RandomStringUtils; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Guice; import org.testng.annotations.Test; -import javax.inject.Inject; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.testng.Assert.*; +import com.google.common.collect.ImmutableList; /** * Unit tests for the new v2 Instance LineageService. @@ -73,6 +80,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest { */ @Test public void testCircularLineage() throws Exception{ + TestUtils.skipForGremlin3EnabledGraphDb(); String entityGuid = getEntityId(HIVE_TABLE_TYPE, "name", "table2"); AtlasLineageInfo circularLineage = getInputLineageInfo(entityGuid, 5); @@ -108,6 +116,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest { @Test public void testGetInputLineageInfo() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); String entityGuid = getEntityId(HIVE_TABLE_TYPE, "name", "sales_fact_monthly_mv"); AtlasLineageInfo inputLineage = getInputLineageInfo(entityGuid, 4); @@ -143,6 +152,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest { @Test public void testGetOutputLineageInfo() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); String entityGuid = getEntityId(HIVE_TABLE_TYPE, "name", "sales_fact"); AtlasLineageInfo outputLineage = getOutputLineageInfo(entityGuid, 4); @@ -178,6 +188,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest { @Test public void testGetLineageInfo() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); String entityGuid = getEntityId(HIVE_TABLE_TYPE, "name", "sales_fact_monthly_mv"); AtlasLineageInfo bothLineage = getBothLineageInfo(entityGuid, 5); @@ -241,6 +252,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest { @Test public void testNewLineageWithDelete() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); String tableName = "table" + random(); createTable(tableName, 3, true); String entityGuid = getEntityId(HIVE_TABLE_TYPE, "name", tableName); @@ -344,4 +356,5 @@ public class EntityLineageServiceTest extends BaseRepositoryTest { private String getEntityId(String typeName, String attributeName, String attributeValue) throws Exception { return repository.getEntityDefinition(typeName, attributeName, attributeValue).getId()._getId(); } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java index 3d17ca9..5bde605 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.audit; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.TestUtils; import org.apache.atlas.typesystem.Referenceable; import org.apache.commons.lang.RandomStringUtils; import org.testng.annotations.BeforeTest; @@ -44,6 +45,7 @@ public class AuditRepositoryTestBase { @Test public void testAddEvents() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); EntityAuditEvent event = new EntityAuditEvent(rand(), System.currentTimeMillis(), "u1", EntityAuditEvent.EntityAuditAction.ENTITY_CREATE, "d1", new Referenceable(rand())); @@ -57,6 +59,7 @@ public class AuditRepositoryTestBase { @Test public void testListPagination() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); String id1 = "id1" + rand(); String id2 = "id2" + rand(); String id3 = "id3" + rand(); @@ -90,6 +93,7 @@ public class AuditRepositoryTestBase { @Test public void testInvalidEntityId() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); List<EntityAuditEvent> events = eventRepository.listEvents(rand(), null, (short) 3); assertEquals(events.size(), 0); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java index 79550ba..a9dfee3 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java @@ -20,6 +20,7 @@ package org.apache.atlas.repository.audit; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.TestUtils; import org.apache.commons.configuration.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -36,6 +37,9 @@ public class HBaseBasedAuditRepositoryTest extends AuditRepositoryTestBase { @BeforeClass public void setup() throws Exception { + //ATLAS-1591 Currently, some tests are skipped for titan1 backened. As these tests are hard coded to use Gremlin2. See ATLAS-1591 once it is fixed, please remove it. + + TestUtils.skipForGremlin3EnabledGraphDb(); eventRepository = new HBaseBasedAuditRepository(); HBaseTestUtils.startCluster(); ((HBaseBasedAuditRepository) eventRepository).start(); @@ -54,6 +58,7 @@ public class HBaseBasedAuditRepositoryTest extends AuditRepositoryTestBase { @Test public void testTableCreated() throws Exception { + TestUtils.skipForGremlin3EnabledGraphDb(); Connection connection = HBaseTestUtils.getConnection(); Admin admin = connection.getAdmin(); assertTrue(admin.tableExists(tableName)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6fd04d9a/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index e7dce78..30b88ff 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -34,8 +34,8 @@ <properties> <projectBaseDir>${project.basedir}/..</projectBaseDir> <debug.jetty.daemon>true</debug.jetty.daemon> + <log4j.configuration.url>file:///${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</log4j.configuration.url> <packages.to.exclude /> - <log4j.configuration.url>file://${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</log4j.configuration.url> </properties> <profiles> @@ -46,7 +46,7 @@ <activeByDefault>false</activeByDefault> </activation> <properties> - <packages.to.exclude>WEB-INF/lib/titan-berkeleyje-${titan.version}.jar,WEB-INF/lib/je-*.jar,WEB-INF/lib/titan-es-${titan.version}.jar,WEB-INF/lib/elasticsearch-*.jar</packages.to.exclude> + <packages.to.exclude>WEB-INF/lib/titan-*.jar,WEB-INF/lib/je-*.jar,WEB-INF/lib/elasticsearch-*.jar,WEB-INF/lib/lucene-*.jar</packages.to.exclude> </properties> </profile> @@ -59,7 +59,6 @@ <packages.to.exclude>WEB-INF/lib/je-*.jar</packages.to.exclude> </properties> </profile> - <profile> <id>Windows</id> <activation> @@ -68,9 +67,48 @@ </os> </activation> <properties> - <log4j.configuration.url>file:/${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</log4j.configuration.url> + <log4j.configuration.url>file:/${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</log4j.configuration.url> </properties> - </profile> + </profile> + <profile> + <id>titan1</id> + <!-- remove conflicting lucene/titan versions from catalog when using titan 1 --> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-catalog</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>com.thinkaurelius.titan</groupId> + <artifactId>titan-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-catalog</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + </profile> </profiles> <dependencies> @@ -79,7 +117,6 @@ <artifactId>atlas-graphdb-impls</artifactId> <type>pom</type> </dependency> - <dependency> <groupId>org.apache.atlas</groupId> <artifactId>atlas-common</artifactId> @@ -100,27 +137,26 @@ <artifactId>atlas-client</artifactId> </dependency> - <dependency> + <dependency> <groupId>org.apache.atlas</groupId> <artifactId>atlas-authorization</artifactId> - <version>${project.version}</version> + <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.atlas</groupId> <artifactId>atlas-notification</artifactId> </dependency> - <dependency> <groupId>org.apache.atlas</groupId> <artifactId>atlas-intg</artifactId> </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minikdc</artifactId> @@ -136,6 +172,11 @@ <artifactId>atlas-catalog</artifactId> </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-graphdb-impls</artifactId> + <type>pom</type> + </dependency> <!-- Zookeeper, curator --> <dependency> <groupId>org.apache.curator</groupId> @@ -155,8 +196,9 @@ <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - </dependency> + + </dependency> <!-- supports simple auth handler --> <dependency> <groupId>org.apache.httpcomponents</groupId> @@ -274,55 +316,50 @@ </dependency> <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-core</artifactId> - <version>${spring.version}</version> + <groupId>org.springframework</groupId> + <artifactId>spring-core</artifactId> + <version>${spring.version}</version> </dependency> <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-web</artifactId> - <version>${spring.version}</version> + <groupId>org.springframework</groupId> + <artifactId>spring-web</artifactId> + <version>${spring.version}</version> </dependency> <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-webmvc</artifactId> - <version>${spring.version}</version> + <groupId>org.springframework</groupId> + <artifactId>spring-webmvc</artifactId> + <version>${spring.version}</version> </dependency> <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-core</artifactId> - <version>${spring.security.version}</version> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-core</artifactId> + <version>${spring.security.version}</version> </dependency> <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-web</artifactId> - <version>${spring.security.version}</version> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-web</artifactId> + <version>${spring.security.version}</version> </dependency> <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-config</artifactId> - <version>${spring.security.version}</version> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-config</artifactId> + <version>${spring.security.version}</version> </dependency> <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-ldap</artifactId> - <version>${spring.security.version}</version> - </dependency> - <dependency> - <groupId>javax.servlet</groupId> - <artifactId>javax.servlet-api</artifactId> - <version>${javax.servlet.version}</version> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-ldap</artifactId> + <version>${spring.security.version}</version> </dependency> <dependency> - <groupId>org.springframework.ldap</groupId> - <artifactId>spring-ldap-core</artifactId> - <version>${spring-ldap-core.version}</version> + <groupId>org.springframework.ldap</groupId> + <artifactId>spring-ldap-core</artifactId> + <version>${spring-ldap-core.version}</version> </dependency> <dependency> <groupId>org.apache.atlas</groupId> @@ -330,6 +367,11 @@ <type>war</type> </dependency> <dependency> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + <version>${javax.servlet.version}</version> + </dependency> + <dependency> <groupId>org.easymock</groupId> <artifactId>easymock</artifactId> <version>3.4</version> @@ -342,15 +384,13 @@ <classifier>tests</classifier> <scope>test</scope> </dependency> - <dependency> <groupId>org.apache.atlas</groupId> <artifactId>atlas-intg</artifactId> <classifier>tests</classifier> <scope>test</scope> </dependency> - - <dependency> + <dependency> <groupId>com.nimbusds</groupId> <artifactId>nimbus-jose-jwt</artifactId> <version>3.9</version> @@ -393,7 +433,7 @@ <artifactId>atlas-dashboardv2</artifactId> </overlay> <overlay> - <!-- empty groupId/artifactId represents the current build --> + <!-- empty groupId/artifactId represents the current build --> </overlay> </overlays> <archive> @@ -439,7 +479,7 @@ </configuration> </plugin> - <!-- Running unit tests in pre-integration-test phase after war is built --> + <!-- Running unit tests in pre-integration-test phase after war is built --> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> @@ -448,7 +488,7 @@ <user.dir>${project.basedir}</user.dir> <projectBaseDir>${project.basedir}/..</projectBaseDir> </systemProperties> - <!--<skipTests>true</skipTests>--> + <!--<skipTests>true</skipTests> --> <forkMode>always</forkMode> <redirectTestOutputToFile>true</redirectTestOutputToFile> <argLine>-Djava.awt.headless=true -Dproject.version=${project.version} @@ -494,8 +534,7 @@ <webApp> <contextPath>/</contextPath> <descriptor>${project.basedir}/src/test/webapp/WEB-INF/web.xml</descriptor> - <!-- ${project.build.directory}/atlas-webapp-${project.version} --> - <extraClasspath>${project.build.directory}/../../webapp/target/test-classes/</extraClasspath> + <extraClasspath>${project.build.testOutputDirectory}</extraClasspath> </webApp> <useTestScope>true</useTestScope> <systemProperties> @@ -512,13 +551,17 @@ <value>${project.build.directory}/logs</value> </systemProperty> <systemProperty> + <name>atlas.graphdb.backend</name> + <value>${graphdb.backend.impl}</value> + </systemProperty> + <systemProperty> <name>keystore.file</name> - <value>${project.build.directory}/../../webapp/target/atlas.keystore + <value>${project.build.directory}/atlas.keystore </value> </systemProperty> <systemProperty> <name>truststore.file</name> - <value>${project.build.directory}/../../webapp/target/atlas.keystore</value> + <value>${project.build.directory}/atlas.keystore</value> </systemProperty> <systemProperty> <name>atlas.home</name> @@ -529,6 +572,10 @@ <value>${project.build.directory}/data</value> </systemProperty> <systemProperty> + <name>atlas.graphdb.backend</name> + <value>${graphdb.backend.impl}</value> + </systemProperty> + <systemProperty> <key>atlas.conf</key> <value>${project.build.directory}/../../typesystem/target/test-classes</value> </systemProperty> @@ -537,26 +584,9 @@ <stopPort>31001</stopPort> <stopWait>${jetty-maven-plugin.stopWait}</stopWait> <daemon>${debug.jetty.daemon}</daemon> - <testClassesDirectory>${project.build.directory}/../../webapp/target/test-classes/</testClassesDirectory> + <testClassesDirectory>${project.build.testOutputDirectory}</testClassesDirectory> <useTestClasspath>true</useTestClasspath> </configuration> - <dependencies> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-jsp</artifactId> - <version>${jetty.version}</version> - <exclusions> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>org.glassfish</groupId> - <artifactId>javax.el</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> <executions> <execution> <id>start-jetty</id> @@ -578,7 +608,7 @@ <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> </plugin> - <plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId> <executions> @@ -655,4 +685,6 @@ </plugin> </plugins> </build> + + </project>
