ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/266d7cc0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/266d7cc0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/266d7cc0 Branch: refs/heads/master Commit: 266d7cc00bef49356bcdb1290930e05748182b12 Parents: b832faf Author: Shwetha GS <[email protected]> Authored: Wed Jul 22 14:22:03 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Wed Jul 22 14:22:03 2015 +0530 ---------------------------------------------------------------------- addons/hive-bridge/pom.xml | 57 +++-- .../atlas/hive/hook/BaseSSLAndKerberosTest.java | 5 +- .../hook/NegativeSSLAndKerberosHiveHookIT.java | 2 +- .../hive/hook/SSLAndKerberosHiveHookIT.java | 2 +- .../apache/atlas/hive/hook/SSLHiveHookIT.java | 4 +- .../src/test/resources/application.properties | 34 ++- .../apache/atlas/security/BaseSecurityTest.java | 108 --------- pom.xml | 191 ++++++++++++++-- release-log.txt | 1 + repository/pom.xml | 16 ++ .../apache/atlas/RepositoryMetadataModule.java | 1 + .../atlas/repository/graph/GraphProvider.java | 1 - .../repository/graph/TitanGraphProvider.java | 28 ++- .../atlas/services/DefaultMetadataService.java | 3 +- .../atlas/RepositoryServiceLoadingTest.java | 3 + .../GraphBackedDiscoveryServiceTest.java | 7 + .../atlas/discovery/HiveLineageServiceTest.java | 223 ++++++++++--------- .../GraphBackedMetadataRepositoryTest.java | 28 ++- .../typestore/GraphBackedTypeStoreTest.java | 13 ++ .../src/test/resources/application.properties | 26 ++- src/conf/application.properties | 17 ++ webapp/pom.xml | 139 ++++++------ .../atlas/web/listeners/GuiceServletConfig.java | 15 +- .../atlas/web/service/EmbeddedServer.java | 28 ++- .../atlas/web/service/SecureEmbeddedServer.java | 54 +++-- .../src/main/resources/application.properties | 19 +- .../MetadataAuthenticationKerberosFilterIT.java | 4 +- .../MetadataAuthenticationSimpleFilterIT.java | 4 +- .../atlas/web/listeners/LoginProcessorIT.java | 2 +- .../web/listeners/TestGuiceServletConfig.java | 57 +++++ .../atlas/web/security/BaseSecurityTest.java | 108 +++++++++ .../web/service/SecureEmbeddedServerIT.java | 3 + .../web/service/SecureEmbeddedServerITBase.java | 2 +- webapp/src/test/webapp/WEB-INF/web.xml | 53 +++++ 34 files changed, 885 insertions(+), 373 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index 2617aeb..117d76c 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -59,6 +59,13 @@ <artifactId>hive-metastore</artifactId> <version>${hive.version}</version> <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> @@ -73,6 +80,17 @@ <artifactId>hive-cli</artifactId> <version>${hive.version}</version> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> @@ -116,8 +134,16 @@ </dependency> <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty</artifactId> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-webapp</artifactId> + <type>test-jar</type> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> <scope>test</scope> </dependency> </dependencies> @@ -229,21 +255,22 @@ </plugin> <plugin> - <groupId>org.mortbay.jetty</groupId> - <artifactId>maven-jetty-plugin</artifactId> - <version>${jetty.version}</version> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-maven-plugin</artifactId> <configuration> <skip>${skipTests}</skip> <!--only skip int tests --> - <connectors> - <connector implementation="org.mortbay.jetty.nio.SelectChannelConnector"> - <port>21000</port> - <maxIdleTime>60000</maxIdleTime> - </connector> - </connectors> - <webApp>../../webapp/target/atlas-webapp-${project.version}.war</webApp> - <contextPath>/</contextPath> - <useTestClasspath>true</useTestClasspath> + <httpConnector> + <port>21000</port> + <idleTimeout>60000</idleTimeout> + </httpConnector> + <war>../../webapp/target/atlas-webapp-${project.version}.war</war> + <daemon>true</daemon> + <webApp> + <contextPath>/</contextPath> + <descriptor>../../webapp/src/test/webapp/WEB-INF/web.xml</descriptor> + </webApp> + <useTestScope>true</useTestScope> <systemProperties> <systemProperty> <name>atlas.log.dir</name> @@ -251,7 +278,7 @@ </systemProperty> <systemProperty> <name>atlas.conf</name> - <value>addons/hive-bridge/src/test/resources</value> + <value>${project.build.directory}/test-classes</value> </systemProperty> </systemProperties> <stopKey>atlas-stop</stopKey> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java index 5ae6fe2..11163c8 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java @@ -16,8 +16,7 @@ */ package org.apache.atlas.hive.hook; -import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; -import org.apache.atlas.security.BaseSecurityTest; +import org.apache.atlas.web.security.BaseSecurityTest; import org.apache.atlas.web.service.SecureEmbeddedServer; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.io.FileUtils; @@ -26,7 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.mortbay.jetty.Server; +import org.eclipse.jetty.server.Server; import java.io.File; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java index 629a9f3..072c36b 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLHostnameVerifier; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebAppContext; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java index b8f29aa..6fab2ee 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java @@ -34,7 +34,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLHostnameVerifier; import org.codehaus.jettison.json.JSONArray; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebAppContext; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java index 54bfaf6..b114d5a 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java @@ -37,8 +37,8 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLHostnameVerifier; import org.codehaus.jettison.json.JSONArray; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.webapp.WebAppContext; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/addons/hive-bridge/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/application.properties b/addons/hive-bridge/src/test/resources/application.properties index 50eefed..dda9a18 100644 --- a/addons/hive-bridge/src/test/resources/application.properties +++ b/addons/hive-bridge/src/test/resources/application.properties @@ -17,23 +17,45 @@ # ######### Graph Database Configs ######### +#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html # Graph Storage -atlas.graph.storage.backend=inmemory +atlas.graph.storage.backend=${titan.storage.backend} -# Graph Search Index -atlas.graph.index.search.backend=lucene -atlas.graph.index.search.directory=target/data/lucene +#Berkeley storage directory +atlas.graph.storage.directory=target/data/berkley +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname=${titan.storage.hostname} + +# Graph Search Index Backend +atlas.graph.index.search.backend=${titan.index.backend} + +#lucene +#atlas.graph.index.search.directory=target/data/lucene + +#elasticsearch +atlas.graph.index.search.directory=./target/data/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +#solr in cloud mode +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} + +#solr in http mode +atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr ######### Hive Lineage Configs ######### -# This models reflects the base super types for Data and Process #atlas.lineage.hive.table.type.name=DataSet #atlas.lineage.hive.process.type.name=Process #atlas.lineage.hive.process.inputs.name=inputs #atlas.lineage.hive.process.outputs.name=outputs ## Schema -#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns +#atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns ######### Security Properties ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java ---------------------------------------------------------------------- diff --git a/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java b/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java deleted file mode 100644 index 598f56b..0000000 --- a/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.security; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.hadoop.minikdc.MiniKdc; -import org.apache.zookeeper.Environment; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.webapp.WebAppContext; -import org.testng.Assert; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.Writer; -import java.nio.file.Files; -import java.util.Locale; -import java.util.Properties; - -/** - * - */ -public class BaseSecurityTest { - private static final String JAAS_ENTRY = "%s { \n" + " %s required\n" - // kerberos module - + " keyTab=\"%s\"\n" + " debug=true\n" + " principal=\"%s\"\n" + " useKeyTab=true\n" - + " useTicketCache=false\n" + " doNotPrompt=true\n" + " storeKey=true;\n" + "}; \n"; - protected MiniKdc kdc; - - protected String getWarPath() { - return String.format("/target/atlas-webapp-%s.war", - System.getProperty("release.version")); - } - - protected void generateTestProperties(Properties props) throws ConfigurationException, IOException { - PropertiesConfiguration config = - new PropertiesConfiguration(System.getProperty("user.dir") + "/../src/conf/application.properties"); - for (String propName : props.stringPropertyNames()) { - config.setProperty(propName, props.getProperty(propName)); - } - File file = new File(System.getProperty("user.dir"), "application.properties"); - file.deleteOnExit(); - Writer fileWriter = new FileWriter(file); - config.save(fileWriter); - } - - protected void startEmbeddedServer(Server server) throws Exception { - WebAppContext webapp = new WebAppContext(); - webapp.setContextPath("/"); - webapp.setWar(System.getProperty("user.dir") + getWarPath()); - server.setHandler(webapp); - - server.start(); - } - - protected File startKDC() throws Exception { - File target = Files.createTempDirectory("sectest").toFile(); - File kdcWorkDir = new File(target, "kdc"); - Properties kdcConf = MiniKdc.createConf(); - kdcConf.setProperty(MiniKdc.DEBUG, "true"); - kdc = new MiniKdc(kdcConf, kdcWorkDir); - kdc.start(); - - Assert.assertNotNull(kdc.getRealm()); - return kdcWorkDir; - } - - public String createJAASEntry(String context, String principal, File keytab) { - String keytabpath = keytab.getAbsolutePath(); - // fix up for windows; no-op on unix - keytabpath = keytabpath.replace('\\', '/'); - return String.format(Locale.ENGLISH, JAAS_ENTRY, context, getKerberosAuthModuleForJVM(), keytabpath, principal); - } - - protected String getKerberosAuthModuleForJVM() { - if (System.getProperty("java.vendor").contains("IBM")) { - return "com.ibm.security.auth.module.Krb5LoginModule"; - } else { - return "com.sun.security.auth.module.Krb5LoginModule"; - } - } - - protected void bindJVMtoJAASFile(File jaasFile) { - String path = jaasFile.getAbsolutePath(); - System.setProperty(Environment.JAAS_CONF_KEY, path); - } - - protected File createKeytab(MiniKdc kdc, File kdcWorkDir, String principal, String filename) throws Exception { - File keytab = new File(kdcWorkDir, filename); - kdc.createPrincipal(keytab, principal, principal + "/localhost", principal + "/127.0.0.1"); - return keytab; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b94c92b..a8afcc3 100755 --- a/pom.xml +++ b/pom.xml @@ -322,11 +322,13 @@ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <slf4j.version>1.7.7</slf4j.version> - <jetty.version>6.1.26</jetty.version> + <jetty.version>9.2.12.v20150709</jetty.version> <jersey.version>1.9</jersey.version> + <jackson.version>1.8.3</jackson.version> <tinkerpop.version>2.6.0</tinkerpop.version> <titan.version>0.5.4</titan.version> <hadoop.version>2.6.0</hadoop.version> + <hbase.version>0.98.9-hadoop2</hbase.version> <!-- scala versions --> <scala.version>2.10.4</scala.version> @@ -351,6 +353,8 @@ <!-- skips checkstyle and find bugs --> <skipCheck>false</skipCheck> + <titan.storage.backend>berkeleyje</titan.storage.backend> + <titan.index.backend>elasticsearch</titan.index.backend> </properties> <profiles> @@ -381,8 +385,22 @@ </python.path.l> </properties> </profile> - </profiles> + <!-- Turning on this profile affects only tests and does not affect packaging --> + <profile> + <id>distributed</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <properties> + <titan.storage.backend>hbase</titan.storage.backend> + <titan.index.backend>solr</titan.index.backend> + <solr.zk.address>localhost:9983</solr.zk.address> + <titan.storage.hostname>localhost</titan.storage.hostname> + </properties> + </profile> + + </profiles> <modules> <module>typesystem</module> <module>client</module> @@ -455,9 +473,15 @@ <dependencyManagement> <dependencies> <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1</artifactId> - <version>6.0.0</version> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jsp</artifactId> + <version>${jetty.version}</version> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> <!-- Logging --> @@ -507,25 +531,68 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.htrace</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> + <artifactId>hadoop-hdfs</artifactId> <version>${hadoop.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-annotations</artifactId> + <artifactId>hadoop-client</artifactId> <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> + <artifactId>hadoop-annotations</artifactId> <version>${hadoop.version}</version> - <scope>test</scope> </dependency> <dependency> @@ -615,6 +682,12 @@ <groupId>com.sun.jersey</groupId> <artifactId>jersey-json</artifactId> <version>${jersey.version}</version> + <exclusions> + <exclusion> + <groupId>javax.xml.stream</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> @@ -639,19 +712,19 @@ <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId> - <version>1.5.2</version> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> - <version>1.5.2</version> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>org.codehaus.jettison</groupId> <artifactId>jettison</artifactId> - <version>1.3</version> + <version>1.3.7</version> </dependency> <dependency> @@ -668,15 +741,15 @@ <!-- Jetty --> <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty</artifactId> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> <version>${jetty.version}</version> <scope>compile</scope> </dependency> <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty-plus</artifactId> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> <version>${jetty.version}</version> <scope>compile</scope> </dependency> @@ -736,12 +809,80 @@ <dependency> <groupId>com.thinkaurelius.titan</groupId> + <artifactId>titan-hbase</artifactId> + <version>${titan.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${hbase.version}</version> + <exclusions> + <exclusion> + <artifactId>avro</artifactId> + <groupId>org.apache.avro</groupId> + </exclusion> + <exclusion> + <artifactId>jruby-complete</artifactId> + <groupId>org.jruby</groupId> + </exclusion> + <exclusion> + <artifactId>asm</artifactId> + <groupId>asm</groupId> + </exclusion> + <exclusion> + <artifactId>*</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>*</artifactId> + <groupId>org.mortbay.jetty</groupId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.thinkaurelius.titan</groupId> <artifactId>titan-es</artifactId> <version>${titan.version}</version> </dependency> <dependency> <groupId>com.thinkaurelius.titan</groupId> + <artifactId>titan-solr</artifactId> + <version>${titan.version}</version> + <exclusions> + <exclusion> + <artifactId>*</artifactId> + <groupId>org.eclipse.jetty</groupId> + </exclusion> + <exclusion> + <artifactId>*</artifactId> + <groupId>javax.servlet</groupId> + </exclusion> + <exclusion> + <artifactId>*</artifactId> + <groupId>org.jruby.joni</groupId> + </exclusion> + <exclusion> + <artifactId>*</artifactId> + <groupId>org.eclipse.jetty.orbit</groupId> + </exclusion> + + <exclusion> + <artifactId>*</artifactId> + <groupId>org.restlet.jee</groupId> + </exclusion> + <exclusion> + <artifactId>*</artifactId> + <groupId>org.ow2.asm</groupId> + </exclusion> + </exclusions> + + </dependency> + + <dependency> + <groupId>com.thinkaurelius.titan</groupId> <artifactId>titan-lucene</artifactId> <version>${titan.version}</version> <!--<scope>test</scope>--> @@ -982,6 +1123,7 @@ <testResources> <testResource> <directory>src/test/resources</directory> + <filtering>true</filtering> </testResource> </testResources> @@ -1072,6 +1214,15 @@ </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.7</version> + <configuration> + <encoding>UTF-8</encoding> + </configuration> + </plugin> + + <plugin> <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> <version>3.2.0</version> @@ -1222,12 +1373,12 @@ <!-- Run the application using "mvn jetty:run" --> <plugin> - <groupId>org.mortbay.jetty</groupId> - <artifactId>maven-jetty-plugin</artifactId> - <version>6.1.16</version> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-maven-plugin</artifactId> + <version>${jetty.version}</version> <configuration> <!-- Log to the console. --> - <requestLog implementation="org.mortbay.jetty.NCSARequestLog"> + <requestLog implementation="org.eclipse.jetty.server.NCSARequestLog"> <!-- This doesn't do anything for Jetty, but is a workaround for a Maven bug that prevents the requestLog from being set. --> <append>true</append> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d655478..76b8778 100644 --- a/release-log.txt +++ b/release-log.txt @@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ALL CHANGES: +ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags) ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found ([email protected] via shwethags) ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags) ATLAS-46 Different data directory with restart (shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/pom.xml ---------------------------------------------------------------------- diff --git a/repository/pom.xml b/repository/pom.xml index b23c289..03aa792 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -101,11 +101,26 @@ <dependency> <groupId>com.thinkaurelius.titan</groupId> + <artifactId>titan-solr</artifactId> + </dependency> + + <dependency> + <groupId>com.thinkaurelius.titan</groupId> <artifactId>titan-berkeleyje</artifactId> </dependency> <dependency> <groupId>com.thinkaurelius.titan</groupId> + <artifactId>titan-hbase</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + </dependency> + + <dependency> + <groupId>com.thinkaurelius.titan</groupId> <artifactId>titan-lucene</artifactId> </dependency> @@ -172,6 +187,7 @@ </dependencies> <build> + <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index e7353f9..fbd01de 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -22,6 +22,7 @@ import com.google.inject.matcher.Matchers; import com.google.inject.multibindings.Multibinder; import com.google.inject.throwingproviders.ThrowingProviderBinder; import com.thinkaurelius.titan.core.TitanGraph; +import com.tinkerpop.blueprints.Graph; import org.aopalliance.intercept.MethodInterceptor; import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.discovery.HiveLineageService; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java index 5472081..f89bdf5 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java @@ -22,7 +22,6 @@ import com.google.inject.throwingproviders.CheckedProvider; import com.tinkerpop.blueprints.Graph; public interface GraphProvider<T extends Graph> extends CheckedProvider<T> { - @Override T get(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java index 3c1d151..d7ed1f1 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Singleton; import java.util.Iterator; +import java.util.Properties; /** * Default implementation for Graph Provider that doles out Titan Graph. @@ -43,11 +44,17 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> { */ private static final String ATLAS_PREFIX = "atlas.graph."; + private static TitanGraph graphInstance; + private static Configuration getConfiguration() throws AtlasException { PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties(); Configuration graphConfig = new PropertiesConfiguration(); + Properties sysProperties = System.getProperties(); + LOG.info("System properties: "); + LOG.info(sysProperties.toString()); + final Iterator<String> iterator = configProperties.getKeys(); while (iterator.hasNext()) { String key = iterator.next(); @@ -66,13 +73,20 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> { @Singleton @Provides public TitanGraph get() { - Configuration config; - try { - config = getConfiguration(); - } catch (AtlasException e) { - throw new RuntimeException(e); - } + if(graphInstance == null) { + synchronized (TitanGraphProvider.class) { + if(graphInstance == null) { + Configuration config; + try { + config = getConfiguration(); + } catch (AtlasException e) { + throw new RuntimeException(e); + } - return TitanFactory.open(config); + graphInstance = TitanFactory.open(config); + } + } + } + return graphInstance; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 9556272..56168db 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -81,7 +81,8 @@ public class DefaultMetadataService implements MetadataService { @Inject DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, - final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException { + final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException { + this.typeStore = typeStore; this.typeSystem = TypeSystem.getInstance(); this.repository = repository; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java index 4195955..18b1d1b 100755 --- a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java +++ b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java @@ -19,8 +19,11 @@ package org.apache.atlas; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import org.apache.atlas.repository.graph.GraphProvider; +import org.apache.atlas.typesystem.types.TypeSystem; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index 67cdd78..127e6d7 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -20,6 +20,7 @@ package org.apache.atlas.discovery; import com.google.common.collect.ImmutableList; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.RepositoryMetadataModule; @@ -117,6 +118,12 @@ public class GraphBackedDiscoveryServiceTest { @AfterClass public void tearDown() throws Exception { TypeSystem.getInstance().reset(); + graphProvider.get().shutdown(); + try { + TitanCleanup.clear(graphProvider.get()); + } catch(Exception e) { + e.printStackTrace(); + } } @Test http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java index ce00f8c..2c7d61a 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,9 +20,12 @@ package org.apache.atlas.discovery; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.repository.EntityNotFoundException; +import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.TypesDef; @@ -38,17 +41,22 @@ import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.util.HashSet; import java.util.List; /** @@ -58,65 +66,76 @@ import java.util.List; public class HiveLineageServiceTest { @Inject + private GraphBackedDiscoveryService discoveryService; + + @Inject private DefaultMetadataService metadataService; @Inject - private GraphBackedDiscoveryService discoveryService; + private GraphProvider<TitanGraph> graphProvider; @Inject private HiveLineageService hiveLineageService; - // @Inject - // private GraphProvider<TitanGraph> graphProvider; @BeforeClass public void setUp() throws Exception { setUpTypes(); setupInstances(); - // TestUtils.dumpGraph(graphProvider.get()); } + @AfterClass + public void tearDown() throws Exception { + TypeSystem.getInstance().reset(); + graphProvider.get().shutdown(); + try { + TitanCleanup.clear(graphProvider.get()); + } catch (Exception e) { + e.printStackTrace(); + } + } + @DataProvider(name = "dslQueriesProvider") private Object[][] createDSLQueries() { return new String[][]{ - // joins - {"hive_table where name=\"sales_fact\", columns"}, - {"hive_table where name=\"sales_fact\", columns select name, dataType, comment"}, - {"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"}, - // {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name, - // table.name"}, - {"from hive_db"}, {"hive_db"}, {"hive_db where hive_db.name=\"Reporting\""}, - {"hive_db hive_db.name = \"Reporting\""}, - {"hive_db where hive_db.name=\"Reporting\" select name, owner"}, {"hive_db has name"}, - // {"hive_db, hive_table"}, - // {"hive_db, hive_process has name"}, - // {"hive_db as db1, hive_table where db1.name = \"Reporting\""}, - // {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System - // .currentTimeMillis()}, - {"from hive_table"}, {"hive_table"}, {"hive_table is Dimension"}, - {"hive_column where hive_column isa PII"}, - // {"hive_column where hive_column isa PII select hive_column.name"}, - {"hive_column select hive_column.name"}, {"hive_column select name"}, - {"hive_column where hive_column.name=\"customer_id\""}, {"from hive_table select hive_table.name"}, - {"hive_db where (name = \"Reporting\")"}, - {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"}, - {"hive_db where hive_db has name"}, - // {"hive_db hive_table"}, - {"hive_db where hive_db has name"}, - // {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"}, - {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "}, - // {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = - // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name = - // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = - // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, - // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = - // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, - // trait searches - {"Dimension"}, {"Fact"}, {"ETL"}, {"Metric"}, {"PII"},}; + // joins + {"hive_table where name=\"sales_fact\", columns"}, + {"hive_table where name=\"sales_fact\", columns select name, dataType, comment"}, + {"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"}, + // {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name, + // table.name"}, + {"from hive_db"}, {"hive_db"}, {"hive_db where hive_db.name=\"Reporting\""}, + {"hive_db hive_db.name = \"Reporting\""}, + {"hive_db where hive_db.name=\"Reporting\" select name, owner"}, {"hive_db has name"}, + // {"hive_db, hive_table"}, + // {"hive_db, hive_process has name"}, + // {"hive_db as db1, hive_table where db1.name = \"Reporting\""}, + // {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System + // .currentTimeMillis()}, + {"from hive_table"}, {"hive_table"}, {"hive_table is Dimension"}, + {"hive_column where hive_column isa PII"}, + // {"hive_column where hive_column isa PII select hive_column.name"}, + {"hive_column select hive_column.name"}, {"hive_column select name"}, + {"hive_column where hive_column.name=\"customer_id\""}, {"from hive_table select hive_table.name"}, + {"hive_db where (name = \"Reporting\")"}, + {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"}, + {"hive_db where hive_db has name"}, + // {"hive_db hive_table"}, + {"hive_db where hive_db has name"}, + // {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"}, + {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "}, + // {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = + // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name = + // \"Reporting\") select db1.name as dbName, tab.name as tabName"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = + // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, + // {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = + // \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"}, + // trait searches + {"Dimension"}, {"Fact"}, {"ETL"}, {"Metric"}, {"PII"},}; } @Test(dataProvider = "dslQueriesProvider") @@ -242,7 +261,7 @@ public class HiveLineageServiceTest { @DataProvider(name = "tableNamesProvider") private Object[][] tableNames() { return new String[][]{{"sales_fact", "4"}, {"time_dim", "3"}, {"sales_fact_daily_mv", "4"}, - {"sales_fact_monthly_mv", "4"}}; + {"sales_fact_monthly_mv", "4"}}; } @Test(dataProvider = "tableNamesProvider") @@ -296,45 +315,45 @@ public class HiveLineageServiceTest { private TypesDef createTypeDefinitions() { HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil - .createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE)); + .createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), + attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), + attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE)); HierarchicalTypeDefinition<ClassType> storageDescClsDef = TypesUtil - .createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE), - attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE), - attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null)); + .createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE), + attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE), + attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null)); HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil - .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); + .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), + attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil - .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE), - attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE), - attrDef("temporary", DataTypes.BOOLEAN_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - // todo - uncomment this, something is broken - // new AttributeDefinition("sd", STORAGE_DESC_TYPE, - // Multiplicity.REQUIRED, true, null), - new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), - Multiplicity.COLLECTION, true, null)); + .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), + attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE), + attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE), + attrDef("temporary", DataTypes.BOOLEAN_TYPE), + new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), + // todo - uncomment this, something is broken + // new AttributeDefinition("sd", STORAGE_DESC_TYPE, + // Multiplicity.REQUIRED, true, null), + new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), + Multiplicity.COLLECTION, true, null)); HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil - .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), - attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE), - attrDef("endTime", DataTypes.LONG_TYPE), - attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)); + .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), + attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE), + attrDef("endTime", DataTypes.LONG_TYPE), + attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)); HierarchicalTypeDefinition<ClassType> viewClsDef = TypesUtil - .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), - new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), - Multiplicity.COLLECTION, false, null)); + .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE), + new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), + new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), + Multiplicity.COLLECTION, false, null)); HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null); @@ -349,8 +368,8 @@ public class HiveLineageServiceTest { HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null); return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef), - ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef)); + ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef), + ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef)); } AttributeDefinition attrDef(String name, IDataType dT) { @@ -362,7 +381,7 @@ public class HiveLineageServiceTest { } AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite, - String reverseAttributeName) { + String reverseAttributeName) { Preconditions.checkNotNull(name); Preconditions.checkNotNull(dT); return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName); @@ -372,62 +391,62 @@ public class HiveLineageServiceTest { Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); Referenceable sd = - storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true); + storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true); List<Referenceable> salesFactColumns = ImmutableList - .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), - column("customer_id", "int", "customer id", "PII"), - column("sales", "double", "product id", "Metric")); + .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), + column("customer_id", "int", "customer id", "PII"), + column("sales", "double", "product id", "Metric")); Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact"); List<Referenceable> timeDimColumns = ImmutableList - .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), - column("weekDay", "int", "week Day")); + .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), + column("weekDay", "int", "week Day")); Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns, - "Dimension"); + "Dimension"); Id reportingDB = - database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); + database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); Id salesFactDaily = - table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", - salesFactColumns, "Metric"); + table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", + salesFactColumns, "Metric"); loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim), - ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); + ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); List<Referenceable> productDimColumns = ImmutableList - .of(column("product_id", "int", "product id"), column("product_name", "string", "product name"), - column("brand_name", "int", "brand name")); + .of(column("product_id", "int", "product id"), column("product_name", "string", "product name"), + column("brand_name", "int", "brand name")); Id productDim = - table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, - "Dimension"); + table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, + "Dimension"); view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess"); List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"), - column("name", "string", "customer name", "PII"), - column("address", "string", "customer address", "PII")); + column("name", "string", "customer name", "PII"), + column("address", "string", "customer address", "PII")); Id customerDim = - table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, - "Dimension"); + table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, + "Dimension"); view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess"); Id salesFactMonthly = - table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", - "Managed", salesFactColumns, "Metric"); + table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", + "Managed", salesFactColumns, "Metric"); loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily), - ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); + ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); } Id database(String name, String description, String owner, String locationUri, String... traitNames) - throws Exception { + throws Exception { Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames); referenceable.set("name", name); referenceable.set("description", description); @@ -439,7 +458,7 @@ public class HiveLineageServiceTest { } Referenceable storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed) - throws Exception { + throws Exception { Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE); referenceable.set("location", location); referenceable.set("inputFormat", inputFormat); @@ -459,7 +478,7 @@ public class HiveLineageServiceTest { } Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType, - List<Referenceable> columns, String... traitNames) throws Exception { + List<Referenceable> columns, String... traitNames) throws Exception { Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames); referenceable.set("name", name); referenceable.set("description", description); @@ -479,8 +498,8 @@ public class HiveLineageServiceTest { } Id loadProcess(String name, String description, String user, List<Id> inputTables, List<Id> outputTables, - String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames) - throws Exception { + String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames) + throws Exception { Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames); referenceable.set("name", name); referenceable.set("description", description); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java index fb697d0..487ac84 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -20,9 +20,11 @@ package org.apache.atlas.repository.graph; import com.google.common.collect.ImmutableList; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Compare; import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.Vertex; +import org.apache.atlas.GraphTransaction; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; @@ -51,6 +53,7 @@ import org.apache.commons.lang.RandomStringUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -105,12 +108,23 @@ public class GraphBackedMetadataRepositoryTest { createHiveTypes(); } -/* - @AfterMethod + + @AfterClass public void tearDown() throws Exception { - TestUtils.dumpGraph(graphProvider.get()); + TypeSystem.getInstance().reset(); + try { + //TODO - Fix failure during shutdown while using BDB + graphProvider.get().shutdown(); + } catch(Exception e) { + e.printStackTrace(); + } + try { + TitanCleanup.clear(graphProvider.get()); + } catch(Exception e) { + e.printStackTrace(); + } } -*/ + @Test public void testSubmitEntity() throws Exception { @@ -189,7 +203,8 @@ public class GraphBackedMetadataRepositoryTest { System.out.println("*** table = " + table); } - private String getGUID() { + @GraphTransaction + String getGUID() { Vertex tableVertex = getTableEntityVertex(); String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY); @@ -199,7 +214,8 @@ public class GraphBackedMetadataRepositoryTest { return guid; } - private Vertex getTableEntityVertex() { + @GraphTransaction + Vertex getTableEntityVertex() { TitanGraph graph = graphProvider.get(); GraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE); Iterator<Vertex> results = query.vertices().iterator(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java index c774123..6a8129f 100755 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.typestore; import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; @@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -63,6 +65,17 @@ public class GraphBackedTypeStoreTest { TestUtils.defineDeptEmployeeTypes(ts); } + @AfterClass + public void tearDown() throws Exception { + ts.reset(); + graphProvider.get().shutdown(); + try { + TitanCleanup.clear(graphProvider.get()); + } catch(Exception e) { + e.printStackTrace(); + } + } + @Test @GraphTransaction public void testStore() throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/repository/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/application.properties b/repository/src/test/resources/application.properties index 20c0c51..d0eaa8c 100755 --- a/repository/src/test/resources/application.properties +++ b/repository/src/test/resources/application.properties @@ -19,15 +19,35 @@ ######### Graph Database Configs ######### #Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html # Graph Storage -atlas.graph.storage.backend=inmemory +atlas.graph.storage.backend=${titan.storage.backend} -# Graph Search Index -atlas.graph.index.search.backend=elasticsearch +#Berkeley storage directory +atlas.graph.storage.directory=target/data/berkley + +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname=${titan.storage.hostname} + +# Graph Search Index Backend +atlas.graph.index.search.backend=${titan.index.backend} + +#lucene +#atlas.graph.index.search.directory=target/data/lucene + +#elasticsearch atlas.graph.index.search.directory=./target/data/es atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +#solr in cloud mode +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} +#solr in http mode +atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr ######### Hive Lineage Configs ######### #atlas.lineage.hive.table.type.name=DataSet http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/src/conf/application.properties ---------------------------------------------------------------------- diff --git a/src/conf/application.properties b/src/conf/application.properties index c6b16cc..5c8c59a 100755 --- a/src/conf/application.properties +++ b/src/conf/application.properties @@ -21,6 +21,23 @@ atlas.graph.storage.backend=berkeleyje atlas.graph.storage.directory=data/berkley +#Hbase as stoarge backend +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +#atlas.graph.storage.hostname=localhost + +#Solr +#atlas.graph.index.search.backend=solr + +# Solr cloud mode properties +#atlas.graph.index.search.solr.mode=cloud +#atlas.graph.index.search.solr.zookeeper-url=localhost:2181 + +#Solr http mode properties +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr + # Graph Search Index atlas.graph.index.search.backend=elasticsearch atlas.graph.index.search.directory=data/es http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index 0ae4a05..86e1ebf 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -116,13 +116,13 @@ </dependency> <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty</artifactId> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> </dependency> <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty-plus</artifactId> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> </dependency> <dependency> @@ -171,8 +171,13 @@ </dependency> <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1</artifactId> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jsp</artifactId> + </dependency> + + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> </dependency> <dependency> @@ -192,36 +197,36 @@ <workingDirectory>../dashboard/v2/</workingDirectory> </configuration> <executions> - <execution> - <id>install node and npm</id> - <goals> - <goal>install-node-and-npm</goal> - </goals> - <configuration> - <nodeVersion>v0.10.30</nodeVersion> - <npmVersion>1.4.3</npmVersion> - </configuration> - </execution> - - <execution> - <id>npm install</id> - <goals> - <goal>npm</goal> - </goals> - <configuration> - <arguments>install</arguments> - </configuration> - </execution> - - <execution> - <id>grunt dist</id> - <goals> - <goal>grunt</goal> - </goals> - <configuration> - <arguments>build</arguments> - </configuration> - </execution> + <execution> + <id>install node and npm</id> + <goals> + <goal>install-node-and-npm</goal> + </goals> + <configuration> + <nodeVersion>v0.10.30</nodeVersion> + <npmVersion>1.4.3</npmVersion> + </configuration> + </execution> + + <execution> + <id>npm install</id> + <goals> + <goal>npm</goal> + </goals> + <configuration> + <arguments>install</arguments> + </configuration> + </execution> + + <execution> + <id>grunt dist</id> + <goals> + <goal>grunt</goal> + </goals> + <configuration> + <arguments>build</arguments> + </configuration> + </execution> </executions> </plugin> <plugin> @@ -245,7 +250,17 @@ </execution> </executions> </plugin> - + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-war-plugin</artifactId> @@ -260,7 +275,16 @@ <directory>src/main/webapp/WEB-INF</directory> <targetPath>WEB-INF</targetPath> </resource> + <resource> + <directory>${project.build.directory}/test-classes</directory> + <targetPath>WEB-INF/classes</targetPath> + </resource> </webResources> + <archive> + <manifest> + <addClasspath>true</addClasspath> + </manifest> + </archive> </configuration> </plugin> @@ -312,30 +336,24 @@ </plugin> <plugin> - <groupId>org.mortbay.jetty</groupId> - <artifactId>maven-jetty-plugin</artifactId> - <version>${jetty.version}</version> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-maven-plugin</artifactId> <configuration> <skip>${skipTests}</skip> <!--only skip int tests --> - <connectors> - <!-- - <connector implementation="org.mortbay.jetty.security.SslSocketConnector"> - <port>21443</port> - <maxIdleTime>60000</maxIdleTime> - <keystore>${project.build.directory}/../../webapp/target/atlas.keystore</keystore> - <keyPassword>atlas-passwd</keyPassword> - <password>atlas-passwd</password> - </connector> - --> - <connector implementation="org.mortbay.jetty.nio.SelectChannelConnector"> - <port>21000</port> - <maxIdleTime>60000</maxIdleTime> - </connector> - </connectors> - <webApp>${project.build.directory}/atlas-webapp-${project.version}</webApp> - <contextPath>/</contextPath> - <useTestClasspath>true</useTestClasspath> + <httpConnector> + <port>21000</port> + <idleTimeout>60000</idleTimeout> + </httpConnector> + <war>${project.build.directory}/atlas-webapp-${project.version}.war</war> + <daemon>true</daemon> + <!--<webAppSourceDirectory>webapp/src/test/webapp</webAppSourceDirectory>--> + <webApp> + <contextPath>/</contextPath> + <descriptor>webapp/src/test/webapp/WEB-INF/web.xml</descriptor> + <!-- ${project.build.directory}/atlas-webapp-${project.version} --> + </webApp> + <useTestScope>true</useTestScope> <systemProperties> <systemProperty> <name>atlas.log.dir</name> @@ -364,11 +382,8 @@ <id>start-jetty</id> <phase>pre-integration-test</phase> <goals> - <goal>run</goal> + <goal>deploy-war</goal> </goals> - <configuration> - <daemon>true</daemon> - </configuration> </execution> <execution> <id>stop-jetty</id> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java index 18ce17c..b643f13 100755 --- a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java +++ b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java @@ -20,14 +20,20 @@ package org.apache.atlas.web.listeners; import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provider; +import com.google.inject.TypeLiteral; import com.google.inject.servlet.GuiceServletContextListener; import com.sun.jersey.api.core.PackagesResourceConfig; import com.sun.jersey.guice.JerseyServletModule; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.thinkaurelius.titan.core.TitanGraph; +import com.tinkerpop.blueprints.Graph; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.PropertiesUtil; import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.web.filters.AtlasAuthenticationFilter; import org.apache.atlas.web.filters.AuditFilter; import org.apache.commons.configuration.ConfigurationException; @@ -35,6 +41,7 @@ import org.apache.commons.configuration.PropertiesConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; import javax.servlet.ServletContextEvent; import java.util.HashMap; import java.util.Map; @@ -45,7 +52,7 @@ public class GuiceServletConfig extends GuiceServletContextListener { private static final String GUICE_CTX_PARAM = "guice.packages"; static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled"; - private Injector injector; + protected Injector injector; @Override protected Injector getInjector() { @@ -106,5 +113,11 @@ public class GuiceServletConfig extends GuiceServletContextListener { @Override public void contextDestroyed(ServletContextEvent servletContextEvent) { super.contextDestroyed(servletContextEvent); + if(injector != null) { + TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {}; + Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType)); + final Graph graph = graphProvider.get().get(); + graph.shutdown(); + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java index 88200f0..16591d8 100755 --- a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java @@ -20,10 +20,13 @@ package org.apache.atlas.web.service; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.webapp.WebAppClassLoader; +import org.eclipse.jetty.webapp.WebAppContext; import java.io.IOException; @@ -40,6 +43,7 @@ public class EmbeddedServer { server.addConnector(connector); WebAppContext application = new WebAppContext(path, "/"); + application.setClassLoader(Thread.currentThread().getContextClassLoader()); server.setHandler(application); } @@ -52,19 +56,21 @@ public class EmbeddedServer { } protected Connector getConnector(int port) throws IOException { - Connector connector = new SocketConnector(); - connector.setPort(port); - connector.setHost("0.0.0.0"); + HttpConfiguration http_config = new HttpConfiguration(); // this is to enable large header sizes when Kerberos is enabled with AD - final Integer bufferSize = getBufferSize(); - connector.setHeaderBufferSize(bufferSize); - connector.setRequestBufferSize(bufferSize); + final int bufferSize = getBufferSize(); + http_config.setResponseHeaderSize(bufferSize); + http_config.setRequestHeaderSize(bufferSize); + ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config)); + connector.setPort(port); + connector.setHost("0.0.0.0"); + server.addConnector(connector); return connector; } - private Integer getBufferSize() { + protected Integer getBufferSize() { try { PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties"); return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
