http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java deleted file mode 100755 index 1ae733f..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.metadata; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.metadata.util.CredentialProviderUtility; -import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer; -import org.apache.hadoop.security.alias.CredentialProvider; -import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.apache.hadoop.security.alias.JavaKeyStoreProvider; -import org.testng.Assert; -import org.testng.annotations.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.*; - -import static org.apache.hadoop.metadata.security.SecurityProperties.*; - -/** - * - */ -public class CredentialProviderUtilityIT { - - private char[] defaultPass = new char[]{'k', 'e', 'y', 'p', 'a', 's', 's'}; - - @Test - public void testEnterValidValues() throws Exception { - Path testPath = null; - try { - testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); - } catch (IOException e) { - e.printStackTrace(); - } - new File(testPath.toUri().getPath()).delete(); - final Path finalTestPath = testPath; - CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() { - @Override - public void printf(String fmt, Object... params) { - System.out.print(String.format(fmt, params)); - } - - public String readLine(String fmt, Object ... args) { - return finalTestPath.toString(); - } - - @Override - public char[] readPassword(String fmt, Object ... args) { - return defaultPass; - } - }; - - CredentialProviderUtility.main(new String[] {}); - - String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri(); - Configuration conf = new Configuration(false); - - conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl); - - CredentialProvider provider = - CredentialProviderFactory.getProviders(conf).get(0); - - CredentialProvider.CredentialEntry entry = - provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - } - - protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry) { - assertCredentialEntryCorrect(entry, defaultPass); - } - - protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry, char[] password) { - Assert.assertNotNull(entry); - Assert.assertEquals(entry.getCredential(), password); - } - - @Test - public void testEnterEmptyValues() throws Exception { - Path testPath = null; - try { - testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); - } catch (IOException e) { - e.printStackTrace(); - } - new File(testPath.toUri().getPath()).delete(); - final Path finalTestPath = testPath; - CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() { - - private Random random = new Random(); - - @Override - public void printf(String fmt, Object... params) { - System.out.print(String.format(fmt, params)); - } - - public String readLine(String fmt, Object ... args) { - return finalTestPath.toString(); - } - - @Override - public char[] readPassword(String fmt, Object ... args) { - List<char[]> responses = new ArrayList<>(); - responses.add(new char[0]); - responses.add(defaultPass); - - int size = responses.size(); - int item = random.nextInt(size); - return responses.get(item); - } - }; - - CredentialProviderUtility.main(new String[] {}); - - String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri(); - Configuration conf = new Configuration(false); - - conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl); - - CredentialProvider provider = - CredentialProviderFactory.getProviders(conf).get(0); - - CredentialProvider.CredentialEntry entry = - provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - } - - @Test - public void testEnterMismatchedValues() throws Exception { - Path testPath = null; - try { - testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); - } catch (IOException e) { - e.printStackTrace(); - } - new File(testPath.toUri().getPath()).delete(); - final Path finalTestPath = testPath; - CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() { - - int i = 0; - @Override - public void printf(String fmt, Object... params) { - System.out.print(String.format(fmt, params)); - } - - public String readLine(String fmt, Object ... args) { - return finalTestPath.toString(); - } - - @Override - public char[] readPassword(String fmt, Object ... args) { - List<char[]> responses = new ArrayList<>(); - responses.add(defaultPass); - responses.add(new char[] {'b', 'a', 'd', 'p', 'a', 's', 's'}); - responses.add(defaultPass); - - int item = i % 3; - i++; - return responses.get(item); - } - }; - - CredentialProviderUtility.main(new String[] {}); - - String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri(); - Configuration conf = new Configuration(false); - - conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl); - - CredentialProvider provider = - CredentialProviderFactory.getProviders(conf).get(0); - - CredentialProvider.CredentialEntry entry = - provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY); - assertCredentialEntryCorrect(entry); - } - - @Test - public void testOverwriteValues() throws Exception { - Path testPath = null; - try { - testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); - } catch (IOException e) { - e.printStackTrace(); - } - new File(testPath.toUri().getPath()).delete(); - final Path finalTestPath = testPath; - CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() { - @Override - public void printf(String fmt, Object... params) { - System.out.print(String.format(fmt, params)); - } - - public String readLine(String fmt, Object ... args) { - return finalTestPath.toString(); - } - - @Override - public char[] readPassword(String fmt, Object ... args) { - return defaultPass; - } - }; - - CredentialProviderUtility.main(new String[] {}); - - // now attempt to overwrite values - CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() { - - int i = 0; - - @Override - public void printf(String fmt, Object... params) { - System.out.print(String.format(fmt, params)); - } - - public String readLine(String fmt, Object ... args) { - return i++ == 0 ? finalTestPath.toString() : "y"; - } - - @Override - public char[] readPassword(String fmt, Object ... args) { - return new char[] {'n', 'e', 'w', 'p', 'a', 's', 's'}; - } - }; - - CredentialProviderUtility.main(new String[] {}); - - String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri(); - Configuration conf = new Configuration(false); - - conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl); - - CredentialProvider provider = - CredentialProviderFactory.getProviders(conf).get(0); - - char[] newpass = "newpass".toCharArray(); - CredentialProvider.CredentialEntry entry = - provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry, newpass); - entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY); - assertCredentialEntryCorrect(entry, newpass); - entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY); - assertCredentialEntryCorrect(entry, newpass); - } -}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java deleted file mode 100644 index 857a42a..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.metadata.web.filters; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.hdfs.web.URLConnectionFactory; -import org.apache.hadoop.metadata.security.BaseSecurityTest; -import org.apache.hadoop.metadata.web.service.EmbeddedServer; -import org.mortbay.jetty.Server; -import org.testng.Assert; -import org.testng.annotations.Test; - -import javax.security.auth.Subject; -import javax.security.auth.callback.*; -import javax.security.auth.login.LoginContext; -import javax.security.auth.login.LoginException; -import java.io.File; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.security.PrivilegedExceptionAction; -import java.util.Properties; - -/** - * - */ -public class MetadataAuthenticationKerberosFilterIT extends BaseSecurityTest { - public static final String TEST_USER_JAAS_SECTION = "TestUser"; - public static final String TESTUSER = "testuser"; - public static final String TESTPASS = "testpass"; - - private File userKeytabFile; - private File httpKeytabFile; - - class TestEmbeddedServer extends EmbeddedServer { - public TestEmbeddedServer(int port, String path) throws IOException { - super(port, path); - } - - Server getServer() { - return server; - } - } - - @Test - public void testKerberosBasedLogin() throws Exception { - String originalConf = System.getProperty("metadata.conf"); - System.setProperty("metadata.conf", System.getProperty("user.dir")); - - setupKDCAndPrincipals(); - TestEmbeddedServer server = null; - - try { - // setup the application.properties file - generateKerberosTestProperties(); - - // need to create the web application programmatically in order to control the injection of the test - // application properties - server = new TestEmbeddedServer(23000, "webapp/target/metadata-governance"); - - startEmbeddedServer(server.getServer()); - - final URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_SYSTEM_CONNECTION_FACTORY; - // attempt to hit server and get rejected - URL url = new URL("http://localhost:23000/";); - HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, false); - connection.setRequestMethod("GET"); - connection.connect(); - - Assert.assertEquals(connection.getResponseCode(), 401); - - // need to populate the ticket cache with a local user, so logging in... - Subject subject = loginTestUser(); - - Subject.doAs(subject, new PrivilegedExceptionAction<Object>() { - @Override - public Object run() throws Exception { - // attempt to hit server and get rejected - URL url = new URL("http://localhost:23000/";); - HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, true); - connection.setRequestMethod("GET"); - connection.connect(); - - Assert.assertEquals(connection.getResponseCode(), 200); - - return null; - } - }); - } finally { - server.getServer().stop(); - kdc.stop(); - - if (originalConf != null) { - System.setProperty("metadata.conf", originalConf); - } else { - System.clearProperty("metadata.conf"); - } - - } - - - } - - protected Subject loginTestUser() throws LoginException, IOException { - LoginContext lc = new LoginContext(TEST_USER_JAAS_SECTION, new CallbackHandler() { - - @Override - public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { - for (int i = 0; i < callbacks.length; i++) { - if (callbacks[i] instanceof PasswordCallback) { - PasswordCallback passwordCallback = (PasswordCallback) callbacks[i]; - passwordCallback.setPassword(TESTPASS.toCharArray()); - } - if (callbacks[i] instanceof NameCallback) { - NameCallback nameCallback = (NameCallback) callbacks[i]; - nameCallback.setName(TESTUSER); - } - } - } - }); - // attempt authentication - lc.login(); - return lc.getSubject(); - } - - protected void generateKerberosTestProperties() throws IOException, ConfigurationException { - Properties props = new Properties(); - props.setProperty("metadata.http.authentication.enabled", "true"); - props.setProperty("metadata.http.authentication.type", "kerberos"); - props.setProperty("metadata.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm()); - props.setProperty("metadata.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath()); - props.setProperty("metadata.http.authentication.kerberos.name.rules", - "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT"); - - generateTestProperties(props); - } - - public void setupKDCAndPrincipals() throws Exception { - // set up the KDC - File kdcWorkDir = startKDC(); - - userKeytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab"); - httpKeytabFile = createKeytab(kdc, kdcWorkDir, "HTTP", "spnego.service.keytab"); - - // create a test user principal - kdc.createPrincipal(TESTUSER, TESTPASS); - - StringBuilder jaas = new StringBuilder(1024); - jaas.append("TestUser {\n" + - " com.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true;\n" + - "};\n"); - jaas.append(createJAASEntry("Client", "dgi", userKeytabFile)); - jaas.append(createJAASEntry("Server", "HTTP", httpKeytabFile)); - - File jaasFile = new File(kdcWorkDir, "jaas.txt"); - FileUtils.write(jaasFile, jaas.toString()); - bindJVMtoJAASFile(jaasFile); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java deleted file mode 100644 index f41ad0a..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.metadata.web.filters; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.hadoop.metadata.security.BaseSecurityTest; -import org.apache.hadoop.metadata.web.service.EmbeddedServer; -import org.mortbay.jetty.Server; -import org.testng.Assert; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.Properties; - -/** - * - */ -public class MetadataAuthenticationSimpleFilterIT extends BaseSecurityTest { - - class TestEmbeddedServer extends EmbeddedServer { - public TestEmbeddedServer(int port, String path) throws IOException { - super(port, path); - } - - Server getServer() { - return server; - } - } - - @Test - public void testSimpleLogin() throws Exception { - String originalConf = System.getProperty("metadata.conf"); - System.setProperty("metadata.conf", System.getProperty("user.dir")); - generateSimpleLoginConfiguration(); - - TestEmbeddedServer server = new TestEmbeddedServer(23001, "webapp/target/metadata-governance"); - - try { - startEmbeddedServer(server.getServer()); - - URL url = new URL("http://localhost:23001";); - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); - connection.setRequestMethod("GET"); - connection.connect(); - - try { - Assert.assertEquals(connection.getResponseCode(), 403); - } catch (Exception e) { - e.printStackTrace(); - } - - url = new URL("http://localhost:23001/?user.name=testuser";); - connection = (HttpURLConnection)url.openConnection(); - connection.setRequestMethod("GET"); - connection.connect(); - - Assert.assertEquals(connection.getResponseCode(), 200); - } finally { - server.getServer().stop(); - if (originalConf != null) { - System.setProperty("metadata.conf", originalConf); - } else { - System.clearProperty("metadata.conf"); - } - } - - - } - - protected void generateSimpleLoginConfiguration() throws IOException, ConfigurationException { - Properties config = new Properties(); - config.setProperty("metadata.http.authentication.enabled", "true"); - config.setProperty("metadata.http.authentication.type", "simple"); - - generateTestProperties(config); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java deleted file mode 100644 index be7171b..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.metadata.web.listeners; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.metadata.security.BaseSecurityTest; -import org.apache.hadoop.security.UserGroupInformation; -import org.testng.Assert; -import org.testng.annotations.Test; - -import java.io.File; - -/** - * - */ -public class LoginProcessorIT extends BaseSecurityTest { - - protected static final String kerberosRule = - "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT"; - - @Test - public void testDefaultSimpleLogin() throws Exception { - LoginProcessor processor = new LoginProcessor() { - @Override - protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException { - return new PropertiesConfiguration(); - } - }; - processor.login(); - - Assert.assertNotNull(UserGroupInformation.getCurrentUser()); - Assert.assertFalse(UserGroupInformation.isLoginKeytabBased()); - Assert.assertFalse(UserGroupInformation.isSecurityEnabled()); - } - - @Test - public void testKerberosLogin() throws Exception { - final File keytab = setupKDCAndPrincipals(); - - LoginProcessor processor = new LoginProcessor() { - @Override - protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException { - PropertiesConfiguration config = new PropertiesConfiguration(); - config.setProperty("metadata.authentication.method", "kerberos"); - config.setProperty("metadata.authentication.principal", "[email protected]"); - config.setProperty("metadata.authentication.keytab", keytab.getAbsolutePath()); - return config; - } - - @Override - protected Configuration getHadoopConfiguration() { - Configuration config = new Configuration(false); - config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - config.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); - config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, kerberosRule); - - return config; - } - - @Override - protected boolean isHadoopCluster() { - return true; - } - }; - processor.login(); - - Assert.assertTrue(UserGroupInformation.getLoginUser().getShortUserName().endsWith("dgi")); - Assert.assertNotNull(UserGroupInformation.getCurrentUser()); - Assert.assertTrue(UserGroupInformation.isLoginKeytabBased()); - Assert.assertTrue(UserGroupInformation.isSecurityEnabled()); - - kdc.stop(); - - } - - private File setupKDCAndPrincipals() throws Exception { - // set up the KDC - File kdcWorkDir = startKDC(); - - Assert.assertNotNull(kdc.getRealm()); - - File keytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab"); - - return keytabFile; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java deleted file mode 100755 index ca51cd6..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.web.resources; - -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.hadoop.metadata.web.util.Servlets; -import org.codehaus.jettison.json.JSONObject; -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.Response; - -/** - * Integration test for Admin jersey resource. - */ -public class AdminJerseyResourceIT extends BaseResourceIT { - - @BeforeClass - public void setUp() throws Exception { - super.setUp(); - } - - @Test - public void testGetVersion() throws Exception { - WebResource resource = service - .path("api/metadata/admin/version"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - PropertiesConfiguration buildConfiguration = - new PropertiesConfiguration("metadata-buildinfo.properties"); - - - JSONObject response = new JSONObject(responseAsString); - Assert.assertEquals(response.get("Version"), - buildConfiguration.getString("build.version")); - Assert.assertEquals(response.get("Name"), - buildConfiguration.getString("project.name")); - Assert.assertEquals(response.get("Description"), - buildConfiguration.getString("project.description")); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java deleted file mode 100755 index e0efe7d..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.web.resources; - -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import org.apache.hadoop.metadata.MetadataServiceClient; -import org.apache.hadoop.metadata.typesystem.Referenceable; -import org.apache.hadoop.metadata.typesystem.TypesDef; -import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization; -import org.apache.hadoop.metadata.typesystem.json.TypesSerialization; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.types.ClassType; -import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition; -import org.apache.hadoop.metadata.web.util.Servlets; -import org.codehaus.jettison.json.JSONObject; -import org.testng.Assert; -import org.testng.annotations.BeforeClass; - -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriBuilder; - -/** - * Base class for integration tests. - * Sets up the web resource and has helper methods to create type and entity. - */ -public abstract class BaseResourceIT { - - protected WebResource service; - protected MetadataServiceClient serviceClient; - public static String baseUrl = "http://localhost:21000/";; - - @BeforeClass - public void setUp() throws Exception { - - DefaultClientConfig config = new DefaultClientConfig(); - Client client = Client.create(config); - client.resource(UriBuilder.fromUri(baseUrl).build()); - - service = client.resource(UriBuilder.fromUri(baseUrl).build()); - serviceClient = new MetadataServiceClient(baseUrl); - } - - protected void createType(TypesDef typesDef) throws Exception { - HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0); - if (serviceClient.getType(sampleType.typeName) == null ) { - String typesAsJSON = TypesSerialization.toJson(typesDef); - createType(typesAsJSON); - } - } - - protected void createType(String typesAsJSON) throws Exception { - WebResource resource = service - .path("api/metadata/types"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.POST, ClientResponse.class, typesAsJSON); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get("types")); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - } - - protected Id createInstance(Referenceable referenceable) throws Exception { - String typeName = referenceable.getTypeName(); - System.out.println("creating instance of type " + typeName); - - String entityJSON = InstanceSerialization.toJson(referenceable, true); - System.out.println("Submitting new entity= " + entityJSON); - JSONObject jsonObject = serviceClient.createEntity(entityJSON); - String guid = jsonObject.getString(MetadataServiceClient.GUID); - System.out.println("created instance for type " + typeName + ", guid: " + guid); - - // return the reference to created instance with guid - return new Id(guid, 0, referenceable.getTypeName()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java deleted file mode 100755 index 00cc234..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java +++ /dev/null @@ -1,679 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.web.resources; - -import com.google.common.collect.ImmutableList; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.metadata.MetadataServiceClient; -import org.apache.hadoop.metadata.MetadataServiceException; -import org.apache.hadoop.metadata.typesystem.IStruct; -import org.apache.hadoop.metadata.typesystem.Referenceable; -import org.apache.hadoop.metadata.typesystem.Struct; -import org.apache.hadoop.metadata.typesystem.TypesDef; -import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization; -import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization$; -import org.apache.hadoop.metadata.typesystem.json.TypesSerialization; -import org.apache.hadoop.metadata.typesystem.json.TypesSerialization$; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition; -import org.apache.hadoop.metadata.typesystem.types.ClassType; -import org.apache.hadoop.metadata.typesystem.types.DataTypes; -import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition; -import org.apache.hadoop.metadata.typesystem.types.EnumValue; -import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition; -import org.apache.hadoop.metadata.typesystem.types.Multiplicity; -import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition; -import org.apache.hadoop.metadata.typesystem.types.TraitType; -import org.apache.hadoop.metadata.typesystem.types.TypeUtils; -import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil; -import org.apache.hadoop.metadata.web.util.Servlets; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.Response; -import java.util.List; -import java.util.UUID; - -/** - * Integration tests for Entity Jersey Resource. - */ -public class EntityJerseyResourceIT extends BaseResourceIT { - - private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class); - - private static final String DATABASE_TYPE = "hive_database"; - private static final String DATABASE_NAME = "foo"; - private static final String TABLE_TYPE = "hive_table_type"; - private static final String TABLE_NAME = "bar"; - private static final String TRAITS = "traits"; - - private Referenceable tableInstance; - private Id tableId; - private String traitName; - - @BeforeClass - public void setUp() throws Exception { - super.setUp(); - - createHiveTypes(); - } - - @Test - public void testSubmitEntity() throws Exception { - tableInstance = createHiveTableInstance(); - tableId = createInstance(tableInstance); - - final String guid = tableId._getId(); - try { - Assert.assertNotNull(UUID.fromString(guid)); - } catch (IllegalArgumentException e) { - Assert.fail("Response is not a guid, " + guid); - } - } - - @DataProvider - public Object[][] invalidAttrValues() { - return new Object[][]{ - {null}, {""}, {" "}}; - } - - @Test(dataProvider = "invalidAttrValues") - public void testEntityInvalidValue(String value) throws Exception { - Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); - databaseInstance.set("name", randomString()); - databaseInstance.set("description", value); - - try { - createInstance(databaseInstance); - Assert.fail("Exptected MetadataServiceException"); - } catch(MetadataServiceException e) { - Assert.assertEquals(e.getStatus(), ClientResponse.Status.BAD_REQUEST); - } - } - - @Test - public void testSubmitEntityWithBadDateFormat() throws Exception { - - try { - Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); - databaseInstance.set("name", DATABASE_NAME); - databaseInstance.set("description", "foo database"); - - Referenceable tableInstance = new Referenceable(TABLE_TYPE, - "classification", "pii", "phi", "pci", "sox", "sec", "finance"); - tableInstance.set("name", TABLE_NAME); - tableInstance.set("description", "bar table"); - tableInstance.set("date", "2014-07-11"); - tableInstance.set("type", "managed"); - tableInstance.set("level", 2); - tableInstance.set("tableType", 1); // enum - tableInstance.set("database", databaseInstance); - tableInstance.set("compressed", false); - - Struct traitInstance = (Struct) tableInstance.getTrait("classification"); - traitInstance.set("tag", "foundation_etl"); - - Struct serde1Instance = new Struct("serdeType"); - serde1Instance.set("name", "serde1"); - serde1Instance.set("serde", "serde1"); - tableInstance.set("serde1", serde1Instance); - - Struct serde2Instance = new Struct("serdeType"); - serde2Instance.set("name", "serde2"); - serde2Instance.set("serde", "serde2"); - tableInstance.set("serde2", serde2Instance); - - tableId = createInstance(tableInstance); - Assert.fail("Was expecting an exception here "); - } catch (MetadataServiceException e) { - Assert.assertTrue( - e.getMessage().contains("\"error\":\"Cannot convert value '2014-07-11' to datatype date\"")); - } - } - - @Test(dependsOnMethods = "testSubmitEntity") - public void testAddProperty() throws Exception { - final String guid = tableId._getId(); - //add property - String description = "bar table - new desc"; - ClientResponse clientResponse = addProperty(guid, "description", description); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String entityRef = getEntityDefinition(getEntityDefinition(guid)); - Assert.assertNotNull(entityRef); - - tableInstance.set("description", description); - - //invalid property for the type - clientResponse = addProperty(guid, "invalid_property", "bar table"); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); - - //non-string property, update - clientResponse = addProperty(guid, "level", "4"); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - entityRef = getEntityDefinition(getEntityDefinition(guid)); - Assert.assertNotNull(entityRef); - - tableInstance.set("level", 4); - } - - @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class) - public void testAddNullProperty() throws Exception { - final String guid = tableId._getId(); - //add property - addProperty(guid, null, "foo bar"); - Assert.fail(); - } - - @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class) - public void testAddNullPropertyValue() throws Exception { - final String guid = tableId._getId(); - //add property - addProperty(guid, "description", null); - Assert.fail(); - } - - @Test(dependsOnMethods = "testSubmitEntity") - public void testAddReferenceProperty() throws Exception { - //Create new db instance - Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); - databaseInstance.set("name", "newdb"); - databaseInstance.set("description", "new database"); - - Id dbInstance = createInstance(databaseInstance); - String dbId = dbInstance._getId(); - - //Add reference property - final String guid = tableId._getId(); - ClientResponse clientResponse = addProperty(guid, "database", dbId); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - } - - @Test(dependsOnMethods = "testSubmitEntity") - public void testGetEntityDefinition() throws Exception { - final String guid = tableId._getId(); - ClientResponse clientResponse = getEntityDefinition(guid); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - - final String definition = response.getString(MetadataServiceClient.DEFINITION); - Assert.assertNotNull(definition); - LOG.debug("tableInstanceAfterGet = " + definition); - InstanceSerialization.fromJsonReferenceable(definition, true); - } - - private ClientResponse addProperty(String guid, String property, String value) { - WebResource resource = service - .path("api/metadata/entities") - .path(guid); - - return resource.queryParam("property", property).queryParam("value", value) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.PUT, ClientResponse.class); - } - - private ClientResponse getEntityDefinition(String guid) { - WebResource resource = service - .path("api/metadata/entities") - .path(guid); - return resource.accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - } - - private String getEntityDefinition(ClientResponse clientResponse) throws Exception { - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - JSONObject response = new JSONObject(clientResponse.getEntity(String.class)); - final String definition = response.getString(MetadataServiceClient.DEFINITION); - Assert.assertNotNull(definition); - - return definition; - } - - @Test - public void testGetInvalidEntityDefinition() throws Exception { - WebResource resource = service - .path("api/metadata/entities") - .path("blah"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.ERROR)); - Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE)); - } - - @Test(dependsOnMethods = "testSubmitEntity") - public void testGetEntityList() throws Exception { - ClientResponse clientResponse = service - .path("api/metadata/entities") - .queryParam("type", TABLE_TYPE) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - - final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); - Assert.assertNotNull(list); - Assert.assertEquals(list.length(), 1); - } - - @Test - public void testGetEntityListForBadEntityType() throws Exception { - ClientResponse clientResponse = service - .path("api/metadata/entities") - .queryParam("type", "blah") - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.ERROR)); - Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE)); - } - - - @Test - public void testGetEntityListForNoInstances() throws Exception { - addNewType(); - - ClientResponse clientResponse = service - .path("api/metadata/entities") - .queryParam("type", "test") - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - - final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); - Assert.assertEquals(list.length(), 0); - } - - private void addNewType() throws Exception { - HierarchicalTypeDefinition<ClassType> testTypeDefinition = - TypesUtil.createClassTypeDef("test", ImmutableList.<String>of(), - TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE)); - - String typesAsJSON = TypesSerialization.toJson(testTypeDefinition); - createType(typesAsJSON); - } - - @Test(dependsOnMethods = "testSubmitEntity") - public void testGetTraitNames() throws Exception { - final String guid = tableId._getId(); - ClientResponse clientResponse = service - .path("api/metadata/entities") - .path(guid) - .path(TRAITS) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - Assert.assertNotNull(response.get("GUID")); - - final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); - Assert.assertEquals(list.length(), 7); - } - - @Test(dependsOnMethods = "testGetTraitNames") - public void testAddTrait() throws Exception { - traitName = "PII_Trait" + randomString(); - HierarchicalTypeDefinition<TraitType> piiTrait = - TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of()); - String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true); - LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON); - createType(traitDefinitionAsJSON); - - Struct traitInstance = new Struct(traitName); - String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true); - LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); - - final String guid = tableId._getId(); - ClientResponse clientResponse = service - .path("api/metadata/entities") - .path(guid) - .path(TRAITS) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - Assert.assertNotNull(response.get(MetadataServiceClient.GUID)); - } - - @Test(dependsOnMethods = "testAddTrait") - public void testAddExistingTrait() throws Exception { - final String traitName = "PII_Trait" + randomString(); - - Struct traitInstance = new Struct(traitName); - String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true); - LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); - - final String guid = tableId._getId(); - ClientResponse clientResponse = service - .path("api/metadata/entities") - .path(guid) - .path(TRAITS) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); - } - - @Test(dependsOnMethods = "testGetTraitNames") - public void testAddTraitWithAttribute() throws Exception { - final String traitName = "PII_Trait" + randomString(); - HierarchicalTypeDefinition<TraitType> piiTrait = - TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of(), - TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE)); - String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true); - LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON); - createType(traitDefinitionAsJSON); - - Struct traitInstance = new Struct(traitName); - traitInstance.set("type", "SSN"); - String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true); - LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); - - final String guid = tableId._getId(); - ClientResponse clientResponse = service - .path("api/metadata/entities") - .path(guid) - .path(TRAITS) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - Assert.assertNotNull(response.get(MetadataServiceClient.GUID)); - - // verify the response - clientResponse = getEntityDefinition(guid); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - - final String definition = response.getString(MetadataServiceClient.DEFINITION); - Assert.assertNotNull(definition); - Referenceable entityRef = InstanceSerialization.fromJsonReferenceable(definition, true); - IStruct traitRef = entityRef.getTrait(traitName); - String type = (String) traitRef.get("type"); - Assert.assertEquals(type, "SSN"); - } - - @Test - public void testAddTraitWithNoRegistration() throws Exception { - final String traitName = "PII_Trait" + randomString(); - HierarchicalTypeDefinition<TraitType> piiTrait = - TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of()); - String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true); - LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON); - - Struct traitInstance = new Struct(traitName); - String traitInstanceAsJSON = InstanceSerialization$.MODULE$.toJson(traitInstance, true); - LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); - - ClientResponse clientResponse = service - .path("api/metadata/entities") - .path("random") - .path(TRAITS) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); - } - - @Test(dependsOnMethods = "testAddTrait") - public void testDeleteTrait() throws Exception { - final String guid = tableId._getId(); - - ClientResponse clientResponse = service - .path("api/metadata/entities") - .path(guid) - .path(TRAITS) - .path(traitName) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.DELETE, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - Assert.assertNotNull(response.get("GUID")); - Assert.assertNotNull(response.get("traitName")); - } - - @Test - public void testDeleteTraitNonExistent() throws Exception { - final String traitName = "blah_trait"; - - ClientResponse clientResponse = service - .path("api/metadata/entities") - .path("random") - .path(TRAITS) - .path(traitName) - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.DELETE, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.ERROR)); - Assert.assertEquals(response.getString(MetadataServiceClient.ERROR), - "trait=" + traitName + " should be defined in type system before it can be deleted"); - Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE)); - } - - private String random() { - return RandomStringUtils.random(10); - } - - private String randomString() { - return RandomStringUtils.randomAlphanumeric(10); - } - - @Test - public void testUTF8() throws Exception { - String classType = random(); - String attrName = random(); - String attrValue = random(); - - HierarchicalTypeDefinition<ClassType> classTypeDefinition = - TypesUtil.createClassTypeDef(classType, ImmutableList.<String>of(), - TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE)); - TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), - ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.of(classTypeDefinition)); - createType(typesDef); - - Referenceable instance = new Referenceable(classType); - instance.set(attrName, attrValue); - Id guid = createInstance(instance); - - ClientResponse response = getEntityDefinition(guid._getId()); - String definition = getEntityDefinition(response); - Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true); - Assert.assertEquals(getReferenceable.get(attrName), attrValue); - } - - private void createHiveTypes() throws Exception { - HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = - TypesUtil.createClassTypeDef(DATABASE_TYPE, - ImmutableList.<String>of(), - TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE)); - - StructTypeDefinition structTypeDefinition = - new StructTypeDefinition("serdeType", - new AttributeDefinition[]{ - TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE) - }); - - EnumValue values[] = { - new EnumValue("MANAGED", 1), - new EnumValue("EXTERNAL", 2), - }; - - EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values); - - HierarchicalTypeDefinition<ClassType> tableTypeDefinition = - TypesUtil.createClassTypeDef(TABLE_TYPE, - ImmutableList.<String>of(), - TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), - TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("date", DataTypes.DATE_TYPE), - TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE), - new AttributeDefinition("tableType", "tableType", - Multiplicity.REQUIRED, false, null), - new AttributeDefinition("serde1", - "serdeType", Multiplicity.REQUIRED, false, null), - new AttributeDefinition("serde2", - "serdeType", Multiplicity.REQUIRED, false, null), - new AttributeDefinition("database", - DATABASE_TYPE, Multiplicity.REQUIRED, true, null), - new AttributeDefinition("compressed", - DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, true, null)); - - HierarchicalTypeDefinition<TraitType> classificationTraitDefinition = - TypesUtil.createTraitTypeDef("classification", - ImmutableList.<String>of(), - TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); - HierarchicalTypeDefinition<TraitType> piiTrait = - TypesUtil.createTraitTypeDef("pii", ImmutableList.<String>of()); - HierarchicalTypeDefinition<TraitType> phiTrait = - TypesUtil.createTraitTypeDef("phi", ImmutableList.<String>of()); - HierarchicalTypeDefinition<TraitType> pciTrait = - TypesUtil.createTraitTypeDef("pci", ImmutableList.<String>of()); - HierarchicalTypeDefinition<TraitType> soxTrait = - TypesUtil.createTraitTypeDef("sox", ImmutableList.<String>of()); - HierarchicalTypeDefinition<TraitType> secTrait = - TypesUtil.createTraitTypeDef("sec", ImmutableList.<String>of()); - HierarchicalTypeDefinition<TraitType> financeTrait = - TypesUtil.createTraitTypeDef("finance", ImmutableList.<String>of()); - - TypesDef typesDef = TypeUtils.getTypesDef( - ImmutableList.of(enumTypeDefinition), - ImmutableList.of(structTypeDefinition), - ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait, - soxTrait, secTrait, financeTrait), - ImmutableList.of(databaseTypeDefinition, tableTypeDefinition)); - createType(typesDef); - } - - private Referenceable createHiveTableInstance() throws Exception { - Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); - databaseInstance.set("name", DATABASE_NAME); - databaseInstance.set("description", "foo database"); - - Referenceable tableInstance = new Referenceable(TABLE_TYPE, - "classification", "pii", "phi", "pci", "sox", "sec", "finance"); - tableInstance.set("name", TABLE_NAME); - tableInstance.set("description", "bar table"); - tableInstance.set("date", "2014-07-11T08:00:00.000Z"); - tableInstance.set("type", "managed"); - tableInstance.set("level", 2); - tableInstance.set("tableType", 1); // enum - tableInstance.set("database", databaseInstance); - tableInstance.set("compressed", false); - - Struct traitInstance = (Struct) tableInstance.getTrait("classification"); - traitInstance.set("tag", "foundation_etl"); - - Struct serde1Instance = new Struct("serdeType"); - serde1Instance.set("name", "serde1"); - serde1Instance.set("serde", "serde1"); - tableInstance.set("serde1", serde1Instance); - - Struct serde2Instance = new Struct("serdeType"); - serde2Instance.set("name", "serde2"); - serde2Instance.set("serde", "serde2"); - tableInstance.set("serde2", serde2Instance); - - List<String> traits = tableInstance.getTraits(); - Assert.assertEquals(traits.size(), 7); - - return tableInstance; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java deleted file mode 100644 index a40f7ce..0000000 --- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java +++ /dev/null @@ -1,399 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.web.resources; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import org.apache.hadoop.metadata.MetadataServiceClient; -import org.apache.hadoop.metadata.typesystem.Referenceable; -import org.apache.hadoop.metadata.typesystem.TypesDef; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition; -import org.apache.hadoop.metadata.typesystem.types.ClassType; -import org.apache.hadoop.metadata.typesystem.types.DataTypes; -import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition; -import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition; -import org.apache.hadoop.metadata.typesystem.types.IDataType; -import org.apache.hadoop.metadata.typesystem.types.Multiplicity; -import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition; -import org.apache.hadoop.metadata.typesystem.types.TraitType; -import org.apache.hadoop.metadata.typesystem.types.TypeUtils; -import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil; -import org.apache.hadoop.metadata.web.util.Servlets; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONObject; -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.Response; -import java.util.List; - -/** - * Hive Lineage Integration Tests. - */ -public class HiveLineageJerseyResourceIT extends BaseResourceIT { - - private static final String BASE_URI = "api/metadata/lineage/hive/table/"; - - @BeforeClass - public void setUp() throws Exception { - super.setUp(); - - setUpTypes(); - setupInstances(); - } - - @Test - public void testInputsGraph() throws Exception { - WebResource resource = service - .path(BASE_URI) - .path("sales_fact_monthly_mv") - .path("inputs") - .path("graph"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - System.out.println("inputs graph = " + responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - - JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS); - Assert.assertNotNull(results); - - JSONObject values = results.getJSONObject("values"); - Assert.assertNotNull(values); - - final JSONObject vertices = values.getJSONObject("vertices"); - Assert.assertEquals(vertices.length(), 4); - - final JSONObject edges = values.getJSONObject("edges"); - Assert.assertEquals(edges.length(), 4); - } - - @Test - public void testOutputsGraph() throws Exception { - WebResource resource = service - .path(BASE_URI) - .path("sales_fact") - .path("outputs") - .path("graph"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - System.out.println("outputs graph= " + responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - - JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS); - Assert.assertNotNull(results); - - JSONObject values = results.getJSONObject("values"); - Assert.assertNotNull(values); - - final JSONObject vertices = values.getJSONObject("vertices"); - Assert.assertEquals(vertices.length(), 3); - - final JSONObject edges = values.getJSONObject("edges"); - Assert.assertEquals(edges.length(), 4); - } - - @Test - public void testSchema() throws Exception { - WebResource resource = service - .path(BASE_URI) - .path("sales_fact") - .path("schema"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); - - String responseAsString = clientResponse.getEntity(String.class); - Assert.assertNotNull(responseAsString); - System.out.println("schema = " + responseAsString); - - JSONObject response = new JSONObject(responseAsString); - Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); - - JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS); - Assert.assertNotNull(results); - - JSONArray rows = results.getJSONArray("rows"); - Assert.assertEquals(rows.length(), 4); - - for (int index = 0; index < rows.length(); index++) { - final JSONObject row = rows.getJSONObject(index); - Assert.assertNotNull(row.getString("name")); - Assert.assertNotNull(row.getString("comment")); - Assert.assertNotNull(row.getString("dataType")); - Assert.assertEquals(row.getString("$typeName$"), "hive_column"); - } - } - - @Test - public void testSchemaForEmptyTable() throws Exception { - WebResource resource = service - .path(BASE_URI) - .path("") - .path("schema"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), - Response.Status.NOT_FOUND.getStatusCode()); - } - - @Test - public void testSchemaForInvalidTable() throws Exception { - WebResource resource = service - .path(BASE_URI) - .path("blah") - .path("schema"); - - ClientResponse clientResponse = resource - .accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE) - .method(HttpMethod.GET, ClientResponse.class); - Assert.assertEquals(clientResponse.getStatus(), - Response.Status.NOT_FOUND.getStatusCode()); - } - - private void setUpTypes() throws Exception { - TypesDef typesDef = createTypeDefinitions(); - createType(typesDef); - } - - private static final String DATABASE_TYPE = "hive_db"; - private static final String HIVE_TABLE_TYPE = "hive_table"; - private static final String COLUMN_TYPE = "hive_column"; - private static final String HIVE_PROCESS_TYPE = "hive_process"; - - private TypesDef createTypeDefinitions() { - HierarchicalTypeDefinition<ClassType> dbClsDef - = TypesUtil.createClassTypeDef(DATABASE_TYPE, null, - attrDef("name", DataTypes.STRING_TYPE), - attrDef("description", DataTypes.STRING_TYPE), - attrDef("locationUri", DataTypes.STRING_TYPE), - attrDef("owner", DataTypes.STRING_TYPE), - attrDef("createTime", DataTypes.INT_TYPE) - ); - - HierarchicalTypeDefinition<ClassType> columnClsDef = - TypesUtil.createClassTypeDef(COLUMN_TYPE, null, - attrDef("name", DataTypes.STRING_TYPE), - attrDef("dataType", DataTypes.STRING_TYPE), - attrDef("comment", DataTypes.STRING_TYPE) - ); - - HierarchicalTypeDefinition<ClassType> tblClsDef = - TypesUtil.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), - attrDef("owner", DataTypes.STRING_TYPE), - attrDef("createTime", DataTypes.INT_TYPE), - attrDef("lastAccessTime", DataTypes.INT_TYPE), - attrDef("tableType", DataTypes.STRING_TYPE), - attrDef("temporary", DataTypes.BOOLEAN_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, - Multiplicity.REQUIRED, false, null), - new AttributeDefinition("columns", - DataTypes.arrayTypeName(COLUMN_TYPE), - Multiplicity.COLLECTION, true, null) - ); - - HierarchicalTypeDefinition<ClassType> loadProcessClsDef = - TypesUtil.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), - attrDef("userName", DataTypes.STRING_TYPE), - attrDef("startTime", DataTypes.INT_TYPE), - attrDef("endTime", DataTypes.INT_TYPE), - attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), - attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED) - ); - - HierarchicalTypeDefinition<TraitType> dimTraitDef = - TypesUtil.createTraitTypeDef("Dimension", null); - - HierarchicalTypeDefinition<TraitType> factTraitDef = - TypesUtil.createTraitTypeDef("Fact", null); - - HierarchicalTypeDefinition<TraitType> metricTraitDef = - TypesUtil.createTraitTypeDef("Metric", null); - - HierarchicalTypeDefinition<TraitType> etlTraitDef = - TypesUtil.createTraitTypeDef("ETL", null); - - - HierarchicalTypeDefinition<TraitType> piiTraitDef = - TypesUtil.createTraitTypeDef("PII", null); - - return TypeUtils.getTypesDef( - ImmutableList.<EnumTypeDefinition>of(), - ImmutableList.<StructTypeDefinition>of(), - ImmutableList.of(dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef, piiTraitDef), - ImmutableList.of(dbClsDef, columnClsDef, tblClsDef, loadProcessClsDef) - ); - } - - AttributeDefinition attrDef(String name, IDataType dT) { - return attrDef(name, dT, Multiplicity.OPTIONAL, false, null); - } - - AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) { - return attrDef(name, dT, m, false, null); - } - - AttributeDefinition attrDef(String name, IDataType dT, - Multiplicity m, boolean isComposite, String reverseAttributeName) { - Preconditions.checkNotNull(name); - Preconditions.checkNotNull(dT); - return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName); - } - - private void setupInstances() throws Exception { - Id salesDB = database( - "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); - - List<Referenceable> salesFactColumns = ImmutableList.of( - column("time_id", "int", "time id"), - column("product_id", "int", "product id"), - column("customer_id", "int", "customer id", "PII"), - column("sales", "double", "product id", "Metric") - ); - - Id salesFact = table("sales_fact", "sales fact table", - salesDB, "Joe", "Managed", salesFactColumns, "Fact"); - - List<Referenceable> timeDimColumns = ImmutableList.of( - column("time_id", "int", "time id"), - column("dayOfYear", "int", "day Of Year"), - column("weekDay", "int", "week Day") - ); - - Id timeDim = table("time_dim", "time dimension table", - salesDB, "John Doe", "External", timeDimColumns, "Dimension"); - - Id reportingDB = database("Reporting", "reporting database", "Jane BI", - "hdfs://host:8000/apps/warehouse/reporting"); - - Id salesFactDaily = table("sales_fact_daily_mv", - "sales fact daily materialized view", - reportingDB, "Joe BI", "Managed", salesFactColumns, "Metric"); - - loadProcess("loadSalesDaily", "John ETL", - ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily), - "create table as select ", "plan", "id", "graph", - "ETL"); - - Id salesFactMonthly = table("sales_fact_monthly_mv", - "sales fact monthly materialized view", - reportingDB, "Jane BI", "Managed", salesFactColumns, "Metric"); - - loadProcess("loadSalesMonthly", "John ETL", - ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly), - "create table as select ", "plan", "id", "graph", - "ETL"); - } - - Id database(String name, String description, - String owner, String locationUri, - String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("description", description); - referenceable.set("owner", owner); - referenceable.set("locationUri", locationUri); - referenceable.set("createTime", System.currentTimeMillis()); - - return createInstance(referenceable); - } - - Referenceable column(String name, String dataType, String comment, - String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("dataType", dataType); - referenceable.set("comment", comment); - - return referenceable; - } - - Id table(String name, String description, Id dbId, - String owner, String tableType, - List<Referenceable> columns, - String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("description", description); - referenceable.set("owner", owner); - referenceable.set("tableType", tableType); - referenceable.set("createTime", System.currentTimeMillis()); - referenceable.set("lastAccessTime", System.currentTimeMillis()); - referenceable.set("retention", System.currentTimeMillis()); - - referenceable.set("db", dbId); - referenceable.set("columns", columns); - - return createInstance(referenceable); - } - - Id loadProcess(String name, String user, - List<Id> inputTables, - List<Id> outputTables, - String queryText, String queryPlan, - String queryId, String queryGraph, - String... traitNames) throws Exception { - Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames); - referenceable.set("name", name); - referenceable.set("user", user); - referenceable.set("startTime", System.currentTimeMillis()); - referenceable.set("endTime", System.currentTimeMillis() + 10000); - - referenceable.set("inputs", inputTables); - referenceable.set("outputs", outputTables); - - referenceable.set("queryText", queryText); - referenceable.set("queryPlan", queryPlan); - referenceable.set("queryId", queryId); - referenceable.set("queryGraph", queryGraph); - - return createInstance(referenceable); - } -}
