Repository: incubator-atlas Updated Branches: refs/heads/master 266d7cc00 -> 9d3037433
ATLAS-80 Support for variables in application properties (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/9d303743 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/9d303743 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/9d303743 Branch: refs/heads/master Commit: 9d303743382549b46aad56f78c650b466efa69f1 Parents: 266d7cc Author: Shwetha GS <[email protected]> Authored: Mon Jul 27 12:21:45 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Mon Jul 27 12:21:45 2015 +0530 ---------------------------------------------------------------------- .../hook/NegativeSSLAndKerberosHiveHookIT.java | 4 +- .../hive/hook/SSLAndKerberosHiveHookIT.java | 3 +- .../org/apache/atlas/ApplicationProperties.java | 124 +++++++++++++++++++ .../main/java/org/apache/atlas/AtlasClient.java | 7 +- .../java/org/apache/atlas/PropertiesUtil.java | 58 --------- .../atlas/security/SecureClientUtils.java | 9 +- .../atlas/security/SecurityProperties.java | 24 ++-- .../apache/atlas/ApplicationPropertiesTest.java | 55 ++++++++ .../src/test/resources/application.properties | 29 +++++ docs/pom.xml | 53 +------- pom.xml | 5 + release-log.txt | 1 + .../atlas/discovery/HiveLineageService.java | 8 +- .../repository/graph/TitanGraphProvider.java | 29 +---- src/bin/atlas_start.py | 4 - src/conf/application.properties | 4 +- src/test/python/scripts/TestMetadata.py | 4 +- webapp/src/main/java/org/apache/atlas/Main.java | 9 +- .../web/filters/AtlasAuthenticationFilter.java | 25 ++-- .../atlas/web/listeners/GuiceServletConfig.java | 7 +- .../atlas/web/listeners/LoginProcessor.java | 25 ++-- .../atlas/web/service/EmbeddedServer.java | 3 +- .../atlas/web/listeners/LoginProcessorIT.java | 4 +- 23 files changed, 281 insertions(+), 213 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java index 072c36b..891e06c 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java @@ -18,7 +18,6 @@ package org.apache.atlas.hive.hook; -import org.apache.atlas.PropertiesUtil; import org.apache.atlas.security.SecurityProperties; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.RandomStringUtils; @@ -100,11 +99,12 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { String confLocation = System.getProperty("atlas.conf"); URL url; if (confLocation == null) { - url = PropertiesUtil.class.getResource("/application.properties"); + url = NegativeSSLAndKerberosHiveHookIT.class.getResource("/application.properties"); } else { url = new File(confLocation, "application.properties").toURI().toURL(); } configuration.load(url); + configuration.setProperty(TLS_ENABLED, true); configuration.setProperty("atlas.http.authentication.enabled", "true"); configuration.setProperty("atlas.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java index 6fab2ee..7471680 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java @@ -20,7 +20,6 @@ package org.apache.atlas.hive.hook; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; -import org.apache.atlas.PropertiesUtil; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.security.SecurityProperties; import org.apache.commons.configuration.PropertiesConfiguration; @@ -116,7 +115,7 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { String confLocation = System.getProperty("atlas.conf"); URL url; if (confLocation == null) { - url = PropertiesUtil.class.getResource("/application.properties"); + url = SSLAndKerberosHiveHookIT.class.getResource("/application.properties"); } else { url = new File(confLocation, "application.properties").toURI().toURL(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/client/src/main/java/org/apache/atlas/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/ApplicationProperties.java b/client/src/main/java/org/apache/atlas/ApplicationProperties.java new file mode 100644 index 0000000..ad87d8d --- /dev/null +++ b/client/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URL; +import java.util.Iterator; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ApplicationProperties extends PropertiesConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class); + + public static final String APPLICATION_PROPERTIES = "application.properties"; + public static final String CLIENT_PROPERTIES = "client.properties"; + + private static Configuration INSTANCE = null; + + private ApplicationProperties(URL url) throws ConfigurationException { + super(url); + } + + public static Configuration get() throws AtlasException { + if (INSTANCE == null) { + synchronized (ApplicationProperties.class) { + if (INSTANCE == null) { + INSTANCE = get(APPLICATION_PROPERTIES); + } + } + } + return INSTANCE; + } + + public static Configuration get(String fileName) throws AtlasException { + String confLocation = System.getProperty("atlas.conf"); + try { + URL url = confLocation == null ? ApplicationProperties.class.getResource("/" + fileName) + : new File(confLocation, fileName).toURI().toURL(); + LOG.info("Loading {} from {}", fileName, url); + + ApplicationProperties configuration = new ApplicationProperties(url); + Iterator<String> keys = configuration.getKeys(); + LOG.debug("Configuration loaded:"); + while(keys.hasNext()) { + String key = keys.next(); + LOG.debug("{} = {}", key, configuration.getProperty(key)); + } + return configuration; + } catch (Exception e) { + throw new AtlasException("Failed to load application properties", e); + } + } + + public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) { + return inConf.subset(prefix); + } + + @Override + public Object getProperty(String key) { + Object value = super.getProperty(key); + if (value instanceof String) { + value = substituteVars((String) value); + } + return value; + } + + private static final Pattern VAR_PATTERN = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); + + private static final int MAX_SUBST = 20; + + private String substituteVars(String expr) { + if (expr == null) { + return null; + } + Matcher match = VAR_PATTERN.matcher(""); + String eval = expr; + + for(int s = 0; s < MAX_SUBST; s++) { + match.reset(eval); + if (!match.find()) { + return eval; + } + String var = match.group(); + var = var.substring(2, var.length() - 1); // remove ${ .. } + String val = null; + try { + val = System.getProperty(var); + } catch(SecurityException se) { + LOG.warn("Unexpected SecurityException in Configuration", se); + } + if (val == null) { + val = getString(var); + } + if (val == null) { + return eval; // return literal ${var}: var is unbound + } + + // substitute + eval = eval.substring(0, match.start()) + val + eval.substring(match.end()); + } + throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/client/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java index ff03bc1..1c7d62a 100755 --- a/client/src/main/java/org/apache/atlas/AtlasClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasClient.java @@ -26,6 +26,7 @@ import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; import org.apache.atlas.security.SecureClientUtils; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.json.InstanceSerialization; +import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.hadoop.security.UserGroupInformation; import org.codehaus.jettison.json.JSONArray; @@ -87,7 +88,7 @@ public class AtlasClient { public AtlasClient(String baseUrl, UserGroupInformation ugi, String doAsUser) { DefaultClientConfig config = new DefaultClientConfig(); - PropertiesConfiguration clientConfig = null; + Configuration clientConfig = null; try { clientConfig = getClientProperties(); if (clientConfig.getBoolean(TLS_ENABLED, false)) { @@ -109,8 +110,8 @@ public class AtlasClient { service = client.resource(UriBuilder.fromUri(baseUrl).build()); } - protected PropertiesConfiguration getClientProperties() throws AtlasException { - return PropertiesUtil.getClientProperties(); + protected Configuration getClientProperties() throws AtlasException { + return ApplicationProperties.get(ApplicationProperties.CLIENT_PROPERTIES); } enum API { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/client/src/main/java/org/apache/atlas/PropertiesUtil.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/PropertiesUtil.java b/client/src/main/java/org/apache/atlas/PropertiesUtil.java deleted file mode 100644 index bc56cbf..0000000 --- a/client/src/main/java/org/apache/atlas/PropertiesUtil.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas; - -import org.apache.commons.configuration.PropertiesConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.net.URL; - -public class PropertiesUtil { - private static final Logger LOG = LoggerFactory.getLogger(PropertiesUtil.class); - - private static final String APPLICATION_PROPERTIES = "application.properties"; - public static final String CLIENT_PROPERTIES = "client.properties"; - - public static PropertiesConfiguration getApplicationProperties() throws AtlasException { - return getPropertiesConfiguration(APPLICATION_PROPERTIES); - } - - public static PropertiesConfiguration getClientProperties() throws AtlasException { - return getPropertiesConfiguration(CLIENT_PROPERTIES); - } - - private static PropertiesConfiguration getPropertiesConfiguration(String name) throws AtlasException { - String confLocation = System.getProperty("atlas.conf"); - URL url; - try { - if (confLocation == null) { - url = PropertiesUtil.class.getResource("/" + name); - } else { - url = new File(confLocation, name).toURI().toURL(); - } - LOG.info("Loading {} from {}", name, url); - return new PropertiesConfiguration(url); - } catch (Exception e) { - throw new AtlasException("Failed to load application properties", e); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java b/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java index eb55b4a..b70c5ce 100644 --- a/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java +++ b/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java @@ -20,8 +20,6 @@ import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; import org.apache.atlas.AtlasException; -import org.apache.atlas.PropertiesUtil; -import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProviderFactory; @@ -62,7 +60,8 @@ public class SecureClientUtils { public static URLConnectionClientHandler getClientConnectionHandler(DefaultClientConfig config, - PropertiesConfiguration clientConfig, final String doAsUser, final UserGroupInformation ugi) { + org.apache.commons.configuration.Configuration clientConfig, final String doAsUser, + final UserGroupInformation ugi) { config.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true); Configuration conf = new Configuration(); conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, "ssl-client.xml")); @@ -177,7 +176,7 @@ public class SecureClientUtils { try { if (confLocation == null) { String persistDir = null; - URL resource = PropertiesUtil.class.getResource("/"); + URL resource = SecureClientUtils.class.getResource("/"); if (resource != null) { persistDir = resource.toURI().getPath(); } @@ -193,7 +192,7 @@ public class SecureClientUtils { return new File(sslDir, SecurityProperties.SSL_CLIENT_PROPERTIES); } - public static void persistSSLClientConfiguration(PropertiesConfiguration clientConfig) + public static void persistSSLClientConfiguration(org.apache.commons.configuration.Configuration clientConfig) throws AtlasException, IOException { //trust settings Configuration configuration = new Configuration(false); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/client/src/main/java/org/apache/atlas/security/SecurityProperties.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/security/SecurityProperties.java b/client/src/main/java/org/apache/atlas/security/SecurityProperties.java index e03ffcc..5029e12 100644 --- a/client/src/main/java/org/apache/atlas/security/SecurityProperties.java +++ b/client/src/main/java/org/apache/atlas/security/SecurityProperties.java @@ -21,16 +21,16 @@ package org.apache.atlas.security; * */ public interface SecurityProperties { - public static final String TLS_ENABLED = "atlas.enableTLS"; - public static final String KEYSTORE_FILE_KEY = "keystore.file"; - public static final String DEFAULT_KEYSTORE_FILE_LOCATION = "target/atlas.keystore"; - public static final String KEYSTORE_PASSWORD_KEY = "keystore.password"; - public static final String TRUSTSTORE_FILE_KEY = "truststore.file"; - public static final String DEFATULT_TRUSTORE_FILE_LOCATION = "target/atlas.keystore"; - public static final String TRUSTSTORE_PASSWORD_KEY = "truststore.password"; - public static final String SERVER_CERT_PASSWORD_KEY = "password"; - public static final String CLIENT_AUTH_KEY = "client.auth.enabled"; - public static final String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path"; - public static final String SSL_CLIENT_PROPERTIES = "ssl-client.xml"; - public static final String BIND_ADDRESS = "atlas.server.bind.address"; + String TLS_ENABLED = "atlas.enableTLS"; + String KEYSTORE_FILE_KEY = "keystore.file"; + String DEFAULT_KEYSTORE_FILE_LOCATION = "target/atlas.keystore"; + String KEYSTORE_PASSWORD_KEY = "keystore.password"; + String TRUSTSTORE_FILE_KEY = "truststore.file"; + String DEFATULT_TRUSTORE_FILE_LOCATION = "target/atlas.keystore"; + String TRUSTSTORE_PASSWORD_KEY = "truststore.password"; + String SERVER_CERT_PASSWORD_KEY = "password"; + String CLIENT_AUTH_KEY = "client.auth.enabled"; + String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path"; + String SSL_CLIENT_PROPERTIES = "ssl-client.xml"; + String BIND_ADDRESS = "atlas.server.bind.address"; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/client/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java ---------------------------------------------------------------------- diff --git a/client/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java b/client/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java new file mode 100644 index 0000000..4acb3e1 --- /dev/null +++ b/client/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import org.apache.commons.configuration.Configuration; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ApplicationPropertiesTest { + + @Test + public void testVariables() throws Exception { + Configuration properties = ApplicationProperties.get(); + + //plain property without variables + Assert.assertEquals(properties.getString("atlas.service"), "atlas"); + + //property containing system property + String data = "/var/data/" + System.getProperty("user.name") + "/atlas"; + Assert.assertEquals(properties.getString("atlas.data"), data); + + //property referencing other property + Assert.assertEquals(properties.getString("atlas.graph.data"), data + "/graph"); + + //invalid system property - not substituted + Assert.assertEquals(properties.getString("atlas.db"), "${atlasdb}"); + } + + @Test + //variable substitutions should work with subset configuration as well + public void testSubset() throws Exception { + Configuration configuration = ApplicationProperties.get(); + Configuration subConfiguration = configuration.subset("atlas"); + + Assert.assertEquals(subConfiguration.getString("service"), "atlas"); + String data = "/var/data/" + System.getProperty("user.name") + "/atlas"; + Assert.assertEquals(subConfiguration.getString("data"), data); + Assert.assertEquals(subConfiguration.getString("graph.data"), data + "/graph"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/client/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/client/src/test/resources/application.properties b/client/src/test/resources/application.properties new file mode 100644 index 0000000..9672d1e --- /dev/null +++ b/client/src/test/resources/application.properties @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#system property +atlas.data=/var/data/${user.name}/atlas + +#re-use existing property +atlas.graph.data=${atlas.data}/graph + +#plain property +atlas.service=atlas + +#invalid system property +atlas.db=${atlasdb} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/docs/pom.xml ---------------------------------------------------------------------- diff --git a/docs/pom.xml b/docs/pom.xml index 55f5f8d..4535fe0 100755 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -32,39 +32,9 @@ <properties> <skipTests>true</skipTests> + <skipDocs>false</skipDocs> </properties> -<!-- - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-site-plugin</artifactId> - <dependencies> - <dependency> - <groupId>org.apache.maven.doxia</groupId> - <artifactId>doxia-module-twiki</artifactId> - <version>1.3</version> - </dependency> - </dependencies> - <executions> - <execution> - <goals> - <goal>site</goal> - </goals> - <phase>prepare-package</phase> - </execution> - </executions> - <configuration> - <generateProjectInfo>false</generateProjectInfo> - <generateReports>false</generateReports> - <skip>false</skip> - </configuration> - </plugin> - </plugins> - </build> ---> - <build> <plugins> <plugin> @@ -92,7 +62,7 @@ </execution> </executions> <configuration> - <!--<outputDirectory>../../site</outputDirectory>--> + <skip>${skipDocs}</skip> <reportPlugins> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -131,25 +101,6 @@ </reportPlugins> </configuration> </plugin> -<!-- - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-war-plugin</artifactId> - <version>2.3</version> - <configuration> - <webResources> - <resource> - <directory>src/site/resources</directory> - <targetPath>pages</targetPath> - </resource> - <resource> - <directory>target/site</directory> - <targetPath>pages</targetPath> - </resource> - </webResources> - </configuration> - </plugin> ---> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a8afcc3..c201b4c 100755 --- a/pom.xml +++ b/pom.xml @@ -1304,6 +1304,7 @@ <plugin> <artifactId>maven-assembly-plugin</artifactId> + <inherited>false</inherited> <configuration> <descriptors> <descriptor>src/main/assemblies/standalone-package.xml</descriptor> @@ -1318,6 +1319,9 @@ <artifactId>maven-surefire-plugin</artifactId> <version>2.18.1</version> <configuration> + <systemProperties> + <user.dir>${project.basedir}</user.dir> + </systemProperties> <!--<skipTests>true</skipTests>--> <forkMode>always</forkMode> <redirectTestOutputToFile>true</redirectTestOutputToFile> @@ -1536,6 +1540,7 @@ </execution> </executions> </plugin> + <plugin> <artifactId>maven-clean-plugin</artifactId> <configuration> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 76b8778..7a7329c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ALL CHANGES: +ATLAS-80 Support for variables in application properties (shwethags) ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags) ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found ([email protected] via shwethags) ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java index 76d94a3..7a6ff55 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java @@ -19,10 +19,10 @@ package org.apache.atlas.discovery; import com.thinkaurelius.titan.core.TitanGraph; +import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.GraphTransaction; import org.apache.atlas.ParamChecker; -import org.apache.atlas.PropertiesUtil; import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.query.Expressions; @@ -33,7 +33,7 @@ import org.apache.atlas.repository.EntityNotFoundException; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.typesystem.persistence.ReferenceableInstance; -import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Option; @@ -63,12 +63,12 @@ public class HiveLineageService implements LineageService { private static final String HIVE_TABLE_EXISTS_QUERY; - private static final PropertiesConfiguration propertiesConf; + private static final Configuration propertiesConf; static { // todo - externalize this using type system - dog food try { - propertiesConf = PropertiesUtil.getApplicationProperties(); + propertiesConf = ApplicationProperties.get(); HIVE_TABLE_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.table.type.name", "DataSet"); HIVE_PROCESS_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.process.type.name", "Process"); HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.inputs.name", "inputs"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java index d7ed1f1..5e61b9a 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java @@ -21,16 +21,13 @@ package org.apache.atlas.repository.graph; import com.google.inject.Provides; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; +import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; -import org.apache.atlas.PropertiesUtil; import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.PropertiesConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Singleton; -import java.util.Iterator; -import java.util.Properties; /** * Default implementation for Graph Provider that doles out Titan Graph. @@ -42,31 +39,13 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> { /** * Constant for the configuration property that indicates the prefix. */ - private static final String ATLAS_PREFIX = "atlas.graph."; + private static final String GRAPH_PREFIX = "atlas.graph"; private static TitanGraph graphInstance; private static Configuration getConfiguration() throws AtlasException { - PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties(); - - Configuration graphConfig = new PropertiesConfiguration(); - - Properties sysProperties = System.getProperties(); - LOG.info("System properties: "); - LOG.info(sysProperties.toString()); - - final Iterator<String> iterator = configProperties.getKeys(); - while (iterator.hasNext()) { - String key = iterator.next(); - if (key.startsWith(ATLAS_PREFIX)) { - String value = (String) configProperties.getProperty(key); - key = key.substring(ATLAS_PREFIX.length()); - graphConfig.setProperty(key, value); - LOG.info("Using graph property {}={}", key, value); - } - } - - return graphConfig; + Configuration configProperties = ApplicationProperties.get(); + return ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/src/bin/atlas_start.py ---------------------------------------------------------------------- diff --git a/src/bin/atlas_start.py b/src/bin/atlas_start.py index 0ad188c..66e8735 100755 --- a/src/bin/atlas_start.py +++ b/src/bin/atlas_start.py @@ -24,7 +24,6 @@ import atlas_config as mc METADATA_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=application.log" METADATA_COMMAND_OPTS="-Datlas.home=%s" METADATA_CONFIG_OPTS="-Datlas.conf=%s" -ATLAS_USER_DIR="-Duser.dir=%s" DEFAULT_JVM_OPTS="-Xmx1024m" def main(): @@ -40,9 +39,6 @@ def main(): cmd_opts = (METADATA_COMMAND_OPTS % metadata_home) jvm_opts_list.extend(cmd_opts.split()) - user_dir = (ATLAS_USER_DIR % metadata_home) - jvm_opts_list.extend(user_dir.split()) - config_opts = (METADATA_CONFIG_OPTS % confdir) jvm_opts_list.extend(config_opts.split()) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/src/conf/application.properties ---------------------------------------------------------------------- diff --git a/src/conf/application.properties b/src/conf/application.properties index 5c8c59a..51f2529 100755 --- a/src/conf/application.properties +++ b/src/conf/application.properties @@ -19,7 +19,7 @@ ######### Graph Database Configs ######### # Graph Storage atlas.graph.storage.backend=berkeleyje -atlas.graph.storage.directory=data/berkley +atlas.graph.storage.directory=${atlas.home}/data/berkley #Hbase as stoarge backend #hbase @@ -40,7 +40,7 @@ atlas.graph.storage.directory=data/berkley # Graph Search Index atlas.graph.index.search.backend=elasticsearch -atlas.graph.index.search.directory=data/es +atlas.graph.index.search.directory=${atlas.home}/data/es atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.local-mode=true atlas.graph.index.search.elasticsearch.create.sleep=2000 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/src/test/python/scripts/TestMetadata.py ---------------------------------------------------------------------- diff --git a/src/test/python/scripts/TestMetadata.py b/src/test/python/scripts/TestMetadata.py index 9ca346d..bb250cb 100644 --- a/src/test/python/scripts/TestMetadata.py +++ b/src/test/python/scripts/TestMetadata.py @@ -51,13 +51,13 @@ class TestMetadata(unittest.TestCase): 'org.apache.atlas.Main', ['-app', 'metadata_home/server/webapp/atlas'], 'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib\\*:metadata_home/libext\\*', - ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Duser.dir=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs') + ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs') else: java_mock.assert_called_with( 'org.apache.atlas.Main', ['-app', 'metadata_home/server/webapp/atlas'], 'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib/*:metadata_home/libext/*', - ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Duser.dir=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs') + ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs') pass def test_jar_java_lookups_fail(self): http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/webapp/src/main/java/org/apache/atlas/Main.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/Main.java b/webapp/src/main/java/org/apache/atlas/Main.java index 6cdeca8..b71ca82 100755 --- a/webapp/src/main/java/org/apache/atlas/Main.java +++ b/webapp/src/main/java/org/apache/atlas/Main.java @@ -24,6 +24,7 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -76,7 +77,7 @@ public final class Main { } setApplicationHome(); - PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties(); + Configuration configuration = ApplicationProperties.get(); final String enableTLSFlag = configuration.getString("atlas.enableTLS"); final int appPort = getApplicationPort(cmd, enableTLSFlag, configuration); final boolean enableTLS = isTLSEnabled(enableTLSFlag, appPort); @@ -100,9 +101,7 @@ public final class Main { return buildConfiguration.getString("project.version"); } - static int getApplicationPort(CommandLine cmd, - String enableTLSFlag, - PropertiesConfiguration configuration) { + static int getApplicationPort(CommandLine cmd, String enableTLSFlag, Configuration configuration) { final int appPort; if (cmd.hasOption(APP_PORT)) { appPort = Integer.valueOf(cmd.getOptionValue(APP_PORT)); @@ -114,7 +113,7 @@ public final class Main { return appPort; } - private static int getPortValue(PropertiesConfiguration configuration, String enableTLSFlag) { + private static int getPortValue(Configuration configuration, String enableTLSFlag) { int appPort; assert configuration != null; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java index 80d586b..ba04da2 100644 --- a/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java @@ -19,9 +19,10 @@ package org.apache.atlas.web.filters; import com.google.inject.Singleton; -import org.apache.atlas.PropertiesUtil; +import org.apache.atlas.ApplicationProperties; import org.apache.atlas.security.SecurityProperties; -import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationConverter; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; @@ -44,18 +45,20 @@ import java.util.Properties; @Singleton public class AtlasAuthenticationFilter extends AuthenticationFilter { private static final Logger LOG = LoggerFactory.getLogger(AtlasAuthenticationFilter.class); - static final String PREFIX = "atlas.http.authentication."; + static final String PREFIX = "atlas.http.authentication"; @Override protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException { - PropertiesConfiguration configuration; + Configuration configuration; try { - configuration = PropertiesUtil.getApplicationProperties(); + configuration = ApplicationProperties.get(); } catch (Exception e) { throw new ServletException(e); } - Properties config = new Properties(); + // transfer application.properties config items starting with defined prefix + Configuration subConfiguration = ApplicationProperties.getSubsetConfiguration(configuration, PREFIX); + Properties config = ConfigurationConverter.getProperties(subConfiguration); config.put(AuthenticationFilter.COOKIE_PATH, "/"); @@ -65,16 +68,6 @@ public class AtlasAuthenticationFilter extends AuthenticationFilter { String name = enumeration.nextElement(); config.put(name, filterConfig.getInitParameter(name)); } - // transfer application.properties config items starting with defined prefix - Iterator<String> itor = configuration.getKeys(); - while (itor.hasNext()) { - String name = itor.next(); - if (name.startsWith(PREFIX)) { - String value = configuration.getString(name); - name = name.substring(PREFIX.length()); - config.put(name, value); - } - } //Resolve _HOST into bind address String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java index b643f13..8371d9c 100755 --- a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java +++ b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java @@ -29,19 +29,18 @@ import com.sun.jersey.guice.JerseyServletModule; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Graph; +import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; -import org.apache.atlas.PropertiesUtil; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.web.filters.AtlasAuthenticationFilter; import org.apache.atlas.web.filters.AuditFilter; +import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; import javax.servlet.ServletContextEvent; import java.util.HashMap; import java.util.Map; @@ -85,7 +84,7 @@ public class GuiceServletConfig extends GuiceServletContextListener { private void configureAuthenticationFilter() throws ConfigurationException { try { - PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties(); + Configuration configuration = ApplicationProperties.get(); if (Boolean.valueOf(configuration.getString(HTTP_AUTHENTICATION_ENABLED))) { filter("/*").through(AtlasAuthenticationFilter.class); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java b/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java index 5f180f5..e17d669 100644 --- a/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java +++ b/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java @@ -16,11 +16,10 @@ */ package org.apache.atlas.web.listeners; +import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; -import org.apache.atlas.PropertiesUtil; import org.apache.atlas.security.SecurityProperties; import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -51,12 +50,7 @@ public class LoginProcessor { // first, let's see if we're running in a hadoop cluster and have the env configured boolean isHadoopCluster = isHadoopCluster(); Configuration hadoopConfig = isHadoopCluster ? getHadoopConfiguration() : new Configuration(false); - PropertiesConfiguration configuration = null; - try { - configuration = getPropertiesConfiguration(); - } catch (ConfigurationException e) { - LOG.warn("Error reading application configuration", e); - } + org.apache.commons.configuration.Configuration configuration = getApplicationConfiguration(); if (!isHadoopCluster) { // need to read the configured authentication choice and create the UGI configuration setupHadoopConfiguration(hadoopConfig, configuration); @@ -64,7 +58,8 @@ public class LoginProcessor { doServiceLogin(hadoopConfig, configuration); } - protected void doServiceLogin(Configuration hadoopConfig, PropertiesConfiguration configuration) { + protected void doServiceLogin(Configuration hadoopConfig, + org.apache.commons.configuration.Configuration configuration) { UserGroupInformation.setConfiguration(hadoopConfig); UserGroupInformation ugi = null; @@ -85,7 +80,7 @@ public class LoginProcessor { } } - private String getHostname(PropertiesConfiguration configuration) { + private String getHostname(org.apache.commons.configuration.Configuration configuration) { String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS); if (bindAddress == null) { LOG.info("No host name configured. Defaulting to local host name."); @@ -98,7 +93,8 @@ public class LoginProcessor { return bindAddress; } - protected void setupHadoopConfiguration(Configuration hadoopConfig, PropertiesConfiguration configuration) { + protected void setupHadoopConfiguration(Configuration hadoopConfig, org.apache.commons.configuration.Configuration + configuration) { String authMethod; authMethod = configuration != null ? configuration.getString(AUTHENTICATION_METHOD) : null; // getString may return null, and would like to log the nature of the default setting @@ -135,12 +131,13 @@ public class LoginProcessor { * @return the metadata configuration. * @throws ConfigurationException */ - protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException { + protected org.apache.commons.configuration.Configuration getApplicationConfiguration() { try { - return PropertiesUtil.getApplicationProperties(); + return ApplicationProperties.get(); } catch (AtlasException e) { - throw new ConfigurationException(e); + LOG.warn("Error reading application configuration", e); } + return null; } /** http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java index 16591d8..80df87d 100755 --- a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java @@ -23,9 +23,8 @@ import org.apache.commons.configuration.PropertiesConfiguration; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.webapp.WebAppClassLoader; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.webapp.WebAppContext; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9d303743/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java index 2148fe2..42692cd 100644 --- a/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java @@ -38,7 +38,7 @@ public class LoginProcessorIT extends BaseSecurityTest { public void testDefaultSimpleLogin() throws Exception { LoginProcessor processor = new LoginProcessor() { @Override - protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException { + protected org.apache.commons.configuration.Configuration getApplicationConfiguration() { return new PropertiesConfiguration(); } }; @@ -55,7 +55,7 @@ public class LoginProcessorIT extends BaseSecurityTest { LoginProcessor processor = new LoginProcessor() { @Override - protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException { + protected org.apache.commons.configuration.Configuration getApplicationConfiguration() { PropertiesConfiguration config = new PropertiesConfiguration(); config.setProperty("atlas.authentication.method", "kerberos"); config.setProperty("atlas.authentication.principal", "[email protected]");
