Repository: incubator-atlas Updated Branches: refs/heads/master 3f47408e7 -> 038b6b315
ATLAS-1030 Atlas hooks should use properties from atlas-application.properties, instead of component's configuration (mneethiraj via sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/038b6b31 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/038b6b31 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/038b6b31 Branch: refs/heads/master Commit: 038b6b315d8353e75fe3fd663ac034e025e541d2 Parents: 3f47408 Author: Suma Shivaprasad <[email protected]> Authored: Mon Jul 18 17:31:18 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Mon Jul 18 17:32:54 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/atlas/falcon/hook/FalconHookIT.java | 2 +- addons/falcon-bridge/src/test/resources/hive-site.xml | 5 ----- .../org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java | 10 +++++----- .../main/java/org/apache/atlas/hive/hook/HiveHook.java | 2 +- .../test/java/org/apache/atlas/hive/hook/HiveHookIT.java | 8 ++++---- addons/hive-bridge/src/test/resources/hive-site.xml | 5 ----- .../main/java/org/apache/atlas/sqoop/hook/SqoopHook.java | 3 +-- .../java/org/apache/atlas/sqoop/hook/SqoopHookIT.java | 2 +- addons/sqoop-bridge/src/test/resources/hive-site.xml | 5 ----- .../java/org/apache/atlas/storm/hook/StormAtlasHook.java | 6 +----- release-log.txt | 1 + 11 files changed, 15 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java index 0139bf5..f7184f9 100644 --- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java +++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java @@ -89,7 +89,7 @@ public class FalconHookIT { return; } - HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient); + HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient); hiveMetaStoreBridge.registerHiveDataModel(); FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/falcon-bridge/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/resources/hive-site.xml b/addons/falcon-bridge/src/test/resources/hive-site.xml index b106903..f058c2e 100644 --- a/addons/falcon-bridge/src/test/resources/hive-site.xml +++ b/addons/falcon-bridge/src/test/resources/hive-site.xml @@ -42,11 +42,6 @@ </property> <property> - <name>atlas.cluster.name</name> - <value>test</value> - </property> - - <property> <name>fs.pfile.impl</name> <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value> </property> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index cd0e964..9a5e279 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -99,16 +99,16 @@ public class HiveMetaStoreBridge { * Construct a HiveMetaStoreBridge. * @param hiveConf {@link HiveConf} for Hive component in the cluster */ - public HiveMetaStoreBridge(HiveConf hiveConf) throws Exception { - this(hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), null); + public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf) throws Exception { + this(atlasProperties, hiveConf, null); } /** * Construct a HiveMetaStoreBridge. * @param hiveConf {@link HiveConf} for Hive component in the cluster */ - public HiveMetaStoreBridge(HiveConf hiveConf, AtlasClient atlasClient) throws Exception { - this(hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClient); + public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf, AtlasClient atlasClient) throws Exception { + this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClient); } AtlasClient getAtlasClient() { @@ -660,7 +660,7 @@ public class HiveMetaStoreBridge { failOnError = true; } - HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient); + HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient); hiveMetaStoreBridge.registerHiveDataModel(); hiveMetaStoreBridge.importHiveMetadata(failOnError); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index 99009ba..0e86333 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -200,7 +200,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { LOG.info("Entered Atlas hook for hook type {} operation {}", event.getHookType(), event.getOperation()); - HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(hiveConf); + HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(atlasProperties, hiveConf); switch (event.getOperation()) { case CREATEDATABASE: http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 734b606..a15aa49 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -87,7 +87,7 @@ public class HiveHookIT { private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HiveHookIT.class); private static final String DGI_URL = "http://localhost:21000/";; - private static final String CLUSTER_NAME = "test"; + private static final String CLUSTER_NAME = "primary"; public static final String DEFAULT_DB = "default"; private static final String PART_FILE = "2015-01-01"; @@ -115,7 +115,7 @@ public class HiveHookIT { Configuration configuration = ApplicationProperties.get(); atlasClient = new AtlasClient(configuration.getString(HiveMetaStoreBridge.ATLAS_ENDPOINT, DGI_URL)); - hiveMetaStoreBridge = new HiveMetaStoreBridge(conf, atlasClient); + hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient); hiveMetaStoreBridge.registerHiveDataModel(); } @@ -1624,7 +1624,7 @@ public class HiveHookIT { verifyEntityProperties(entityType, entityName, expectedProps, false); if (entityType != Entity.Type.DATABASE) { - //Database unset properties doesnt work strangely - alter database %s unset DBPROPERTIES doesnt work + //Database unset properties doesnt work - alter database %s unset DBPROPERTIES doesnt work //Unset all the props StringBuilder sb = new StringBuilder("'"); query = String.format(fmtQuery, entityName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\'')); @@ -1928,7 +1928,7 @@ public class HiveHookIT { fail("Assertions failed. Failing after waiting for timeout " + timeout + " msecs", e); } LOG.debug("Waiting up to " + (mustEnd - System.currentTimeMillis()) + " msec as assertion failed", e); - Thread.sleep(400); + Thread.sleep(5000); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/hive-site.xml b/addons/hive-bridge/src/test/resources/hive-site.xml index 3fe57df..876f41d 100644 --- a/addons/hive-bridge/src/test/resources/hive-site.xml +++ b/addons/hive-bridge/src/test/resources/hive-site.xml @@ -57,11 +57,6 @@ </property> <property> - <name>atlas.cluster.name</name> - <value>test</value> - </property> - - <property> <name>fs.pfile.impl</name> <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value> </property> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java index c412658..e43fc86 100644 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java +++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java @@ -163,8 +163,7 @@ public class SqoopHook extends SqoopJobDataPublisher { @Override public void publish(SqoopJobDataPublisher.Data data) throws Exception { Configuration atlasProperties = ApplicationProperties.get(); - org.apache.hadoop.conf.Configuration sqoopConf = new org.apache.hadoop.conf.Configuration(); - String clusterName = sqoopConf.get(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); + String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); Referenceable dbStoreRef = createDBStoreInstance(data); Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java index 72d9201..e965c7d 100644 --- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java +++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java @@ -54,7 +54,7 @@ public class SqoopHookIT { private void registerDataModels(AtlasClient client) throws Exception { // Make sure hive model exists - HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient); + HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient); hiveMetaStoreBridge.registerHiveDataModel(); SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/sqoop-bridge/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/test/resources/hive-site.xml b/addons/sqoop-bridge/src/test/resources/hive-site.xml index b106903..f058c2e 100644 --- a/addons/sqoop-bridge/src/test/resources/hive-site.xml +++ b/addons/sqoop-bridge/src/test/resources/hive-site.xml @@ -42,11 +42,6 @@ </property> <property> - <name>atlas.cluster.name</name> - <value>test</value> - </property> - - <property> <name>fs.pfile.impl</name> <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value> </property> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java index 5bd5397..c4b4976 100644 --- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java +++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java @@ -370,10 +370,6 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook { } private String getClusterName(Map stormConf) { - String clusterName = AtlasConstants.DEFAULT_CLUSTER_NAME; - if (stormConf.containsKey(AtlasConstants.CLUSTER_NAME_KEY)) { - clusterName = (String)stormConf.get(AtlasConstants.CLUSTER_NAME_KEY); - } - return clusterName; + return atlasProperties.getString(AtlasConstants.CLUSTER_NAME_KEY, AtlasConstants.DEFAULT_CLUSTER_NAME); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index f6128dc..eb7d458 100644 --- a/release-log.txt +++ b/release-log.txt @@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES: ALL CHANGES: +ATLAS-1027 Atlas hooks should use properties from atlas-application.properties, instead of component's configuration (mneethiraj via sumasai) ATLAS-1030 Add instrumentation to measure performance: REST API (mneethiraj via sumasai) ATLAS-996 DSL queries with comparsions of many primitive types fail (jnhagelb via shwethags) ATLAS-971 UI not displaying results for this query - Eg: "hive_table as t where qualifiedName = 'default.input@cl1' select t" (kevalbhatt18 via shwethags)
