Repository: ambari Updated Branches: refs/heads/trunk bc465f1b1 -> 0e426b0d9
AMBARI-17203 - ATLAS conf dir needs to be present in all ATLAS hook deployed hosts Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e426b0d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e426b0d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e426b0d Branch: refs/heads/trunk Commit: 0e426b0d9e2b4034fc1df64fd5c2a1769a235e7d Parents: bc465f1 Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Tue Jun 14 04:54:23 2016 -0400 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Tue Jun 14 04:54:39 2016 -0400 ---------------------------------------------------------------------- .../FALCON/0.5.0.2.1/package/scripts/params_linux.py | 3 ++- .../0.5.0.2.1/package/scripts/setup_atlas_falcon.py | 8 ++++++++ .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 2 +- .../HIVE/0.12.0.2.0/package/scripts/params_linux.py | 9 ++------- .../0.12.0.2.0/package/scripts/setup_atlas_hive.py | 4 ++-- .../SQOOP/1.4.4.2.0/package/scripts/params_linux.py | 3 ++- .../1.4.4.2.0/package/scripts/setup_atlas_sqoop.py | 11 ++++++++--- .../STORM/0.9.1/package/scripts/params_linux.py | 2 ++ .../STORM/0.9.1/package/scripts/setup_atlas_storm.py | 8 ++++++++ .../stacks/HDP/2.5/services/FALCON/metainfo.xml | 3 +++ .../stacks/HDP/2.5/services/HIVE/metainfo.xml | 3 +++ .../stacks/HDP/2.5/services/SQOOP/metainfo.xml | 3 +++ .../stacks/HDP/2.5/services/STORM/metainfo.xml | 3 +++ .../stacks/HDP/2.5/services/stack_advisor.py | 14 +++++++++----- 14 files changed, 56 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 441f0da..86d76ff 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -138,10 +138,11 @@ atlas_plugin_package = "atlas-metadata*-hive-plugin" atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" if has_atlas: - atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file'] + atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook", "falcon", "*") + os.pathsep + atlas_props = default('/configurations/application-properties', {}) hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py index 67077c4..1dce515 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py @@ -18,7 +18,9 @@ limitations under the License. """ +from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.resources.packaging import Package +from resource_management.libraries.functions.format import format from ambari_commons import OSCheck import os @@ -31,3 +33,9 @@ def setup_atlas_falcon(): if not params.host_sys_prepped: Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) + + PropertiesFile(format('{falcon_conf_dir}/{atlas_conf_file}'), + properties = params.atlas_props, + owner = params.falcon_user, + group = params.user_group, + mode = 0644) http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml index c20523d..69de51d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml @@ -235,7 +235,7 @@ else fi export METASTORE_PORT={{hive_metastore_port}} -export HADOOP_CLASSPATH={{atlas_conf_dir}}:{{atlas_home_dir}}/hook/hive:${HADOOP_CLASSPATH} +export HADOOP_CLASSPATH={{atlas_home_dir}}/hook/hive:${HADOOP_CLASSPATH} </value> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index fea0635..a669ec7 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -515,15 +515,10 @@ atlas_plugin_package = "atlas-metadata*-hive-plugin" atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" if has_atlas: + atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' - # client.properties - atlas_client_props = {} - auth_enabled = config['configurations']['application-properties'].get( - 'atlas.http.authentication.enabled', False) - atlas_client_props['atlas.http.authentication.enabled'] = auth_enabled - if auth_enabled: - atlas_client_props['atlas.http.authentication.type'] = config['configurations']['application-properties'].get('atlas.http.authentication.type', 'simple') + atlas_props = default('/configurations/application-properties', {}) ######################################################## ########### WebHCat related params ##################### http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py index e78190f..d1bd8ea 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py @@ -34,8 +34,8 @@ def setup_atlas_hive(configuration_directory=None): Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, # FIXME HACK: install the package during RESTART/START when install_packages is not triggered. retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) - PropertiesFile(format('{configuration_directory}/client.properties'), - properties = params.atlas_client_props, + PropertiesFile(format('{configuration_directory}/{atlas_conf_file}'), + properties = params.atlas_props, owner = params.hive_user, group = params.user_group, mode = 0644) http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py index c154a16..b2a6802 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py @@ -132,7 +132,8 @@ atlas_plugin_package = "atlas-metadata*-hive-plugin" atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" if has_atlas: - atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file'] + atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format("{stack_root}/current/atlas-server") atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' + atlas_props = default('/configurations/application-properties', {}) job_data_publish_class = 'org.apache.atlas.sqoop.hook.SqoopHook' http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py index d18d820..76c1cda 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py @@ -18,8 +18,10 @@ limitations under the License. """ +from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.resources.packaging import Package from resource_management.core.resources.system import Link +from resource_management.libraries.functions.format import format from ambari_commons import OSCheck import os @@ -33,11 +35,14 @@ def setup_atlas_sqoop(): Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) + PropertiesFile(format('{sqoop_conf_dir}/{atlas_conf_file}'), + properties = params.atlas_props, + owner = params.sqoop_user, + group = params.user_group, + mode = 0644) + atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop") if os.path.exists(atlas_sqoop_hook_dir): - Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file), - to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file) - ) src_files = os.listdir(atlas_sqoop_hook_dir) for file_name in src_files: http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index eadbd4a..0c62399 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -214,8 +214,10 @@ atlas_plugin_package = "atlas-metadata*-hive-plugin" atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" if has_atlas: + atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else stack_root + '/current/atlas-server' atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' + atlas_props = default('/configurations/application-properties', {}) jar_jvm_opts = '-Datlas.conf=' + atlas_conf_dir # ranger host http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py index 6c3e91f..76ca1d3 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py @@ -18,8 +18,10 @@ limitations under the License. """ +from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.resources.packaging import Package from resource_management.core.resources.system import Link +from resource_management.libraries.functions.format import format from ambari_commons import OSCheck import os @@ -33,6 +35,12 @@ def setup_atlas_storm(): Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) + PropertiesFile(format('{conf_dir}/{atlas_conf_file}'), + properties = params.atlas_props, + owner = params.storm_user, + group = params.user_group, + mode = 0644) + atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm") if os.path.exists(atlas_storm_hook_dir): storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml index 1998131..93f4847 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml @@ -21,6 +21,9 @@ <service> <name>FALCON</name> <version>0.6.1.2.5</version> + <configuration-dependencies> + <config-type>application-properties</config-type> + </configuration-dependencies> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml index ac1f9e7..b44c063 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml @@ -233,6 +233,9 @@ </packages> </osSpecific> </osSpecifics> + <configuration-dependencies> + <config-type>application-properties</config-type> + </configuration-dependencies> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/stacks/HDP/2.5/services/SQOOP/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SQOOP/metainfo.xml index eb67d63..f4e2e95 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SQOOP/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SQOOP/metainfo.xml @@ -21,6 +21,9 @@ <service> <name>SQOOP</name> <version>1.4.6.2.5</version> + <configuration-dependencies> + <config-type>application-properties</config-type> + </configuration-dependencies> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml index 3faadc0..4f92cde 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml @@ -23,6 +23,9 @@ <name>STORM</name> <version>1.0.1.2.5</version> <extends>common-services/STORM/1.0.1</extends> + <configuration-dependencies> + <config-type>application-properties</config-type> + </configuration-dependencies> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/0e426b0d/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index e221415..614f0b1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -81,7 +81,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): "item": self.getErrorItem( "If KAFKA is not installed then the Kafka zookeeper quorum configuration must be specified.")}) - if application_properties['atlas.graph.storage.backend'] == 'hbase': + if application_properties['atlas.graph.storage.backend'] == 'hbase' and 'hbase-site' in services['configurations']: hbase_zookeeper_quorum = services['configurations']['hbase-site']['properties']['hbase.zookeeper.quorum'] if not application_properties['atlas.graph.storage.hostname']: @@ -272,7 +272,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putAtlasApplicationProperty('atlas.server.ids', server_ids) putAtlasApplicationProperty('atlas.server.address.id1', server_hosts) - if "LOGSEARCH" in servicesList: + if "LOGSEARCH" in servicesList and 'logsearch-solr-env' in services['configurations']: if 'logsearch_solr_znode' in services['configurations']['logsearch-solr-env']['properties']: logsearch_solr_znode = services['configurations']['logsearch-solr-env']['properties']['logsearch_solr_znode'] @@ -295,7 +295,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): else: putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', "") - if "KAFKA" in servicesList: + if "KAFKA" in servicesList and 'kafka-broker' in services['configurations']: kafka_hosts = self.getHostNamesWithComponent("KAFKA", "KAFKA_BROKER", services) @@ -321,8 +321,12 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putAtlasApplicationProperty('atlas.kafka.bootstrap.servers', "") putAtlasApplicationProperty('atlas.kafka.zookeeper.connect', "") - if "HBASE" in servicesList: - hbase_zookeeper_quorum = services['configurations']['hbase-site']['properties']['hbase.zookeeper.quorum'] + if "HBASE" in servicesList and 'hbase-site' in services['configurations']: + if 'hbase.zookeeper.quorum' in services['configurations']['hbase-site']['properties']: + hbase_zookeeper_quorum = services['configurations']['hbase-site']['properties']['hbase.zookeeper.quorum'] + else: + hbase_zookeeper_quorum = "" + putAtlasApplicationProperty('atlas.graph.storage.hostname', hbase_zookeeper_quorum) putAtlasApplicationProperty('atlas.audit.hbase.zookeeper.quorum', hbase_zookeeper_quorum) else:
