Repository: ambari Updated Branches: refs/heads/trunk 601cbefe1 -> 18182c54e
AMBARI-17780. Atlas hooks for Hive, Falcon, Storm missing atlas.jaas.KafkaClient.option.principal & atlas.jaas.KafkaClient.option.keytab (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18182c54 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18182c54 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18182c54 Branch: refs/heads/trunk Commit: 18182c54e13b62e2cb384f8cfa38bd5196dacff8 Parents: 601cbef Author: Alejandro Fernandez <[email protected]> Authored: Tue Jul 19 11:41:09 2016 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Tue Jul 19 11:41:09 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/setup_atlas_hook.py | 4 +- .../BlueprintConfigurationProcessor.java | 4 +- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 1 + .../FALCON/0.5.0.2.1/package/scripts/params.py | 3 + .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../1.4.4.2.0/package/scripts/params_linux.py | 1 + .../STORM/0.9.1/package/scripts/params_linux.py | 1 + .../common-services/STORM/1.0.1/kerberos.json | 9 +++ .../stacks/HDP/2.3/services/stack_advisor.py | 1 + .../HDP/2.5/services/FALCON/kerberos.json | 77 ++++++++++++++++++++ .../stacks/HDP/2.5/services/HIVE/kerberos.json | 9 +++ 11 files changed, 106 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py index 947d441..7cbdc91 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py @@ -82,9 +82,7 @@ SHARED_ATLAS_HOOK_CONFIGS = set( "atlas.jaas.KafkaClient.loginModuleControlFlag", "atlas.jaas.KafkaClient.option.useKeyTab", "atlas.jaas.KafkaClient.option.storeKey", - "atlas.jaas.KafkaClient.option.serviceName", - "atlas.jaas.KafkaClient.option.keyTab", - "atlas.jaas.KafkaClient.option.principal"] + "atlas.jaas.KafkaClient.option.serviceName"] ) def has_atlas_in_cluster(): http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 939e232..36cea56 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2454,7 +2454,7 @@ public class BlueprintConfigurationProcessor { } }); - //todo: john - this property should be moved to atlas configuration + // TODO AMBARI-17782, remove this property from hive-site only in HDP 2.5 and higher. hiveSiteNonTopologyMap.put("atlas.cluster.name", new NonTopologyUpdater() { @Override public String updateForClusterCreate(String propertyName, @@ -2490,7 +2490,7 @@ public class BlueprintConfigurationProcessor { } }); - //todo: john - this property should be removed + // TODO AMBARI-17782, remove this property only from HDP 2.5 and higher. hiveSiteMap.put("atlas.rest.address", new SingleHostTopologyUpdater("ATLAS_SERVER") { @Override public String updateForClusterCreate(String propertyName, http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 5a3c723..3425ef2 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -72,6 +72,7 @@ def configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled): config = Script.get_config() stack_root = Script.get_stack_root() +# Needed since this is an Atlas Hook service. cluster_name = config['clusterName'] java_version = expect("/hostLevelParams/java_version", int) http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py index f2d01fc..2942992 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py @@ -26,3 +26,6 @@ else: from params_linux import * host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) + +# Needed since this is an Atlas Hook service. +cluster_name = config['clusterName'] http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 6f8d588..8f95bf3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -56,6 +56,7 @@ stack_name_uppercase = stack_name.upper() agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) +# Needed since this is an Atlas Hook service. cluster_name = config['clusterName'] # node hostname http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py index 5bca720..df418f0 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py @@ -41,6 +41,7 @@ component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, config = Script.get_config() stack_root = Script.get_stack_root() +# Needed since this is an Atlas Hook service. cluster_name = config['clusterName'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 9c6f384..22d6f1b 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -46,6 +46,7 @@ tmp_dir = Script.get_tmp_dir() stack_root = status_params.stack_root sudo = AMBARI_SUDO_BINARY +# Needed since this is an Atlas Hook service. cluster_name = config['clusterName'] stack_name = status_params.stack_name http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/common-services/STORM/1.0.1/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/kerberos.json b/ambari-server/src/main/resources/common-services/STORM/1.0.1/kerberos.json index 5a9f24c..8c0b517 100644 --- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/kerberos.json +++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/kerberos.json @@ -28,6 +28,15 @@ }, "configuration": "storm-env/storm_keytab" } + }, + { + "name": "/STORM/storm_components", + "principal": { + "configuration": "storm-atlas-application.properties/atlas.jaas.KafkaClient.option.principal" + }, + "keytab": { + "configuration": "storm-atlas-application.properties/atlas.jaas.KafkaClient.option.keyTab" + } } ], "configurations": [ http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 6081a04..e1bedb7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -257,6 +257,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): hooks_value = " " putHiveSiteProperty(hooks_property, hooks_value) + # This is no longer used in HDP 2.5, but still needed in HDP 2.3 and 2.4 atlas_server_host_info = self.getHostWithComponent("ATLAS", "ATLAS_SERVER", services, hosts) if include_atlas and atlas_server_host_info: atlas_rest_host = atlas_server_host_info['Hosts']['host_name'] http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/kerberos.json new file mode 100644 index 0000000..0c4507d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/kerberos.json @@ -0,0 +1,77 @@ +{ + "services": [ + { + "name": "FALCON", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "configurations": [ + { + "falcon-startup.properties": { + "*.falcon.authentication.type": "kerberos", + "*.falcon.http.authentication.type": "kerberos", + "*.dfs.namenode.kerberos.principal": "nn/_HOST@${realm}" + } + } + ], + "auth_to_local_properties" : [ + "falcon-startup.properties/*.falcon.http.authentication.kerberos.name.rules|new_lines_escaped" + ], + "components": [ + { + "name": "FALCON_SERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "falcon_server", + "principal": { + "value": "falcon/_HOST@${realm}", + "type" : "service", + "configuration": "falcon-startup.properties/*.falcon.service.authentication.kerberos.principal", + "local_username" : "${falcon-env/falcon_user}" + }, + "keytab": { + "file": "${keytab_dir}/falcon.service.keytab", + "owner": { + "name": "${falcon-env/falcon_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "falcon-startup.properties/*.falcon.service.authentication.kerberos.keytab" + } + }, + { + "name": "/FALCON/FALCON_SERVER/falcon_server", + "principal": { + "configuration": "falcon-atlas-application.properties/atlas.jaas.KafkaClient.option.principal" + }, + "keytab": { + "configuration": "falcon-atlas-application.properties/atlas.jaas.KafkaClient.option.keyTab" + } + }, + { + "name": "/spnego", + "principal": { + "value": "HTTP/_HOST@${realm}", + "configuration": "falcon-startup.properties/*.falcon.http.authentication.kerberos.principal" + }, + "keytab": { + "configuration": "falcon-startup.properties/*.falcon.http.authentication.kerberos.keytab" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/18182c54/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json index b5ec1c9..f9a0caf 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json @@ -79,6 +79,15 @@ } }, { + "name": "/HIVE/HIVE_SERVER/hive_server_hive", + "principal": { + "configuration": "hive-atlas-application.properties/atlas.jaas.KafkaClient.option.principal" + }, + "keytab": { + "configuration": "hive-atlas-application.properties/atlas.jaas.KafkaClient.option.keyTab" + } + }, + { "name": "/spnego", "principal": { "configuration": "hive-site/hive.server2.authentication.spnego.principal"
