Repository: ambari Updated Branches: refs/heads/trunk 63a15872f -> d3429491a
AMBARI-22485 : Allow Ambari to support non-kerberos SASL mechanisms for Kafka (ydavis via mradhakrishnan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d3429491 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d3429491 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d3429491 Branch: refs/heads/trunk Commit: d3429491a231f921ca3960ac0363c52690fb9dfa Parents: 63a1587 Author: Madhuvanthi Radhakrishnan <[email protected]> Authored: Thu Nov 30 11:24:44 2017 -0800 Committer: Madhuvanthi Radhakrishnan <[email protected]> Committed: Thu Nov 30 11:24:44 2017 -0800 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../KAFKA/0.8.1/configuration/kafka-env.xml | 2 +- .../KAFKA/0.8.1/package/scripts/kafka.py | 4 +- .../KAFKA/0.8.1/package/scripts/params.py | 16 +++++-- .../0.8.1/package/scripts/setup_ranger_kafka.py | 8 ++-- .../KAFKA/0.8.1/package/scripts/upgrade.py | 3 +- .../package/templates/kafka_client_jaas.conf.j2 | 3 ++ .../0.8.1/package/templates/kafka_jaas.conf.j2 | 47 ++++++++++++++++++++ .../KAFKA/0.9.0/configuration/kafka-broker.xml | 2 +- .../HDP/2.0.6/properties/stack_features.json | 5 +++ 10 files changed, 77 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index f46b7cf..b811861 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -120,3 +120,4 @@ class StackFeature: RANGER_KMS_SSL = "ranger_kms_ssl" KAFKA_ACL_MIGRATION_SUPPORT = "kafka_acl_migration_support" ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support" + KAFKA_EXTENDED_SASL_SUPPORT = "kafka_extended_sasl_support" http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml index 7607c22..ea6c50c 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml @@ -94,7 +94,7 @@ export JAVA_HOME={{java64_home}} export PATH=$PATH:$JAVA_HOME/bin export PID_DIR={{kafka_pid_dir}} export LOG_DIR={{kafka_log_dir}} -{% if security_enabled %} +{% if kerberos_security_enabled or kafka_other_sasl_enabled %} export KAFKA_KERBEROS_PARAMS="-Djavax.security.auth.useSubjectCredsOnly=false {{kafka_kerberos_params}}" {% else %} export KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}} http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py index 3aa3473..3adea69 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py @@ -53,7 +53,7 @@ def kafka(upgrade_type=None): Logger.info(format("Kafka listeners: {listeners}")) kafka_server_config['listeners'] = listeners - if params.security_enabled and params.kafka_kerberos_enabled: + if params.kerberos_security_enabled and params.kafka_kerberos_enabled: Logger.info("Kafka kerberos security is enabled.") kafka_server_config['advertised.listeners'] = listeners Logger.info(format("Kafka advertised listeners: {listeners}")) @@ -115,7 +115,7 @@ def kafka(upgrade_type=None): content=InlineTemplate(params.log4j_props) ) - if params.security_enabled and params.kafka_kerberos_enabled: + if (params.kerberos_security_enabled and params.kafka_kerberos_enabled) or params.kafka_other_sasl_enabled: if params.kafka_jaas_conf_template: File(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index a62265b..46fdfba 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -157,13 +157,19 @@ if has_metric_collector: pass # Security-related params -security_enabled = config['configurations']['cluster-env']['security_enabled'] +kerberos_security_enabled = config['configurations']['cluster-env']['security_enabled'] + kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and ((config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "PLAINTEXTSASL") or (config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "SASL_PLAINTEXT"))) +kafka_other_sasl_enabled = not kerberos_security_enabled and check_stack_feature(StackFeature.KAFKA_LISTENERS, stack_version_formatted) and \ + check_stack_feature(StackFeature.KAFKA_EXTENDED_SASL_SUPPORT, stack_version_formatted) and \ + (("SASL_PLAINTEXT" in config['configurations']['kafka-broker']['listeners']) or + ("PLAINTEXTSASL" in config['configurations']['kafka-broker']['listeners']) or + ("SASL_SSL" in config['configurations']['kafka-broker']['listeners'])) -if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \ +if kerberos_security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \ and check_stack_feature(StackFeature.KAFKA_KERBEROS, stack_version_formatted): _hostname_lowercase = config['hostname'].lower() _kafka_principal_name = config['configurations']['kafka-env']['kafka_principal_name'] @@ -171,6 +177,8 @@ if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab'] kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name) kafka_kerberos_params = "-Djava.security.auth.login.config="+ conf_dir +"/kafka_jaas.conf" +elif kafka_other_sasl_enabled: + kafka_kerberos_params = "-Djava.security.auth.login.config="+ conf_dir +"/kafka_jaas.conf" else: kafka_kerberos_params = '' kafka_jaas_principal = None @@ -266,7 +274,7 @@ if enable_ranger_kafka and is_supported_kafka_ranger: if len(custom_ranger_service_config) > 0: ranger_plugin_config.update(custom_ranger_service_config) - if stack_supports_ranger_kerberos and security_enabled: + if stack_supports_ranger_kerberos and kerberos_security_enabled: ranger_plugin_config['policy.download.auth.users'] = kafka_user ranger_plugin_config['tag.download.auth.users'] = kafka_user ranger_plugin_config['ambari.service.check.user'] = policy_user @@ -327,7 +335,7 @@ HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled = security_enabled, + security_enabled = kerberos_security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, hadoop_bin_dir = hadoop_bin_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py index e9719aa..9aa09df 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py @@ -66,10 +66,10 @@ def setup_ranger_kafka(): credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble, - is_security_enabled = params.security_enabled, + is_security_enabled = params.kerberos_security_enabled, is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos, - component_user_principal=params.kafka_jaas_principal if params.security_enabled else None, - component_user_keytab=params.kafka_keytab_path if params.security_enabled else None) + component_user_principal=params.kafka_jaas_principal if params.kerberos_security_enabled else None, + component_user_keytab=params.kafka_keytab_path if params.kerberos_security_enabled else None) if params.enable_ranger_kafka: Execute(('cp', '--remove-destination', params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target), @@ -81,7 +81,7 @@ def setup_ranger_kafka(): group = params.user_group, mode = 0755 ) - if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.security_enabled: + if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.kerberos_security_enabled: Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions") setup_core_site_for_required_plugins(component_user=params.kafka_user,component_group=params.user_group,create_core_site_path = params.conf_dir, config = params.config) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py index e79a8ad..b327e54 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py @@ -1,4 +1,3 @@ - #!/usr/bin/env python """ Licensed to the Apache Software Foundation (ASF) under one @@ -43,7 +42,7 @@ def run_migration(env, upgrade_type): if params.upgrade_direction is None: raise Fail('Parameter "upgrade_direction" is missing.') - if not params.security_enabled: + if not params.kerberos_security_enabled: Logger.info("Skip running the Kafka ACL migration script since cluster security is not enabled.") return http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 index 7f81d85..873d030 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 @@ -15,12 +15,15 @@ # See the License for the specific language governing permissions and # limitations under the License. #} + +{% if kerberos_security_enabled %} KafkaClient { com.sun.security.auth.module.Krb5LoginModule required useTicketCache=true renewTicket=true serviceName="{{kafka_bare_jaas_principal}}"; }; +{% endif %} Client { com.sun.security.auth.module.Krb5LoginModule required useTicketCache=true http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 index 1d9e61d..68b5e44 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 @@ -15,6 +15,51 @@ # See the License for the specific language governing permissions and # limitations under the License. #} + +/** +* Example of SASL/PLAIN Configuration +* +* KafkaServer { +* org.apache.kafka.common.security.plain.PlainLoginModule required +* username="admin" +* password="admin-secret" +* user_admin="admin-secret" +* user_alice="alice-secret"; +* }; +* +* Example of SASL/SCRAM +* +* KafkaServer { +* org.apache.kafka.common.security.scram.ScramLoginModule required +* username="admin" +* password="admin-secret" +* }; +* +* Example of Enabling multiple SASL mechanisms in a broker: +* +* KafkaServer { +* +* com.sun.security.auth.module.Krb5LoginModule required +* useKeyTab=true +* storeKey=true +* keyTab="/etc/security/keytabs/kafka_server.keytab" +* principal="kafka/[email protected]"; +* +* org.apache.kafka.common.security.plain.PlainLoginModule required +* username="admin" +* password="admin-secret" +* user_admin="admin-secret" +* user_alice="alice-secret"; +* +* org.apache.kafka.common.security.scram.ScramLoginModule required +* username="scram-admin" +* password="scram-admin-secret"; +* }; +* +**/ + +{% if kerberos_security_enabled %} + KafkaServer { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true @@ -50,3 +95,5 @@ com.sun.security.jgss.krb5.initiate { serviceName="{{kafka_bare_jaas_principal}}" principal="{{kafka_jaas_principal}}"; }; + +{% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml index 4cd2b0d..fbc1c64 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml @@ -17,7 +17,7 @@ * limitations under the License. */ --> -<configuration xmlns:xi="http://www.w3.org/2001/XInclude"; supports_final="true"> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> <property> <name>listeners</name> <value>PLAINTEXT://localhost:6667</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index f79cfe0..2109a5d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -447,6 +447,11 @@ "name": "sam_storage_core_in_registry", "description": "Storage core module moved to registry", "min_version": "2.6.3.0" + }, + { + "name": "kafka_extended_sasl_support", + "description": "Support SASL PLAIN and GSSAPI", + "min_version": "2.6.5.0" } ] }
