Repository: ambari Updated Branches: refs/heads/branch-2.6 51bd023a2 -> 2b49f4582
AMBARI-22485 : Allow Ambari to support non-kerberos SASL mechanisms for Kafka (ydavis via mradhakrishnan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2b49f458 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2b49f458 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2b49f458 Branch: refs/heads/branch-2.6 Commit: 2b49f4582c1e12b62baa8a0f4d98650867ff8fe0 Parents: 51bd023 Author: Madhuvanthi Radhakrishnan <[email protected]> Authored: Thu Nov 30 11:24:44 2017 -0800 Committer: Madhuvanthi Radhakrishnan <[email protected]> Committed: Thu Nov 30 11:26:52 2017 -0800 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../KAFKA/0.8.1/configuration/kafka-env.xml | 2 +- .../KAFKA/0.8.1/package/scripts/kafka.py | 4 +- .../KAFKA/0.8.1/package/scripts/params.py | 16 +++++-- .../0.8.1/package/scripts/setup_ranger_kafka.py | 8 ++-- .../KAFKA/0.8.1/package/scripts/upgrade.py | 3 +- .../package/templates/kafka_client_jaas.conf.j2 | 3 ++ .../0.8.1/package/templates/kafka_jaas.conf.j2 | 47 ++++++++++++++++++++ .../KAFKA/0.9.0/configuration/kafka-broker.xml | 2 +- .../HDP/2.0.6/properties/stack_features.json | 5 +++ 10 files changed, 77 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index f46b7cf..b811861 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -120,3 +120,4 @@ class StackFeature: RANGER_KMS_SSL = "ranger_kms_ssl" KAFKA_ACL_MIGRATION_SUPPORT = "kafka_acl_migration_support" ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support" + KAFKA_EXTENDED_SASL_SUPPORT = "kafka_extended_sasl_support" http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml index ad81d66..a68feb2 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml @@ -88,7 +88,7 @@ export JAVA_HOME={{java64_home}} export PATH=$PATH:$JAVA_HOME/bin export PID_DIR={{kafka_pid_dir}} export LOG_DIR={{kafka_log_dir}} -{% if security_enabled %} +{% if kerberos_security_enabled or kafka_other_sasl_enabled %} export KAFKA_KERBEROS_PARAMS="-Djavax.security.auth.useSubjectCredsOnly=false {{kafka_kerberos_params}}" {% else %} export KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}} http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py index 85c3347..f03bccb 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py @@ -52,7 +52,7 @@ def kafka(upgrade_type=None): listeners = kafka_server_config['listeners'].replace("localhost", params.hostname) Logger.info(format("Kafka listeners: {listeners}")) - if params.security_enabled and params.kafka_kerberos_enabled: + if params.kerberos_security_enabled and params.kafka_kerberos_enabled: Logger.info("Kafka kerberos security is enabled.") if "SASL" not in listeners: listeners = listeners.replace("PLAINTEXT", "PLAINTEXTSASL") @@ -120,7 +120,7 @@ def kafka(upgrade_type=None): content=InlineTemplate(params.log4j_props) ) - if params.security_enabled and params.kafka_kerberos_enabled: + if (params.kerberos_security_enabled and params.kafka_kerberos_enabled) or params.kafka_other_sasl_enabled: if params.kafka_jaas_conf_template: File(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 539b469..c6b36a1 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -159,13 +159,19 @@ if has_metric_collector: pass # Security-related params -security_enabled = config['configurations']['cluster-env']['security_enabled'] +kerberos_security_enabled = config['configurations']['cluster-env']['security_enabled'] + kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and ((config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "PLAINTEXTSASL") or (config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "SASL_PLAINTEXT"))) +kafka_other_sasl_enabled = not kerberos_security_enabled and check_stack_feature(StackFeature.KAFKA_LISTENERS, stack_version_formatted) and \ + check_stack_feature(StackFeature.KAFKA_EXTENDED_SASL_SUPPORT, stack_version_formatted) and \ + (("SASL_PLAINTEXT" in config['configurations']['kafka-broker']['listeners']) or + ("PLAINTEXTSASL" in config['configurations']['kafka-broker']['listeners']) or + ("SASL_SSL" in config['configurations']['kafka-broker']['listeners'])) -if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \ +if kerberos_security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \ and check_stack_feature(StackFeature.KAFKA_KERBEROS, stack_version_formatted): _hostname_lowercase = config['hostname'].lower() _kafka_principal_name = config['configurations']['kafka-env']['kafka_principal_name'] @@ -173,6 +179,8 @@ if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab'] kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name) kafka_kerberos_params = "-Djava.security.auth.login.config="+ conf_dir +"/kafka_jaas.conf" +elif kafka_other_sasl_enabled: + kafka_kerberos_params = "-Djava.security.auth.login.config="+ conf_dir +"/kafka_jaas.conf" else: kafka_kerberos_params = '' kafka_jaas_principal = None @@ -268,7 +276,7 @@ if enable_ranger_kafka and is_supported_kafka_ranger: if len(custom_ranger_service_config) > 0: ranger_plugin_config.update(custom_ranger_service_config) - if stack_supports_ranger_kerberos and security_enabled: + if stack_supports_ranger_kerberos and kerberos_security_enabled: ranger_plugin_config['policy.download.auth.users'] = kafka_user ranger_plugin_config['tag.download.auth.users'] = kafka_user ranger_plugin_config['ambari.service.check.user'] = policy_user @@ -329,7 +337,7 @@ HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled = security_enabled, + security_enabled = kerberos_security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, hadoop_bin_dir = hadoop_bin_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py index e9719aa..9aa09df 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py @@ -66,10 +66,10 @@ def setup_ranger_kafka(): credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble, - is_security_enabled = params.security_enabled, + is_security_enabled = params.kerberos_security_enabled, is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos, - component_user_principal=params.kafka_jaas_principal if params.security_enabled else None, - component_user_keytab=params.kafka_keytab_path if params.security_enabled else None) + component_user_principal=params.kafka_jaas_principal if params.kerberos_security_enabled else None, + component_user_keytab=params.kafka_keytab_path if params.kerberos_security_enabled else None) if params.enable_ranger_kafka: Execute(('cp', '--remove-destination', params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target), @@ -81,7 +81,7 @@ def setup_ranger_kafka(): group = params.user_group, mode = 0755 ) - if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.security_enabled: + if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.kerberos_security_enabled: Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions") setup_core_site_for_required_plugins(component_user=params.kafka_user,component_group=params.user_group,create_core_site_path = params.conf_dir, config = params.config) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py index e119211..27af7f7 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py @@ -1,4 +1,3 @@ - #!/usr/bin/env python """ Licensed to the Apache Software Foundation (ASF) under one @@ -43,7 +42,7 @@ def run_migration(env, upgrade_type): if params.upgrade_direction is None: raise Fail('Parameter "upgrade_direction" is missing.') - if not params.security_enabled: + if not params.kerberos_security_enabled: Logger.info("Skip running the Kafka ACL migration script since cluster security is not enabled.") return http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 index 7f81d85..873d030 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 @@ -15,12 +15,15 @@ # See the License for the specific language governing permissions and # limitations under the License. #} + +{% if kerberos_security_enabled %} KafkaClient { com.sun.security.auth.module.Krb5LoginModule required useTicketCache=true renewTicket=true serviceName="{{kafka_bare_jaas_principal}}"; }; +{% endif %} Client { com.sun.security.auth.module.Krb5LoginModule required useTicketCache=true http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 index 1d9e61d..68b5e44 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 @@ -15,6 +15,51 @@ # See the License for the specific language governing permissions and # limitations under the License. #} + +/** +* Example of SASL/PLAIN Configuration +* +* KafkaServer { +* org.apache.kafka.common.security.plain.PlainLoginModule required +* username="admin" +* password="admin-secret" +* user_admin="admin-secret" +* user_alice="alice-secret"; +* }; +* +* Example of SASL/SCRAM +* +* KafkaServer { +* org.apache.kafka.common.security.scram.ScramLoginModule required +* username="admin" +* password="admin-secret" +* }; +* +* Example of Enabling multiple SASL mechanisms in a broker: +* +* KafkaServer { +* +* com.sun.security.auth.module.Krb5LoginModule required +* useKeyTab=true +* storeKey=true +* keyTab="/etc/security/keytabs/kafka_server.keytab" +* principal="kafka/[email protected]"; +* +* org.apache.kafka.common.security.plain.PlainLoginModule required +* username="admin" +* password="admin-secret" +* user_admin="admin-secret" +* user_alice="alice-secret"; +* +* org.apache.kafka.common.security.scram.ScramLoginModule required +* username="scram-admin" +* password="scram-admin-secret"; +* }; +* +**/ + +{% if kerberos_security_enabled %} + KafkaServer { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true @@ -50,3 +95,5 @@ com.sun.security.jgss.krb5.initiate { serviceName="{{kafka_bare_jaas_principal}}" principal="{{kafka_jaas_principal}}"; }; + +{% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml index f15dcad..3660813 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml @@ -17,7 +17,7 @@ * limitations under the License. */ --> -<configuration xmlns:xi="http://www.w3.org/2001/XInclude"; supports_final="true"> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> <property> <name>listeners</name> <value>PLAINTEXT://localhost:6667</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/2b49f458/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index f79cfe0..2109a5d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -447,6 +447,11 @@ "name": "sam_storage_core_in_registry", "description": "Storage core module moved to registry", "min_version": "2.6.3.0" + }, + { + "name": "kafka_extended_sasl_support", + "description": "Support SASL PLAIN and GSSAPI", + "min_version": "2.6.5.0" } ] }
