Ottomata has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/394438 )
Change subject: Improvements for Kafka + SSL
......................................................................
Improvements for Kafka + SSL
- Reduce default authorizer log level, DEBUG is way to much!
- Assume that all kafka brokers in this cluster use the same certificate and key
- Certificate should be subjectless with CN=kafka_${cluster_name}_broker
for easier DN specification
- default allow_everyone_if_no_acl_found to true; we want to restrict certain
topics with ACLs, not all by default.
- Auto grant --cluster permissions to User:CN=kafka_${cluster_name}_broker
Bug: T167304
Change-Id: I97089ca833d74717c07aab008277a867651a71b1
---
M hieradata/role/common/kafka/jumbo/broker.yaml
M hieradata/role/common/kafka/simple/broker.yaml
M modules/confluent/manifests/kafka/broker.pp
M modules/confluent/templates/kafka/log4j.properties.erb
M modules/confluent/templates/kafka/server.properties.erb
M modules/profile/manifests/kafka/broker.pp
6 files changed, 70 insertions(+), 31 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/hieradata/role/common/kafka/jumbo/broker.yaml
b/hieradata/role/common/kafka/jumbo/broker.yaml
index 768635f..89264ae 100644
--- a/hieradata/role/common/kafka/jumbo/broker.yaml
+++ b/hieradata/role/common/kafka/jumbo/broker.yaml
@@ -2,15 +2,15 @@
standard::has_ganglia: false
profile::kafka::broker::kafka_cluster_name: jumbo
-# Enable basic ACL handling via Zookeeper stored rules
-# More info https://phabricator.wikimedia.org/T167304#3478277
-profile::kafka::broker::auth_acls_enabled: true
-
# Enable SSL/TLS for Kafka. This requires
# that keystore and truststore files, and
# profile::kafka::broker::ssl_password, are committed in
# the expected location in ops/puppet/private.
-profile::kafka::broker::ssl_enabled: false
+profile::kafka::broker::ssl_enabled: true
+
+# Enable basic ACL handling via Zookeeper stored rules
+# More info https://phabricator.wikimedia.org/T167304#3478277
+profile::kafka::broker::auth_acls_enabled: true
# Enable Monitoring (via Prometheus) and icinga alerts
profile::kafka::broker::monitoring_enabled: true
diff --git a/hieradata/role/common/kafka/simple/broker.yaml
b/hieradata/role/common/kafka/simple/broker.yaml
index 5e5c531..4159621 100644
--- a/hieradata/role/common/kafka/simple/broker.yaml
+++ b/hieradata/role/common/kafka/simple/broker.yaml
@@ -4,8 +4,6 @@
profile::kafka::broker::monitoring_enabled: false
profile::kafka::broker::log_dirs: [/srv/kafka/data]
profile::kafka::broker::plaintext: true
-profile::kafka::broker::tls_secrets_path: false
-profile::kafka::broker::tls_key_password: false
profile::kafka::broker::auto_leader_rebalance_enable: true
profile::kafka::broker::nofiles_ulimit: 8192
diff --git a/modules/confluent/manifests/kafka/broker.pp
b/modules/confluent/manifests/kafka/broker.pp
index f617bb4..d4243fd 100644
--- a/modules/confluent/manifests/kafka/broker.pp
+++ b/modules/confluent/manifests/kafka/broker.pp
@@ -206,11 +206,22 @@
# The maximum message size allowed.
# Default: 1048576
#
+# [*allow_everyone_if_no_acl_found*]
+# If this value is on true, only the topics on which are ACLs are set are
secured.
+# Default: true
+#
+# [*super_users*]
+# List of super user CNs. If configuring SSL, this should at least include
the cluster's SSL
+# principal so the cluster can operate.
+#
# [*authorizer_class_name*]
# Sets up the ACL authorization provider specified
# as parameter. It also set up a more verbose log4j logging related
# to ACL authorization events.
# Default: undef
+#
+# [*authorizer_log_level*]
+# Default: INFO
#
class confluent::kafka::broker(
$enabled = true,
@@ -286,7 +297,10 @@
$message_max_bytes = 1048576,
+ $allow_everyone_if_no_acl_found = true,
+ $super_users = undef,
$authorizer_class_name = undef,
+ $authorizer_log_level = 'INFO',
) {
# confluent::kafka::common installs the kafka package
# and a handy wrapper script.
diff --git a/modules/confluent/templates/kafka/log4j.properties.erb
b/modules/confluent/templates/kafka/log4j.properties.erb
index 563309c..3fdfa13 100644
--- a/modules/confluent/templates/kafka/log4j.properties.erb
+++ b/modules/confluent/templates/kafka/log4j.properties.erb
@@ -75,11 +75,8 @@
log4j.additivity.state.change.logger=false
<%if @authorizer_class_name -%>
-# authorizer.class.name set in server.properties, therefore a more verbose
-# log accounting is needed.
-log4j.logger.kafka.authorizer.logger=DEBUG, authorizerAppender
-<% else -%>
-#Change this to debug to get the actual audit log for authorizer.
-log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender
+# Set this to DEBUG to get actual audit log for authorizer.
+# This will make logs very verbose!
+log4j.logger.kafka.authorizer.logger=<%= @authorizer_log_level %>,
authorizerAppender
<% end -%>
log4j.additivity.kafka.authorizer.logger=false
diff --git a/modules/confluent/templates/kafka/server.properties.erb
b/modules/confluent/templates/kafka/server.properties.erb
index a350618..c484d7c 100644
--- a/modules/confluent/templates/kafka/server.properties.erb
+++ b/modules/confluent/templates/kafka/server.properties.erb
@@ -42,6 +42,14 @@
<%if @authorizer_class_name -%>
######################### ACL handling ##################################
authorizer.class.name=<%= @authorizer_class_name %>
+<%if @allow_everyone_if_no_acl_found -%>
+allow.everyone.if.no.acl.found=<%= @allow_everyone_if_no_acl_found %>
+<% end -%>
+
+<% if @super_users -%>
+super.users=<%= Array(@super_users).join(';') %>
+<% end -%>
+
<% end -%>
######################### Socket Server Settings ########################
diff --git a/modules/profile/manifests/kafka/broker.pp
b/modules/profile/manifests/kafka/broker.pp
index 9c5a188..5e37836 100644
--- a/modules/profile/manifests/kafka/broker.pp
+++ b/modules/profile/manifests/kafka/broker.pp
@@ -8,15 +8,25 @@
# To configure SSL for Kafka brokers, you need the following files
distributable by our Puppet
# secret() function.
#
-# - A keystore.jks file - Contains the key and certificate for this broker
-# - A truststore.jks file - Contains the CA certificate that signed the
broker's certificate
+# - A keystore.jks file - Contains the key and certificate for this kafka
cluster's brokers.
+# - A truststore.jks file - Contains the CA certificate that signed the
cluster certificate
#
# It is expected that the CA certificate in the truststore will also be used
to sign
# all Kafka client certificates. These should be checked into the Puppet
private repository's
# secret module at
#
-# -
secrets/certificates/kafka_broker_${::hostname}/kafka_broker_$hostname.keystore.jks
-# - secrets/certificates/kafka_broker_${::hostname}/truststore.jks
+# -
secrets/certificates/kafka_${kafka_cluster_name_full}_broker/kafka_${kafka_cluster_name_full}_broker.keystore.jks
+# -
secrets/certificates/kafka_${kafka_cluster_name_full}_broker/truststore.jks
+#
+# Where ${kafka_cluster_name_full} is the fully qualified Kafka cluster name
that matches
+# entries in the $kafka_clusters hash. E.g. jumbo-eqiad, main-codfw, etc.
+#
+# If both $ssl_enabled and $auth_acls_enabled, this class will grant cluster
level
+# permissions to the broker's SSL certificate.
+# It is expected that the certificate is subjectless, i.e. it's DN can be
specified
+# simply as CN=kafka_${kafka_cluster_name_full}_broker. This will be used as
the
+# Kafka cluster broker principal. --cluster ACLs will automatically be
granted for
+# User:CN=kafka_${kafka_cluster_name_full}_broker.
#
# This layout is built to work with certificates generated using cergen like
# cergen --base-path /srv/private/modules/secret/secrets/certificates ...
@@ -31,7 +41,7 @@
#
# [*kafka_cluster_name*]
# Kafka cluster name. This should be the non DC/project suffixed cluster
name,
-# e.g. main, aggregate, simple, etc. The kafka_cluster_name puppet parser
+# e.g. main, aggregate, simple, etc. The kafka_cluster_name() puppet parser
# function will determine the proper full cluster name based on $::site
# and/or $::labsproject. Hiera: kafka_cluster_name
#
@@ -88,8 +98,7 @@
#
# [*auth_acls_enabled*]
# Enables the kafka.security.auth.SimpleAclAuthorizer bundled with Kafka.
-# This will also increase the verbosity of authorization logs for a better
-# user accounting. Default: false
+# Default: false
#
# [*monitoring_enabled*]
# Enable monitoring and alerts for this broker. Default: false
@@ -115,10 +124,6 @@
$auth_acls_enabled =
hiera('profile::kafka::broker::auth_acls_enabled', false),
$monitoring_enabled =
hiera('profile::kafka::broker::monitoring_enabled', false),
) {
- # TODO: WIP
- $tls_secrets_path = undef
- $tls_key_password = undef
-
$config = kafka_config($kafka_cluster_name)
$cluster_name = $config['name']
$zookeeper_url = $config['zookeeper']['url']
@@ -146,7 +151,7 @@
$ssl_listener = "SSL://:${ssl_port}"
# Conditionally set $listeners and $ssl_client_auth
- # based on values of $tls and $plaintext.
+ # based on values of $ssl_enabled and $plaintext.
if $ssl_enabled and $plaintext {
$listeners = [$plaintext_listener, $ssl_listener]
$ssl_client_auth = 'requested'
@@ -160,7 +165,7 @@
$ssl_client_auth = 'required'
}
else {
- fatal('Must set at least one of $plaintext or $ssl_enabled to true.')
+ fail('Must set at least one of $plaintext or $ssl_enabled to true.')
}
if $ssl_enabled {
@@ -170,10 +175,10 @@
# Distribute Java keystore and truststore for this broker.
$ssl_location = '/etc/kafka/ssl'
- $ssl_keystore_secrets_path =
"certificates/kafka_broker_${::hostname}/kafka_broker_${::hostname}.keystore.jks"
- $ssl_keystore_location =
"${ssl_location}/kafka_broker_${::hostname}.keystore.jks"
+ $ssl_keystore_secrets_path =
"certificates/kafka_${cluster_name}_broker/kafka_${cluster_name}_broker.keystore.jks"
+ $ssl_keystore_location =
"${ssl_location}/kafka_${cluster_name}_broker.keystore.jks"
- $ssl_truststore_secrets_path =
"certificates/kafka_broker_${::hostname}/truststore.jks"
+ $ssl_truststore_secrets_path =
"certificates/kafka_${cluster_name}_broker/truststore.jks"
$ssl_truststore_location = "${ssl_location}/truststore.jks"
file { $ssl_location:
@@ -274,6 +279,23 @@
authorizer_class_name => $authorizer_class_name,
}
+ # If both auth ACLs AND SSL are enabled, then we will need cluster level
+ # ACLs for the brokers to be able to talk to each other.
+ if $auth_acls_enabled and $ssl_enabled {
+ $cluster_principal = "User:CN=kafka_${cluster_name}_broker"
+ $kafka_acls_command = "/usr/bin/kafka-acls --authorizer-properties
zookeeper.connect=${config['zookeeper']['url']}"
+ exec { "kafka_grant_cluster_acl_to_${cluster_principal}":
+ command => "${kafka_acls_command} --add --cluster
--allow-principal ${cluster_principal}",
+ unless => "${kafka_acls_command} --list | grep -A 1 'Current ACLs
for resource `Cluster:kafka-cluster`:' | grep -q '${cluster_principal} has
Allow permission for operations: All'",
+ require => Class['::confluent::kafka::broker'],
+ }
+ }
+
+ $ferm_srange = $::realm ? {
+ 'production' => '($PRODUCTION_NETWORKS $FRACK_NETWORKS)',
+ 'labs' => '($LABS_NETWORKS)',
+ }
+
$ferm_plaintext_ensure = $plaintext ? {
false => 'absent',
undef => 'absent',
@@ -285,7 +307,7 @@
proto => 'tcp',
port => $plaintext_port,
notrack => true,
- srange => '($PRODUCTION_NETWORKS $FRACK_NETWORKS)',
+ srange => $ferm_srange,
}
$ferm_ssl_ensure = $ssl_enabled ? {
@@ -299,7 +321,7 @@
proto => 'tcp',
port => $ssl_port,
notrack => true,
- srange => '($PRODUCTION_NETWORKS $FRACK_NETWORKS)',
+ srange => $ferm_srange,
}
# In case of mediawiki spikes we've been seeing up to 300k connections,
--
To view, visit https://gerrit.wikimedia.org/r/394438
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I97089ca833d74717c07aab008277a867651a71b1
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Elukey <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
