Ottomata has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/394438 )
Change subject: Improvements for Kafka + SSL
......................................................................
Improvements for Kafka + SSL
- Reduce default authorizer log level, DEBUG is way to much!
- Assume that all kafka brokers in this cluster use the same certificate and key
-- Certificate should be subjectless with CN=kafka_${kafka_cluster_name}_broker
for easier DN specification
- default allow_everyone_if_no_acl_found to true; we want to restrict certain
topics with ACLs, not all by default.
Bug: T167304
Change-Id: I97089ca833d74717c07aab008277a867651a71b1
---
M hieradata/role/common/kafka/simple/broker.yaml
M modules/confluent/manifests/kafka/broker.pp
M modules/confluent/templates/kafka/log4j.properties.erb
M modules/confluent/templates/kafka/server.properties.erb
M modules/profile/manifests/kafka/broker.pp
5 files changed, 56 insertions(+), 20 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/38/394438/1
diff --git a/hieradata/role/common/kafka/simple/broker.yaml
b/hieradata/role/common/kafka/simple/broker.yaml
index 5e5c531..4159621 100644
--- a/hieradata/role/common/kafka/simple/broker.yaml
+++ b/hieradata/role/common/kafka/simple/broker.yaml
@@ -4,8 +4,6 @@
profile::kafka::broker::monitoring_enabled: false
profile::kafka::broker::log_dirs: [/srv/kafka/data]
profile::kafka::broker::plaintext: true
-profile::kafka::broker::tls_secrets_path: false
-profile::kafka::broker::tls_key_password: false
profile::kafka::broker::auto_leader_rebalance_enable: true
profile::kafka::broker::nofiles_ulimit: 8192
diff --git a/modules/confluent/manifests/kafka/broker.pp
b/modules/confluent/manifests/kafka/broker.pp
index f617bb4..d4243fd 100644
--- a/modules/confluent/manifests/kafka/broker.pp
+++ b/modules/confluent/manifests/kafka/broker.pp
@@ -206,11 +206,22 @@
# The maximum message size allowed.
# Default: 1048576
#
+# [*allow_everyone_if_no_acl_found*]
+# If this value is on true, only the topics on which are ACLs are set are
secured.
+# Default: true
+#
+# [*super_users*]
+# List of super user CNs. If configuring SSL, this should at least include
the cluster's SSL
+# principal so the cluster can operate.
+#
# [*authorizer_class_name*]
# Sets up the ACL authorization provider specified
# as parameter. It also set up a more verbose log4j logging related
# to ACL authorization events.
# Default: undef
+#
+# [*authorizer_log_level*]
+# Default: INFO
#
class confluent::kafka::broker(
$enabled = true,
@@ -286,7 +297,10 @@
$message_max_bytes = 1048576,
+ $allow_everyone_if_no_acl_found = true,
+ $super_users = undef,
$authorizer_class_name = undef,
+ $authorizer_log_level = 'INFO',
) {
# confluent::kafka::common installs the kafka package
# and a handy wrapper script.
diff --git a/modules/confluent/templates/kafka/log4j.properties.erb
b/modules/confluent/templates/kafka/log4j.properties.erb
index 563309c..3fdfa13 100644
--- a/modules/confluent/templates/kafka/log4j.properties.erb
+++ b/modules/confluent/templates/kafka/log4j.properties.erb
@@ -75,11 +75,8 @@
log4j.additivity.state.change.logger=false
<%if @authorizer_class_name -%>
-# authorizer.class.name set in server.properties, therefore a more verbose
-# log accounting is needed.
-log4j.logger.kafka.authorizer.logger=DEBUG, authorizerAppender
-<% else -%>
-#Change this to debug to get the actual audit log for authorizer.
-log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender
+# Set this to DEBUG to get actual audit log for authorizer.
+# This will make logs very verbose!
+log4j.logger.kafka.authorizer.logger=<%= @authorizer_log_level %>,
authorizerAppender
<% end -%>
log4j.additivity.kafka.authorizer.logger=false
diff --git a/modules/confluent/templates/kafka/server.properties.erb
b/modules/confluent/templates/kafka/server.properties.erb
index a350618..c484d7c 100644
--- a/modules/confluent/templates/kafka/server.properties.erb
+++ b/modules/confluent/templates/kafka/server.properties.erb
@@ -42,6 +42,14 @@
<%if @authorizer_class_name -%>
######################### ACL handling ##################################
authorizer.class.name=<%= @authorizer_class_name %>
+<%if @allow_everyone_if_no_acl_found -%>
+allow.everyone.if.no.acl.found=<%= @allow_everyone_if_no_acl_found %>
+<% end -%>
+
+<% if @super_users -%>
+super.users=<%= Array(@super_users).join(';') %>
+<% end -%>
+
<% end -%>
######################### Socket Server Settings ########################
diff --git a/modules/profile/manifests/kafka/broker.pp
b/modules/profile/manifests/kafka/broker.pp
index 9c5a188..e417a60 100644
--- a/modules/profile/manifests/kafka/broker.pp
+++ b/modules/profile/manifests/kafka/broker.pp
@@ -8,15 +8,18 @@
# To configure SSL for Kafka brokers, you need the following files
distributable by our Puppet
# secret() function.
#
-# - A keystore.jks file - Contains the key and certificate for this broker
-# - A truststore.jks file - Contains the CA certificate that signed the
broker's certificate
+# - A keystore.jks file - Contains the key and certificate for this kafka
cluster's brokers.
+# - A truststore.jks file - Contains the CA certificate that signed the
cluster certificate
#
# It is expected that the CA certificate in the truststore will also be used
to sign
# all Kafka client certificates. These should be checked into the Puppet
private repository's
# secret module at
#
-# -
secrets/certificates/kafka_broker_${::hostname}/kafka_broker_$hostname.keystore.jks
-# - secrets/certificates/kafka_broker_${::hostname}/truststore.jks
+# -
secrets/certificates/kafka_${kafka_cluster_name}_broker/kafka_${kafka_cluster_name}_broker.keystore.jks
+# - secrets/certificates/kafka_${kafka_cluster_name}_broker/truststore.jks
+#
+# It is expected that the certificate is subjectless, i.e. it's DN can be
specified
+# simply as CN=kafka_${kafka_cluster_name}_broker.
#
# This layout is built to work with certificates generated using cergen like
# cergen --base-path /srv/private/modules/secret/secrets/certificates ...
@@ -88,8 +91,7 @@
#
# [*auth_acls_enabled*]
# Enables the kafka.security.auth.SimpleAclAuthorizer bundled with Kafka.
-# This will also increase the verbosity of authorization logs for a better
-# user accounting. Default: false
+# Default: false
#
# [*monitoring_enabled*]
# Enable monitoring and alerts for this broker. Default: false
@@ -160,7 +162,7 @@
$ssl_client_auth = 'required'
}
else {
- fatal('Must set at least one of $plaintext or $ssl_enabled to true.')
+ fail('Must set at least one of $plaintext or $ssl_enabled to true.')
}
if $ssl_enabled {
@@ -170,10 +172,10 @@
# Distribute Java keystore and truststore for this broker.
$ssl_location = '/etc/kafka/ssl'
- $ssl_keystore_secrets_path =
"certificates/kafka_broker_${::hostname}/kafka_broker_${::hostname}.keystore.jks"
- $ssl_keystore_location =
"${ssl_location}/kafka_broker_${::hostname}.keystore.jks"
+ $ssl_keystore_secrets_path =
"certificates/kafka_${kafka_cluster_name}_broker/kafka_${kafka_cluster_name}_broker.keystore.jks"
+ $ssl_keystore_location =
"${ssl_location}/kafka_${kafka_cluster_name}_broker.keystore.jks"
- $ssl_truststore_secrets_path =
"certificates/kafka_broker_${::hostname}/truststore.jks"
+ $ssl_truststore_secrets_path =
"certificates/kafka_${kafka_cluster_name}_broker/truststore.jks"
$ssl_truststore_location = "${ssl_location}/truststore.jks"
file { $ssl_location:
@@ -243,6 +245,16 @@
$authorizer_class_name = undef
}
+ # If both auth ACLs AND SSL are enabled, use the expected DN the
+ # of the broker's certificate. This should be a subjectless
+ # DN, with CN only.
+ if $auth_acls_enabled and $ssl_enabled {
+ $super_users = ["User:CN=kafka_${kafka_cluster_name}_broker"]
+ }
+ else {
+ $super_users = undef
+ }
+
class { '::confluent::kafka::broker':
log_dirs => $log_dirs,
brokers => $config['brokers']['hash'],
@@ -272,6 +284,13 @@
num_replica_fetchers => $num_replica_fetchers,
message_max_bytes => $message_max_bytes,
authorizer_class_name => $authorizer_class_name,
+ super_users => $super_users,
+ }
+
+
+ $ferm_srange = $::realm ? {
+ 'production' => '($PRODUCTION_NETWORKS $FRACK_NETWORKS)',
+ 'labs' => '($LABS_NETWORKS)',
}
$ferm_plaintext_ensure = $plaintext ? {
@@ -285,7 +304,7 @@
proto => 'tcp',
port => $plaintext_port,
notrack => true,
- srange => '($PRODUCTION_NETWORKS $FRACK_NETWORKS)',
+ srange => $ferm_srange,
}
$ferm_ssl_ensure = $ssl_enabled ? {
@@ -299,7 +318,7 @@
proto => 'tcp',
port => $ssl_port,
notrack => true,
- srange => '($PRODUCTION_NETWORKS $FRACK_NETWORKS)',
+ srange => $ferm_srange,
}
# In case of mediawiki spikes we've been seeing up to 300k connections,
--
To view, visit https://gerrit.wikimedia.org/r/394438
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I97089ca833d74717c07aab008277a867651a71b1
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
