Ottomata has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/396103 )
Change subject: Configure LDAP proxy and authentication for Superset ...................................................................... Configure LDAP proxy and authentication for Superset Bug: T166689 Change-Id: I55aca23a9d34ef3496d0f7d837a9f606aeb6319e --- M hieradata/role/common/statistics/web.yaml M modules/profile/manifests/superset.pp M modules/superset/manifests/init.pp A modules/superset/manifests/proxy.pp A modules/superset/templates/superset.wikimedia.org.erb M modules/superset/templates/superset_config.py.erb 6 files changed, 148 insertions(+), 6 deletions(-) Approvals: Ottomata: Verified; Looks good to me, approved diff --git a/hieradata/role/common/statistics/web.yaml b/hieradata/role/common/statistics/web.yaml index 2a48bc2..1b0b388 100644 --- a/hieradata/role/common/statistics/web.yaml +++ b/hieradata/role/common/statistics/web.yaml @@ -8,3 +8,5 @@ # Database password will be filled added in profile::superset from the # profile::superset::database_password hiera variable stored in the private repo. profile::superset::database_uri: mysql://[email protected]/superset +profile::superset::workers: 4 +profile::superset::ldap_proxy_enabled: true diff --git a/modules/profile/manifests/superset.pp b/modules/profile/manifests/superset.pp index 0621e34..d2e86f2 100644 --- a/modules/profile/manifests/superset.pp +++ b/modules/profile/manifests/superset.pp @@ -26,16 +26,24 @@ # passwords to external databases to be provided via configuration, # rather than the web UI. # +# [*ldap_proxy_enabled*] +# If true, an Apache HTTP proxy will be configured to authenticate users with WMF (labs) LDAP. +# Only users in the 'wmf' and 'nda' LDAP groups will be allowed to authenticate. +# This will configure superset with AUTH_TYPE = AUTH_REMOTE_USER, and the authenticated +# HTTP remote user will be used to log into superset. +# # [*statsd*] # statsd host:port # class profile::superset( + $workers = hiera('profile::superset::workers', 1), $database_uri = hiera('profile::superset::database_uri', 'sqlite:////var/lib/superset/superset.db'), $database_password = hiera('profile::superset::database_password', undef), $admin_user = hiera('profile::superset::admin_user', 'admin'), $admin_password = hiera('profile::superset::admin_password', 'admin'), $secret_key = hiera('profile::superset::secret_key', 'not_really_a_secret_key'), $password_mapping = hiera('profile::superset::password_mapping', undef), + $ldap_proxy_enabled = hiera('profile::superset::ldap_proxy_enabled', false), $statsd = hiera('statsd', undef), ) { # If given $database_password, insert it into $database_uri. @@ -44,11 +52,25 @@ default => regsubst($database_uri, '(\w+)://(\w*)@(.*)', "\\1://\\2:${database_password}@\\3") } + if $ldap_proxy_enabled { + # Include the Superset HTTP WMF LDAP auth proxy + class { '::superset::proxy': } + + # Use AUTH_REMOTE_USER if we are using + # LDAP authenticated HTTP proxy. + $auth_type = 'AUTH_REMOTE_USER' + } + else { + $auth_type = undef + } + class { '::superset': + workers => $workers, database_uri => $full_database_uri, secret_key => $secret_key, admin_user => $admin_user, admin_password => $admin_password, + auth_type => $auth_type, password_mapping => $password_mapping, statsd => $statsd, } @@ -56,6 +78,7 @@ monitoring::service { 'superset': description => 'superset', check_command => "check_tcp!${::superset::port}", - require => Systemd::Service['superset'], + require => Class['::superset'], } + } diff --git a/modules/superset/manifests/init.pp b/modules/superset/manifests/init.pp index 2041a3e..73ef7cc 100644 --- a/modules/superset/manifests/init.pp +++ b/modules/superset/manifests/init.pp @@ -4,6 +4,16 @@ # If you are providing a custom $database_uri, you must ensure that the database exists # and the configured db user/pass in the URI can write to that database. # Tables will be auto created. + +# An admin user with $admin_user and $admin_password will be created for you in the +# database. However, if you are using a custom $auth_type, this admin user will not +# be useable. To set up an admin user for your auth type,, run the +# fabmanager create-user command and add a user with details that match the user's account +# in whatever alternative auth you are using, minus the password. E.g.: +# /srv/deployment/analytics/superset/venv/bin/fabmanager create-admin --app superset --username Ottomata --email [email protected] --firstname Andrew --lastname Otto --password BLANK +# This should match exactly the user in your alternative auth (e.g. LDAP), +# except for the password. The password here does not matter, as the value stored in the +# database will not be used for authentication with your alternative auth type. # # == Parameters # @@ -31,6 +41,12 @@ # passwords to external databases to be provided via configuration, # rather than the web UI. # +# [*auth_type*] +# An auth type from flask_appbuilder.security.manager. +# +# [*auth_settings*] +# Hash of additional auth settings to render in superset_config.py +# # [*statsd*] # statsd host:port # @@ -40,22 +56,29 @@ class superset( $port = 9080, $database_uri = 'sqlite:////var/lib/superset/superset.db', - $workers = 4, + $workers = 1, $admin_user = 'admin', $admin_password = 'admin', $secret_key = 'not_really_a_secret_key', $password_mapping = undef, + $auth_type = undef, + $auth_settings = undef, $statsd = undef, $deployment_user = 'analytics_deploy', ) { requires_os('debian >= jessie') - require_package('python', 'virtualenv', 'firejail') + require_package( + 'python', + 'virtualenv', + 'firejail', + # superset runs gunicorn with gevent, + # use debian provided python-gevent. + 'python-gevent' + ) $deployment_dir = '/srv/deployment/analytics/superset/deploy' $virtualenv_dir = '/srv/deployment/analytics/superset/venv' - # superset runs gunicorn with gevent, use debian provided python-gevent. - require_package('python-gevent') scap::target { 'analytics/superset/deploy': deploy_user => $deployment_user, diff --git a/modules/superset/manifests/proxy.pp b/modules/superset/manifests/proxy.pp new file mode 100644 index 0000000..1e35bc4 --- /dev/null +++ b/modules/superset/manifests/proxy.pp @@ -0,0 +1,24 @@ +# == Class superset::proxy +# Sets up a WMF HTTP LDAP auth proxy. +# +class superset::proxy { + include ::apache::mod::proxy_http + include ::apache::mod::proxy + include ::apache::mod::auth_basic + include ::apache::mod::authnz_ldap + include ::apache::mod::headers + include ::passwords::ldap::production + + $proxypass = $passwords::ldap::production::proxypass + + # Set up the VirtualHost + apache::site { 'superset.wikimedia.org': + content => template('superset/superset.wikimedia.org.erb'), + } + + ferm::service { 'superset-http': + proto => 'tcp', + port => '80', + srange => '$CACHE_MISC', + } +} diff --git a/modules/superset/templates/superset.wikimedia.org.erb b/modules/superset/templates/superset.wikimedia.org.erb new file mode 100644 index 0000000..8d82904 --- /dev/null +++ b/modules/superset/templates/superset.wikimedia.org.erb @@ -0,0 +1,43 @@ +##################################################################### +### THIS FILE IS MANAGED BY PUPPET +##################################################################### +# vim: filetype=apache + +<VirtualHost *:80> + ServerName superset.wikimedia.org + ServerSignature Off + + # Not used since this VHost will only act as proxy, + # but it is needed to avoid error messages in the httpd + # logs and logrotate crons. + DocumentRoot /var/www + + <Directory /> + Options FollowSymLinks + AllowOverride None + Require all denied + </Directory> + + <Location /> + AuthName "WMF Labs (use wiki login name not shell)" + AuthType Basic + AuthBasicProvider ldap + AuthLDAPBindDN cn=proxyagent,ou=profile,dc=wikimedia,dc=org + AuthLDAPBindPassword <%= @proxypass %> + AuthLDAPURL "ldaps://ldap-labs.eqiad.wikimedia.org ldap-labs.codfw.wikimedia.org/ou=people,dc=wikimedia,dc=org?cn" + Require ldap-group cn=wmf,ou=groups,dc=wikimedia,dc=org + Require ldap-group cn=nda,ou=groups,dc=wikimedia,dc=org + # Set a header so that superset/flask can authenticate the user. + # This requires that middlewhere pull the HTTP_X_REMOTE_USER + # from the app environ and set environ['REMOTE_USER'] to it. + RequestHeader set X-Remote-User expr=%{REMOTE_USER} + </Location> + + CustomLog /var/log/apache2/superset.wikimedia.org-access.log wmf + ErrorLog /var/log/apache2/superset.wikimedia.org-error.log + + LogLevel warn + + ProxyPass / http://localhost:9080/ + ProxyPassReverse / http://localhost:9080/ +</VirtualHost> diff --git a/modules/superset/templates/superset_config.py.erb b/modules/superset/templates/superset_config.py.erb index c7f3772..eb62719 100644 --- a/modules/superset/templates/superset_config.py.erb +++ b/modules/superset/templates/superset_config.py.erb @@ -13,6 +13,32 @@ # you want to explore are managed directly in the web UI SQLALCHEMY_DATABASE_URI = '<%= @database_uri %>' +<% if @auth_type -%> +from flask_appbuilder.security.manager import <%= @auth_type %> +AUTH_TYPE = <%= @auth_type %> +<% if @auth_type == 'AUTH_REMOTE_USER' -%> +ENABLE_PROXY_FIX = True + +# Set environ['REMOTE_USER'] = HTTP_X_REMOTE_USER header. +# Make sure your HTTP proxy that handles athentication +# sets the X-Remote-User header properly. +class RemoteUserMiddleware(object): + def __init__(self, app): + self.app = app + def __call__(self, environ, start_response): + user = environ.pop('HTTP_X_REMOTE_USER', None) + environ['REMOTE_USER'] = user + return self.app(environ, start_response) + +ADDITIONAL_MIDDLEWARE = [RemoteUserMiddleware, ] +<% end -%> + +<% end -%> +<% if @auth_settings -%> +<% @auth_settings.keys.sort.each do |key| -%> +<%= key %> = '<%= @auth_settings[key] %>' +<% end -%> +<% end -%> <% # If @password_mapping is given, output a python dict @@ -26,7 +52,7 @@ if @password_mapping -%> password_mapping = { -<% @password_mapping.keys.sort do |uri| -%> +<% @password_mapping.keys.sort.each do |uri| -%> '<%= uri %>': '<%= @password_mapping[uri] %>', <% end -%> } @@ -35,3 +61,4 @@ SQLALCHEMY_CUSTOM_PASSWORD_STORE = lookup_password <% end -%> + -- To view, visit https://gerrit.wikimedia.org/r/396103 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I55aca23a9d34ef3496d0f7d837a9f606aeb6319e Gerrit-PatchSet: 6 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <[email protected]> Gerrit-Reviewer: Elukey <[email protected]> Gerrit-Reviewer: Ottomata <[email protected]> Gerrit-Reviewer: Volans <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
