Ottomata has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/392978 )
Change subject: Puppetization for superset
......................................................................
Puppetization for superset
This just sets up the superset service.
TODO: LVS, DNS, LDAP auth, etc.
Bug: T166689
Change-Id: Ib63b35e409ba4c8fbc77f342220f1f4c4f6fecea
---
M hieradata/labs/deployment-prep/common.yaml
M hieradata/role/common/deployment_server.yaml
M hieradata/role/common/statistics/web.yaml
A modules/profile/manifests/superset.pp
M modules/role/manifests/statistics/web.pp
A modules/superset/files/superset.profile.firejail
A modules/superset/manifests/init.pp
A modules/superset/templates/gunicorn_config.py.erb
A modules/superset/templates/initscripts/superset.systemd.erb
A modules/superset/templates/superset_config.py.erb
10 files changed, 313 insertions(+), 0 deletions(-)
Approvals:
Ottomata: Looks good to me, approved
Elukey: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/hieradata/labs/deployment-prep/common.yaml
b/hieradata/labs/deployment-prep/common.yaml
index 9930669..f0f3209 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -311,6 +311,10 @@
eventstreams/deploy:
repository: mediawiki/services/eventstreams/deploy
+ # Superset
+ analytics/superset/deploy:
+ repository: analytics/superset/deploy
+
# IEG grant review
iegreview/iegreview:
repository: iegreview
diff --git a/hieradata/role/common/deployment_server.yaml
b/hieradata/role/common/deployment_server.yaml
index 5c5e154..490d237 100644
--- a/hieradata/role/common/deployment_server.yaml
+++ b/hieradata/role/common/deployment_server.yaml
@@ -107,6 +107,9 @@
# Public EventStreams service
eventstreams/deploy:
repository: mediawiki/services/eventstreams/deploy
+ # Superset
+ analytics/superset/deploy:
+ repository: analytics/superset/deploy
gerrit/gerrit:
repository: operations/software/gerrit
graphoid/deploy: {}
diff --git a/hieradata/role/common/statistics/web.yaml
b/hieradata/role/common/statistics/web.yaml
index f54e5f2..2a48bc2 100644
--- a/hieradata/role/common/statistics/web.yaml
+++ b/hieradata/role/common/statistics/web.yaml
@@ -4,3 +4,7 @@
- statistics-web-users
- statistics-admins
profile::statistics::web::geowiki_host: 'stat1006.eqiad.wmnet'
+
+# Database password will be filled added in profile::superset from the
+# profile::superset::database_password hiera variable stored in the private
repo.
+profile::superset::database_uri:
mysql://[email protected]/superset
diff --git a/modules/profile/manifests/superset.pp
b/modules/profile/manifests/superset.pp
new file mode 100644
index 0000000..7115dad
--- /dev/null
+++ b/modules/profile/manifests/superset.pp
@@ -0,0 +1,61 @@
+# == Class profile::superset
+# Sets up superset site.
+#
+# == Parameters
+#
+# [*database_uri*]
+# SQL Alchemy database URI to use as the superset state store.
+# This can be a passwordless URI if $database_password is provided.
+#
+# [*database_password*]
+# If given, it will be inserted into the $database_uri. This expects that
$database_uri
+# is of the form 'protocol://username@hostname/databasename'.
+#
+# [*admin_user*]
+# Web UI admin user
+#
+# [*admin_password*]
+# Web UI admin user password
+
+# [*secret_key*]
+# flask secret key
+#
+# [*password_mapping*]
+# Hash of sqlalchemy URIs to passwords. This will be used
+# for the SQLALCHEMY_CUSTOM_PASSWORD_STORE, to allow for
+# passwords to external databases to be provided via configuration,
+# rather than the web UI.
+#
+# [*statsd*]
+# statsd host:port
+#
+class profile::superset(
+ $database_uri = hiera('profile::superset::database_uri',
'sqlite:////var/lib/superset/superset.db'),
+ $database_password = hiera('profile::superset::database_password', undef),
+ $admin_user = hiera('profile::superset::admin_user', 'admin'),
+ $admin_password = hiera('profile::superset::admin_pass', 'admin'),
+ $secret_key = hiera('profile::superset::secret_key',
'not_really_a_secret_key'),
+ $password_mapping = hiera('profile::superset::password_mapping', undef),
+ $statsd = hiera('statsd', undef),
+) {
+ # If given $database_password, insert it into $database_uri.
+ $full_database_uri = $database_password ? {
+ undef => $database_uri,
+ default => regsubst($database_uri, '(\w+)://(\w*)@(.*)',
"\\1://\\2:${database_password}@\\3")
+ }
+
+ class { '::superset':
+ database_uri => $full_database_uri,
+ secret_key => $secret_key,
+ admin_user => $admin_user,
+ admin_password => $admin_password,
+ password_mapping => $password_mapping,
+ statsd => $statsd,
+ }
+
+ monitoring::service { 'superset':
+ description => 'superset',
+ check_command => "check_tcp!${::superset::port}",
+ require => Systemd::Service['superset'],
+ }
+}
diff --git a/modules/role/manifests/statistics/web.pp
b/modules/role/manifests/statistics/web.pp
index 839fa77..dda3959 100644
--- a/modules/role/manifests/statistics/web.pp
+++ b/modules/role/manifests/statistics/web.pp
@@ -5,4 +5,7 @@
}
include ::profile::statistics::web
+
+ # Superset. T166689
+ include ::profile::superset
}
diff --git a/modules/superset/files/superset.profile.firejail
b/modules/superset/files/superset.profile.firejail
new file mode 100644
index 0000000..acd4e84
--- /dev/null
+++ b/modules/superset/files/superset.profile.firejail
@@ -0,0 +1,26 @@
+# system directories
+blacklist /sbin
+blacklist /usr/sbin
+blacklist /usr/local/sbin
+
+# system management
+blacklist ${PATH}/umount
+blacklist ${PATH}/mount
+blacklist ${PATH}/fusermount
+blacklist ${PATH}/su
+blacklist ${PATH}/sudo
+blacklist ${PATH}/xinput
+blacklist ${PATH}/evtest
+blacklist ${PATH}/xev
+blacklist ${PATH}/strace
+blacklist ${PATH}/nc
+blacklist ${PATH}/ncat
+
+blacklist /etc/shadow
+blacklist /etc/ssh
+blacklist /root
+blacklist /home
+noroot
+caps.drop all
+seccomp
+private-dev
diff --git a/modules/superset/manifests/init.pp
b/modules/superset/manifests/init.pp
new file mode 100644
index 0000000..2041a3e
--- /dev/null
+++ b/modules/superset/manifests/init.pp
@@ -0,0 +1,141 @@
+# == Class superset
+# Installs superset via scap and runs it.
+#
+# If you are providing a custom $database_uri, you must ensure that the
database exists
+# and the configured db user/pass in the URI can write to that database.
+# Tables will be auto created.
+#
+# == Parameters
+#
+# [*port*]
+# Port on which superset will listen for HTTP connections.
+#
+# [*database_uri*]
+# SQL Alchemy database URI.
+#
+# [*workers*]
+# Number of gevent workers
+#
+# [*admin_user*]
+# Web UI admin user
+#
+# [*admin_password*]
+# Web UI admin user password
+#
+# [*secret_key*]
+# flask secret key
+#
+# [*password_mapping*]
+# Hash of sqlalchemy URIs to passwords. This will be used
+# for the SQLALCHEMY_CUSTOM_PASSWORD_STORE, to allow for
+# passwords to external databases to be provided via configuration,
+# rather than the web UI.
+#
+# [*statsd*]
+# statsd host:port
+#
+# [*deployment_user*]
+# scap deployment user
+#
+class superset(
+ $port = 9080,
+ $database_uri = 'sqlite:////var/lib/superset/superset.db',
+ $workers = 4,
+ $admin_user = 'admin',
+ $admin_password = 'admin',
+ $secret_key = 'not_really_a_secret_key',
+ $password_mapping = undef,
+ $statsd = undef,
+ $deployment_user = 'analytics_deploy',
+) {
+ requires_os('debian >= jessie')
+ require_package('python', 'virtualenv', 'firejail')
+
+ $deployment_dir = '/srv/deployment/analytics/superset/deploy'
+ $virtualenv_dir = '/srv/deployment/analytics/superset/venv'
+
+ # superset runs gunicorn with gevent, use debian provided python-gevent.
+ require_package('python-gevent')
+
+ scap::target { 'analytics/superset/deploy':
+ deploy_user => $deployment_user,
+ service_name => 'superset',
+ }
+
+ group { 'superset':
+ ensure => present,
+ system => true,
+ }
+
+ user { 'superset':
+ gid => 'superset',
+ shell => '/bin/bash',
+ system => true,
+ managehome => true,
+ home => '/var/lib/superset',
+ require => Group['superset'],
+ }
+
+ file { '/etc/firejail/superset.profile':
+ ensure => 'present',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source => 'puppet:///modules/superset/superset.profile.firejail',
+ }
+
+ file { '/etc/superset':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+
+ file { '/etc/superset/gunicorn_config.py':
+ ensure => 'present',
+ owner => 'root',
+ group => 'superset',
+ mode => '0444',
+ content => template('superset/gunicorn_config.py.erb'),
+ }
+
+ file { '/etc/superset/superset_config.py':
+ ensure => 'present',
+ owner => 'root',
+ group => 'superset',
+ mode => '0440',
+ content => template('superset/superset_config.py.erb'),
+ }
+
+ # This will create tables in the superset database and add the web GUI
admin user.
+ exec { 'init_superset':
+ command => "${deployment_dir}/init_superset.sh ${admin_user}
${admin_password}",
+ # Don't run init_superset.sh if superset database exists.
+ unless => "${virtualenv_dir}/bin/python
${deployment_dir}/superset_database_exists.py ${database_uri}",
+ user => 'superset',
+ # Set PYTHONPATH to read superset_config.py
+ environment => ['PYTHONPATH=/etc/superset',
'SUPERSET_HOME=/var/lib/superset'],
+ require => [
+ Scap::Target['analytics/superset/deploy'],
+ File['/etc/superset/superset_config.py'],
+ ],
+ }
+
+ systemd::syslog { 'superset':
+ readable_by => 'all',
+ base_dir => '/var/log',
+ group => 'root',
+ }
+
+ systemd::service { 'superset':
+ ensure => 'present',
+ content => systemd_template('superset'),
+ restart => true,
+ require => [
+ File['/etc/firejail/superset.profile'],
+ File['/etc/superset/gunicorn_config.py'],
+ Exec['init_superset'],
+ Systemd::Syslog['superset'],
+ ],
+ }
+}
diff --git a/modules/superset/templates/gunicorn_config.py.erb
b/modules/superset/templates/gunicorn_config.py.erb
new file mode 100644
index 0000000..351cb24
--- /dev/null
+++ b/modules/superset/templates/gunicorn_config.py.erb
@@ -0,0 +1,7 @@
+bind = '0.0.0.0:<%= @port %>'
+workers = <%= @workers %>
+worker_class = 'gevent'
+timeout = 120
+<% if @statsd -%>
+statsd_host = '<%= @statsd %>'
+<% end -%>
diff --git a/modules/superset/templates/initscripts/superset.systemd.erb
b/modules/superset/templates/initscripts/superset.systemd.erb
new file mode 100644
index 0000000..c6354f6
--- /dev/null
+++ b/modules/superset/templates/initscripts/superset.systemd.erb
@@ -0,0 +1,27 @@
+# NOTE: This file is managed by Puppet.
+# Systemd unit for the AirBnB Superset UI
+[Unit]
+Description="superset service"
+After=network.target
+
+[Service]
+User=superset
+Group=superset
+
+# /etc/superset contains superset_config.py, which will be used if it is in
PYTHONPATH.
+Environment="PYTHONPATH=/etc/superset"
+Environment="SUPERSET_HOME=/var/lib/superset"
+
+Restart=always
+RestartSec=2s
+# wait 60 seconds for a graceful restart before killing the master
+TimeoutStopSec=60
+WorkingDirectory=<%= @deployment_dir %>
+ExecStart=/usr/bin/firejail --profile=/etc/firejail/superset.profile -- <%=
@virtualenv_dir %>/bin/gunicorn \
+ --config /etc/superset/gunicorn_config.py \
+ superset:app
+
+SyslogIdentifier=superset
+
+[Install]
+WantedBy=multi-user.target
diff --git a/modules/superset/templates/superset_config.py.erb
b/modules/superset/templates/superset_config.py.erb
new file mode 100644
index 0000000..c7f3772
--- /dev/null
+++ b/modules/superset/templates/superset_config.py.erb
@@ -0,0 +1,37 @@
+# NOTE: This file is managed by Puppet.
+
+#---------------------------------------------------------
+# Flask App Builder configuration
+#---------------------------------------------------------
+# Your App secret key
+SECRET_KEY = '<%= @secret_key %>'
+
+# The SQLAlchemy connection string to your database backend
+# This connection defines the path to the database that stores your
+# superset metadata (slices, connections, tables, dashboards, ...).
+# Note that the connection information to connect to the datasources
+# you want to explore are managed directly in the web UI
+SQLALCHEMY_DATABASE_URI = '<%= @database_uri %>'
+
+
+<%
+# If @password_mapping is given, output a python dict
+# mapping sqlalchemy URIs to passwords. Then create
+# a function that returns password given the uri, and
+# assign that to SQLALCHEMY_CUSTOM_PASSWORD_STORE.
+#
+# This allows for configuration of databases in the
+# superset GUI, without having to enter passwords through it.
+#
+if @password_mapping
+-%>
+password_mapping = {
+<% @password_mapping.keys.sort do |uri| -%>
+ '<%= uri %>': '<%= @password_mapping[uri] %>',
+<% end -%>
+}
+def lookup_password(uri):
+ return password_mapping.get(uri, None)
+SQLALCHEMY_CUSTOM_PASSWORD_STORE = lookup_password
+
+<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/392978
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib63b35e409ba4c8fbc77f342220f1f4c4f6fecea
Gerrit-PatchSet: 14
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Elukey <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
