Elukey has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/398869 )
Change subject: profile::mariadb::misc::el::master: apply data sanitization
policies
......................................................................
profile::mariadb::misc::el::master: apply data sanitization policies
Bug: T108850
Change-Id: I1f4d273e1e1d537fdbd978cb69ea804ee9ea8802
---
M modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
A modules/profile/manifests/mariadb/misc/eventlogging/sanitization.pp
M modules/role/manifests/mariadb/misc/eventlogging/master.pp
M modules/role/manifests/mariadb/misc/eventlogging/replica.pp
4 files changed, 123 insertions(+), 73 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/69/398869/1
diff --git a/modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
b/modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
index 76ea7cb..fdb5531 100644
--- a/modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
+++ b/modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
@@ -50,27 +50,22 @@
require => Package['python3-pymysql'],
}
- file { '/etc/eventlogging':
- ensure => 'directory',
- owner => 'root',
- group => 'root',
- mode => '0755',
+ if ! defined(File['/etc/eventlogging']) {
+ file { '/etc/eventlogging':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
}
- file { '/var/log/eventlogging':
- ensure => 'directory',
- owner => 'root',
- group => 'eventlog',
- mode => '0775',
- }
-
- file { '/etc/eventlogging/whitelist.tsv':
- ensure => 'present',
- owner => 'root',
- group => 'root',
- mode => '0444',
- source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv',
- require => File['/etc/eventlogging'],
+ if ! defined(File['/var/log/eventlogging']) {
+ file { '/var/log/eventlogging':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'eventlog',
+ mode => '0775',
+ }
}
file { '/usr/local/bin/eventlogging_sync.sh':
@@ -81,9 +76,9 @@
source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_sync.sh',
}
- logrotate::rule { 'eventlogging':
+ logrotate::rule { 'eventlogging-sync':
ensure => present,
- file_glob => '/var/log/eventlogging/eventlogging_*.log',
+ file_glob => '/var/log/eventlogging/eventlogging_sync.log',
frequency => 'daily',
copy_truncate => true,
compress => true,
@@ -93,59 +88,16 @@
su => 'root eventlog',
}
- # Custom init scripts only that should be deprecated as soon as
- # the profile will run on Debian OS only.
- # The init script manages stdout/stderr to two separate files,
- # meanwhile the systemd unit used below will use a rsyslog dedicated
config.
- if os_version('ubuntu >= trusty') {
- file { '/etc/init.d/eventlogging_sync':
- owner => 'root',
- group => 'root',
- mode => '0555',
- content =>
template('profile/initscripts/mariadb/misc/eventlogging/eventlogging_sync.sysvinit.erb'),
- require => File['/usr/local/bin/eventlogging_sync.sh'],
- notify => Service['eventlogging_sync'],
- }
+ rsyslog::conf { 'eventlogging_sync':
+ source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_sync_rsyslog.conf',
+ priority => 20,
+ }
- service { 'eventlogging_sync':
- ensure => running,
- enable => true,
- }
- } else {
- rsyslog::conf { 'eventlogging_sync':
- source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_sync_rsyslog.conf',
- priority => 20,
- }
-
- $eventlogging_sync_uid = 'root'
- $eventlogging_sync_gid = 'root'
- base::service_unit { 'eventlogging_sync':
- ensure => present,
- systemd =>
systemd_template('mariadb/misc/eventlogging/eventlogging_sync'),
- }
-
- # Sanitization of data in the log database via a custom script
- # The eventlogging_cleaner script uses the --start-ts-file file option,
- # that forces it to look for a file containing a timestamp in the
format
- # %Y%m%d%H%M%S. If the file is not existent, the script will fail
gracefully
- # without doing any action to the db. This is useful to avoid gaps in
- # records sanitized if the script fails and does not commit a new
timestamp.
- $eventlogging_cleaner_command = '/usr/local/bin/eventlogging_cleaner
--whitelist /etc/eventlogging/whitelist.tsv --older-than 90 --start-ts-file
/var/run/eventlogging_cleaner --batch-size 10000 --sleep-between-batches 2'
- $command = "/usr/bin/flock --verbose -n /var/lock/eventlogging_cleaner
${eventlogging_cleaner_command} >>
/var/log/eventlogging/eventlogging_cleaner.log"
- cron { 'eventlogging_cleaner daily sanitization':
- ensure => present,
- command => $command,
- user => 'eventlogcleaner',
- minute => 0,
- hour => 11,
- environment => '[email protected]',
- require => [
- File['/usr/local/bin/eventlogging_cleaner'],
- File['/etc/eventlogging/whitelist.tsv'],
- File['/var/log/eventlogging'],
- User['eventlogcleaner'],
- ]
- }
+ $eventlogging_sync_uid = 'root'
+ $eventlogging_sync_gid = 'root'
+ base::service_unit { 'eventlogging_sync':
+ ensure => present,
+ systemd =>
systemd_template('mariadb/misc/eventlogging/eventlogging_sync'),
}
nrpe::monitor_service { 'eventlogging_sync':
diff --git
a/modules/profile/manifests/mariadb/misc/eventlogging/sanitization.pp
b/modules/profile/manifests/mariadb/misc/eventlogging/sanitization.pp
new file mode 100644
index 0000000..2392339
--- /dev/null
+++ b/modules/profile/manifests/mariadb/misc/eventlogging/sanitization.pp
@@ -0,0 +1,90 @@
+# == Class profile::mariadb::misc::eventlogging::sanitization
+#
+# Deploys the eventlogging_cleaner.py script to apply Analytics data
+# retention policies to the log database running in localhost.
+#
+class profile::mariadb::misc::eventlogging::sanitization {
+
+ user { 'eventlogcleaner':
+ gid => 'eventlog',
+ shell => '/bin/false',
+ home => '/nonexistent',
+ comment => 'EventLogging cleaner user',
+ system => true,
+ managehome => false,
+ require => Group['eventlog'],
+ }
+
+ require_package('python3-pymysql')
+
+ file { '/usr/local/bin/eventlogging_cleaner':
+ ensure => present,
+ owner => 'eventlogcleaner',
+ group => 'eventlog',
+ mode => '0550',
+ source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_cleaner.py',
+ require => Package['python3-pymysql'],
+ }
+
+ if !defined(File['/etc/eventlogging']) {
+ file { '/etc/eventlogging':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+ }
+
+ if !defined(File['/var/log/eventlogging']) {
+ file { '/var/log/eventlogging':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'eventlog',
+ mode => '0775',
+ }
+ }
+
+ file { '/etc/eventlogging/whitelist.tsv':
+ ensure => 'present',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv',
+ require => File['/etc/eventlogging'],
+ }
+
+ logrotate::rule { 'eventlogging-cleaner':
+ ensure => present,
+ file_glob => '/var/log/eventlogging/eventlogging_cleaner.log',
+ frequency => 'daily',
+ copy_truncate => true,
+ compress => true,
+ missing_ok => true,
+ not_if_empty => true,
+ rotate => 14,
+ su => 'root eventlog',
+ }
+
+ # Sanitization of data in the log database via a custom script
+ # The eventlogging_cleaner script uses the --start-ts-file file option,
+ # that forces it to look for a file containing a timestamp in the format
+ # %Y%m%d%H%M%S. If the file is not existent, the script will fail
gracefully
+ # without doing any action to the db. This is useful to avoid gaps in
+ # records sanitized if the script fails and does not commit a new
timestamp.
+ $eventlogging_cleaner_command = '/usr/local/bin/eventlogging_cleaner
--whitelist /etc/eventlogging/whitelist.tsv --older-than 90 --start-ts-file
/var/run/eventlogging_cleaner --batch-size 10000 --sleep-between-batches 2'
+ $command = "/usr/bin/flock --verbose -n /var/lock/eventlogging_cleaner
${eventlogging_cleaner_command} >>
/var/log/eventlogging/eventlogging_cleaner.log"
+ cron { 'eventlogging_cleaner daily sanitization':
+ ensure => present,
+ command => $command,
+ user => 'eventlogcleaner',
+ minute => 0,
+ hour => 11,
+ environment => '[email protected]',
+ require => [
+ File['/usr/local/bin/eventlogging_cleaner'],
+ File['/etc/eventlogging/whitelist.tsv'],
+ File['/var/log/eventlogging'],
+ User['eventlogcleaner'],
+ ]
+ }
+}
\ No newline at end of file
diff --git a/modules/role/manifests/mariadb/misc/eventlogging/master.pp
b/modules/role/manifests/mariadb/misc/eventlogging/master.pp
index 52d56c1..ab1b779 100644
--- a/modules/role/manifests/mariadb/misc/eventlogging/master.pp
+++ b/modules/role/manifests/mariadb/misc/eventlogging/master.pp
@@ -5,6 +5,11 @@
include ::profile::mariadb::monitor
include ::profile::mariadb::misc::eventlogging::database
+
+ # custom data sanitization setup to apply the Analytics
+ # data retention policies
+ include ::profile::mariadb::misc::eventlogging::sanitization
+
system::role { 'role::mariadb::misc::eventlogging::master':
description => 'Eventlogging Master datastore',
}
diff --git a/modules/role/manifests/mariadb/misc/eventlogging/replica.pp
b/modules/role/manifests/mariadb/misc/eventlogging/replica.pp
index fca537a..23d26c5 100644
--- a/modules/role/manifests/mariadb/misc/eventlogging/replica.pp
+++ b/modules/role/manifests/mariadb/misc/eventlogging/replica.pp
@@ -8,6 +8,9 @@
include ::profile::mariadb::misc::eventlogging::database
# custom manual replication setup
include ::profile::mariadb::misc::eventlogging::replication
+ # custom data sanitization setup to apply the Analytics
+ # data retention policies
+ include ::profile::mariadb::misc::eventlogging::sanitization
system::role { 'role::mariadb::misc::eventlogging::replica':
description => 'Eventlogging Datastore Custom Replica',
--
To view, visit https://gerrit.wikimedia.org/r/398869
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1f4d273e1e1d537fdbd978cb69ea804ee9ea8802
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
