Dzahn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/269603
Change subject: parsoid: one file per role, move to module/role
......................................................................
parsoid: one file per role, move to module/role
Split the single large role file into one file
per class per style guide for easier readibality.
Move them into the module/role/ structure which for
proper autoload layout and which also makes lint happy.
This should not result in any change.
Change-Id: I379220a4f2c6d41874b75b253d22f2c08100f762
---
D manifests/role/parsoid.pp
A modules/role/parsoid/beta.pp
A modules/role/parsoid/common.pp
A modules/role/parsoid/production.pp
A modules/role/parsoid/testing.pp
5 files changed, 351 insertions(+), 351 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/03/269603/1
diff --git a/manifests/role/parsoid.pp b/manifests/role/parsoid.pp
deleted file mode 100644
index 16fb67c..0000000
--- a/manifests/role/parsoid.pp
+++ /dev/null
@@ -1,351 +0,0 @@
-# vim: set ts=4 et sw=4:
-class role::parsoid::common {
- package { [
- 'nodejs',
- 'npm',
- 'build-essential',
- ]: ensure => present,
- }
-
- file { '/var/lib/parsoid':
- ensure => directory,
- owner => parsoid,
- group => wikidev,
- mode => '2775',
- }
-
- file { '/usr/bin/parsoid':
- ensure => present,
- owner => root,
- group => root,
- mode => '0555',
- source => 'puppet:///modules/parsoid/parsoid',
- }
-
- ferm::service { 'parsoid':
- proto => 'tcp',
- port => '8000',
- }
-}
-
-class role::parsoid::production {
- system::role { 'role::parsoid::production':
- description => 'Parsoid server'
- }
-
- include role::parsoid::common
- include standard
- include lvs::realserver
- include base::firewall
-
- package { 'parsoid/deploy':
- provider => 'trebuchet',
- }
-
- group { 'parsoid':
- ensure => present,
- name => 'parsoid',
- system => true,
- }
-
- user { 'parsoid':
- gid => 'parsoid',
- home => '/var/lib/parsoid',
- managehome => true,
- system => true,
- }
-
- file { '/var/lib/parsoid/deploy':
- ensure => link,
- target => '/srv/deployment/parsoid/deploy',
- }
-
- file { '/etc/init/parsoid.conf':
- ensure => present,
- owner => root,
- group => root,
- mode => '0444',
- source => 'puppet:///modules/parsoid/parsoid.upstart',
- }
-
- file { '/var/log/parsoid':
- ensure => directory,
- owner => parsoid,
- group => parsoid,
- mode => '0775',
- }
-
- $parsoid_log_file = '/var/log/parsoid/parsoid.log'
- #TODO: Should we explicitly set this to
'/srv/deployment/parsoid/deploy/node_modules'
- #just like beta labs
- $parsoid_node_path = '/var/lib/parsoid/deploy/node_modules'
- $parsoid_settings_file =
'/srv/deployment/parsoid/deploy/conf/wmf/localsettings.js'
- $parsoid_base_path = '/var/lib/parsoid/deploy/src'
-
- #TODO: Duplication of code from beta class, deduplicate somehow
- file { '/etc/default/parsoid':
- ensure => present,
- owner => root,
- group => root,
- mode => '0444',
- content => template('parsoid/parsoid.default.erb'),
- require => File['/var/log/parsoid'],
- }
-
- file { '/etc/logrotate.d/parsoid':
- ensure => present,
- owner => root,
- group => root,
- mode => '0444',
- content => template('parsoid/parsoid.logrotate.erb'),
- }
-
- cron { 'parsoid-hourly-logrot':
- ensure => present,
- command => '/usr/sbin/logrotate /etc/logrotate.d/parsoid',
- user => 'root',
- hour => '*',
- minute => '12',
- require => File['/etc/logrotate.d/parsoid'],
- }
-
- service { 'parsoid':
- ensure => running,
- hasstatus => true,
- hasrestart => true,
- provider => 'upstart',
- subscribe => [
- File['/etc/default/parsoid'],
- File['/etc/init/parsoid.conf'],
- ],
- require => Package['parsoid/deploy'],
- }
-
- monitoring::service { 'parsoid':
- description => 'Parsoid',
- check_command => 'check_http_on_port!8000',
- }
- # until logging is handled differently, rt 6851
- nrpe::monitor_service { 'parsoid_disk_space':
- description => 'parsoid disk space',
- nrpe_command => '/usr/lib/nagios/plugins/check_disk -w 40% -c 3% -l
-e',
- critical => true,
- }
-
- # Monitor TCP Connection States
- diamond::collector { 'TcpConnStates':
- source => 'puppet:///modules/diamond/collector/tcpconnstates.py',
- }
-}
-
-class role::parsoid::beta {
- system::role { 'role::parsoid::beta':
- description => 'Parsoid server (on beta)'
- }
-
- include role::parsoid::common
-
- sudo::user { 'jenkins-deploy': privileges => [
- # Need to allow jenkins-deploy to reload parsoid
- # Since the "root" user is local, we cant add the sudo policy in
- # OpenStack manager interface at wikitech
- # TODO: adjust for upstart!
- 'ALL = (root) NOPASSWD:/etc/init.d/parsoid',
- ] }
-
- file { '/var/lib/parsoid/Parsoid':
- ensure => link,
- target => '/srv/mediawiki/php-master/extensions/Parsoid',
- owner => parsoid,
- group => wikidev,
- mode => '2775',
- }
-
- # Jenkins copy repositories and config under /srv/deployment
- file { '/srv/deployment':
- ensure => directory,
- owner => root,
- group => root,
- mode => '0755',
- }
- file { '/srv/deployment/parsoid':
- ensure => directory,
- owner => jenkins-deploy,
- group => wikidev,
- mode => '0755',
- }
-
- # Delete the puppet copy of this file
- file { '/srv/deployment/parsoid/localsettings.js':
- ensure => absent,
- }
-
- # beta uses upstart:
- # FIXME: move users to upstart
- file { '/etc/init.d/parsoid':
- ensure => 'link',
- target => '/lib/init/upstart-job',
- }
- file { '/etc/init/parsoid.conf':
- ensure => present,
- owner => root,
- group => root,
- mode => '0444',
- source => 'puppet:///modules/parsoid/parsoid.upstart',
- }
-
- $parsoid_log_file = '/data/project/parsoid/parsoid.log'
- # Make sure the directory exists on beta
- file { '/data/project/parsoid':
- ensure => directory,
- owner => parsoid,
- group => parsoid,
- mode => '0775',
- }
-
- # For beta, override NODE_PATH
- # The packages are installed by Jenkins via npm install using pinned
- # versions in npm-shrinkwrap.json
- $parsoid_node_path = '/srv/deployment/parsoid/parsoid/node_modules'
-
- # Override PARSOID_SETTINGS_FILE
- # The setting file is in parsoid/deploy.git
- $parsoid_settings_file =
'/srv/deployment/parsoid/deploy/conf/wmf/betalabs.localsettings.js'
-
- # Checkout of mediawiki/services/parsoid
- $parsoid_base_path = '/srv/deployment/parsoid/parsoid'
-
- file { '/etc/default/parsoid':
- ensure => present,
- owner => root,
- group => root,
- mode => '0444',
- content => template('parsoid/parsoid.default.erb'),
- require => File['/data/project/parsoid'],
- }
-
- file { '/etc/logrotate.d/parsoid':
- ensure => present,
- owner => root,
- group => root,
- mode => '0444',
- content => template('parsoid/parsoid.logrotate.erb'),
- }
-
- service { 'parsoid':
- ensure => running,
- hasstatus => true,
- hasrestart => true,
- provider => 'upstart',
- subscribe => [
- File['/etc/default/parsoid'],
- File['/etc/init/parsoid.conf'],
- ],
- }
-
- # Instance got to be a Jenkins slave so we can update Parsoid whenever a
- # change is made on mediawiki/services/parsoid repository
- include role::ci::slave::labs::common
- # Also need the slave scripts for multi-git.sh
- include contint::slave_scripts
-}
-
-# This role is used by testing services
-# Ex: Parsoid roundtrip testing, Parsoid & PHP parser visual diff testing
-class role::parsoid::testing {
- system::role { 'role::parsoid::testing':
- description => 'Parsoid server (rt-testing, visual-diffing, etc.)'
- }
-
- # Some visual diff node modules reference 'node' which this package
provides
- require_package('nodejs-legacy')
-
- include role::parsoid::common
-
- group { 'parsoid':
- ensure => present,
- name => 'parsoid',
- system => true,
- }
-
- user { 'parsoid':
- gid => 'parsoid',
- home => '/var/lib/parsoid',
- managehome => true,
- system => true,
- }
-
- # We clone the git repo and let testing services
- # update / modify the repo as appropriate
- # (via scripts, manually, however).
- # FIXME: Should we move this to /srv/parsoid ?
- # I am picking /usr/lib to minimize changes to
- # ruthenium setup.
- git::clone { 'mediawiki/services/parsoid/deploy':
- owner => 'root',
- group => 'wikidev',
- recurse_submodules => true,
- directory => '/usr/lib/parsoid',
- before => Service['parsoid'],
- }
-
- file { '/lib/systemd/system/parsoid.service':
- source => 'puppet:///modules/parsoid/parsoid_testing.systemd.service',
- owner => 'root',
- group => 'root',
- mode => '0444',
- before => Service['parsoid'],
- }
-
- file { '/var/log/parsoid':
- ensure => directory,
- owner => 'parsoid',
- group => 'parsoid',
- mode => '0775',
- before => Service['parsoid'],
- }
-
- file { '/usr/local/bin/update_parsoid.sh':
- source =>
'puppet:///modules/parsoid/parsoid_testing.update_parsoid.sh',
- owner => 'root',
- group => 'root',
- mode => '0555',
- }
-
- # Use this parsoid instance for parsoid rt-testing
- file { '/usr/lib/parsoid/src/localsettings.js':
- source =>
'puppet:///modules/testreduce/parsoid-rt-client.rttest.localsettings.js',
- owner => 'root',
- group => 'wikidev',
- mode => '0444',
- before => Service['parsoid'],
- }
-
- service { 'parsoid':
- hasstatus => true,
- hasrestart => true,
- subscribe => [
- File['/lib/systemd/system/parsoid.service'],
- ],
- }
-
- # mysql client and configuration to provide command line access to
- # parsoid testing database
- include ::passwords::testreduce::mysql
- $parsoid_cli_password = $passwords::testreduce::mysql::mysql_client_pass
- $parsoid_test_db_host = 'm5-master.eqiad.wmnet'
-
- package { [
- 'mysql-client',
- ]: ensure => present,
- }
-
- file { '/etc/my.cnf':
- content => template('mariadb/parsoid_testing.my.cnf'),
- owner => 'root',
- group => 'parsoid-test-roots',
- mode => '0440',
- }
-
-
-}
diff --git a/modules/role/parsoid/beta.pp b/modules/role/parsoid/beta.pp
new file mode 100644
index 0000000..24a0667
--- /dev/null
+++ b/modules/role/parsoid/beta.pp
@@ -0,0 +1,112 @@
+class role::parsoid::beta {
+ system::role { 'role::parsoid::beta':
+ description => 'Parsoid server (on beta)'
+ }
+
+ include role::parsoid::common
+
+ sudo::user { 'jenkins-deploy': privileges => [
+ # Need to allow jenkins-deploy to reload parsoid
+ # Since the "root" user is local, we cant add the sudo policy in
+ # OpenStack manager interface at wikitech
+ # TODO: adjust for upstart!
+ 'ALL = (root) NOPASSWD:/etc/init.d/parsoid',
+ ] }
+
+ file { '/var/lib/parsoid/Parsoid':
+ ensure => link,
+ target => '/srv/mediawiki/php-master/extensions/Parsoid',
+ owner => parsoid,
+ group => wikidev,
+ mode => '2775',
+ }
+
+ # Jenkins copy repositories and config under /srv/deployment
+ file { '/srv/deployment':
+ ensure => directory,
+ owner => root,
+ group => root,
+ mode => '0755',
+ }
+ file { '/srv/deployment/parsoid':
+ ensure => directory,
+ owner => jenkins-deploy,
+ group => wikidev,
+ mode => '0755',
+ }
+
+ # Delete the puppet copy of this file
+ file { '/srv/deployment/parsoid/localsettings.js':
+ ensure => absent,
+ }
+
+ # beta uses upstart:
+ # FIXME: move users to upstart
+ file { '/etc/init.d/parsoid':
+ ensure => 'link',
+ target => '/lib/init/upstart-job',
+ }
+ file { '/etc/init/parsoid.conf':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ source => 'puppet:///modules/parsoid/parsoid.upstart',
+ }
+
+ $parsoid_log_file = '/data/project/parsoid/parsoid.log'
+ # Make sure the directory exists on beta
+ file { '/data/project/parsoid':
+ ensure => directory,
+ owner => parsoid,
+ group => parsoid,
+ mode => '0775',
+ }
+
+ # For beta, override NODE_PATH
+ # The packages are installed by Jenkins via npm install using pinned
+ # versions in npm-shrinkwrap.json
+ $parsoid_node_path = '/srv/deployment/parsoid/parsoid/node_modules'
+
+ # Override PARSOID_SETTINGS_FILE
+ # The setting file is in parsoid/deploy.git
+ $parsoid_settings_file =
'/srv/deployment/parsoid/deploy/conf/wmf/betalabs.localsettings.js'
+
+ # Checkout of mediawiki/services/parsoid
+ $parsoid_base_path = '/srv/deployment/parsoid/parsoid'
+
+ file { '/etc/default/parsoid':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ content => template('parsoid/parsoid.default.erb'),
+ require => File['/data/project/parsoid'],
+ }
+
+ file { '/etc/logrotate.d/parsoid':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ content => template('parsoid/parsoid.logrotate.erb'),
+ }
+
+ service { 'parsoid':
+ ensure => running,
+ hasstatus => true,
+ hasrestart => true,
+ provider => 'upstart',
+ subscribe => [
+ File['/etc/default/parsoid'],
+ File['/etc/init/parsoid.conf'],
+ ],
+ }
+
+ # Instance got to be a Jenkins slave so we can update Parsoid whenever a
+ # change is made on mediawiki/services/parsoid repository
+ include role::ci::slave::labs::common
+ # Also need the slave scripts for multi-git.sh
+ include contint::slave_scripts
+}
+
diff --git a/modules/role/parsoid/common.pp b/modules/role/parsoid/common.pp
new file mode 100644
index 0000000..591a7d0
--- /dev/null
+++ b/modules/role/parsoid/common.pp
@@ -0,0 +1,30 @@
+# vim: set ts=4 et sw=4:
+class role::parsoid::common {
+ package { [
+ 'nodejs',
+ 'npm',
+ 'build-essential',
+ ]: ensure => present,
+ }
+
+ file { '/var/lib/parsoid':
+ ensure => directory,
+ owner => parsoid,
+ group => wikidev,
+ mode => '2775',
+ }
+
+ file { '/usr/bin/parsoid':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0555',
+ source => 'puppet:///modules/parsoid/parsoid',
+ }
+
+ ferm::service { 'parsoid':
+ proto => 'tcp',
+ port => '8000',
+ }
+}
+
diff --git a/modules/role/parsoid/production.pp
b/modules/role/parsoid/production.pp
new file mode 100644
index 0000000..4ac954b
--- /dev/null
+++ b/modules/role/parsoid/production.pp
@@ -0,0 +1,110 @@
+class role::parsoid::production {
+ system::role { 'role::parsoid::production':
+ description => 'Parsoid server'
+ }
+
+ include role::parsoid::common
+ include standard
+ include lvs::realserver
+ include base::firewall
+
+ package { 'parsoid/deploy':
+ provider => 'trebuchet',
+ }
+
+ group { 'parsoid':
+ ensure => present,
+ name => 'parsoid',
+ system => true,
+ }
+
+ user { 'parsoid':
+ gid => 'parsoid',
+ home => '/var/lib/parsoid',
+ managehome => true,
+ system => true,
+ }
+
+ file { '/var/lib/parsoid/deploy':
+ ensure => link,
+ target => '/srv/deployment/parsoid/deploy',
+ }
+
+ file { '/etc/init/parsoid.conf':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ source => 'puppet:///modules/parsoid/parsoid.upstart',
+ }
+
+ file { '/var/log/parsoid':
+ ensure => directory,
+ owner => parsoid,
+ group => parsoid,
+ mode => '0775',
+ }
+
+ $parsoid_log_file = '/var/log/parsoid/parsoid.log'
+ #TODO: Should we explicitly set this to
'/srv/deployment/parsoid/deploy/node_modules'
+ #just like beta labs
+ $parsoid_node_path = '/var/lib/parsoid/deploy/node_modules'
+ $parsoid_settings_file =
'/srv/deployment/parsoid/deploy/conf/wmf/localsettings.js'
+ $parsoid_base_path = '/var/lib/parsoid/deploy/src'
+
+ #TODO: Duplication of code from beta class, deduplicate somehow
+ file { '/etc/default/parsoid':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ content => template('parsoid/parsoid.default.erb'),
+ require => File['/var/log/parsoid'],
+ }
+
+ file { '/etc/logrotate.d/parsoid':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0444',
+ content => template('parsoid/parsoid.logrotate.erb'),
+ }
+
+ cron { 'parsoid-hourly-logrot':
+ ensure => present,
+ command => '/usr/sbin/logrotate /etc/logrotate.d/parsoid',
+ user => 'root',
+ hour => '*',
+ minute => '12',
+ require => File['/etc/logrotate.d/parsoid'],
+ }
+
+ service { 'parsoid':
+ ensure => running,
+ hasstatus => true,
+ hasrestart => true,
+ provider => 'upstart',
+ subscribe => [
+ File['/etc/default/parsoid'],
+ File['/etc/init/parsoid.conf'],
+ ],
+ require => Package['parsoid/deploy'],
+ }
+
+ monitoring::service { 'parsoid':
+ description => 'Parsoid',
+ check_command => 'check_http_on_port!8000',
+ }
+ # until logging is handled differently, rt 6851
+ nrpe::monitor_service { 'parsoid_disk_space':
+ description => 'parsoid disk space',
+ nrpe_command => '/usr/lib/nagios/plugins/check_disk -w 40% -c 3% -l
-e',
+ critical => true,
+ }
+
+ # Monitor TCP Connection States
+ diamond::collector { 'TcpConnStates':
+ source => 'puppet:///modules/diamond/collector/tcpconnstates.py',
+ }
+}
+
diff --git a/modules/role/parsoid/testing.pp b/modules/role/parsoid/testing.pp
new file mode 100644
index 0000000..26017a0
--- /dev/null
+++ b/modules/role/parsoid/testing.pp
@@ -0,0 +1,99 @@
+# This role is used by testing services
+# Ex: Parsoid roundtrip testing, Parsoid & PHP parser visual diff testing
+class role::parsoid::testing {
+ system::role { 'role::parsoid::testing':
+ description => 'Parsoid server (rt-testing, visual-diffing, etc.)'
+ }
+
+ # Some visual diff node modules reference 'node' which this package
provides
+ require_package('nodejs-legacy')
+
+ include role::parsoid::common
+
+ group { 'parsoid':
+ ensure => present,
+ name => 'parsoid',
+ system => true,
+ }
+
+ user { 'parsoid':
+ gid => 'parsoid',
+ home => '/var/lib/parsoid',
+ managehome => true,
+ system => true,
+ }
+
+ # We clone the git repo and let testing services
+ # update / modify the repo as appropriate
+ # (via scripts, manually, however).
+ # FIXME: Should we move this to /srv/parsoid ?
+ # I am picking /usr/lib to minimize changes to
+ # ruthenium setup.
+ git::clone { 'mediawiki/services/parsoid/deploy':
+ owner => 'root',
+ group => 'wikidev',
+ recurse_submodules => true,
+ directory => '/usr/lib/parsoid',
+ before => Service['parsoid'],
+ }
+
+ file { '/lib/systemd/system/parsoid.service':
+ source => 'puppet:///modules/parsoid/parsoid_testing.systemd.service',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ before => Service['parsoid'],
+ }
+
+ file { '/var/log/parsoid':
+ ensure => directory,
+ owner => 'parsoid',
+ group => 'parsoid',
+ mode => '0775',
+ before => Service['parsoid'],
+ }
+
+ file { '/usr/local/bin/update_parsoid.sh':
+ source =>
'puppet:///modules/parsoid/parsoid_testing.update_parsoid.sh',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ }
+
+ # Use this parsoid instance for parsoid rt-testing
+ file { '/usr/lib/parsoid/src/localsettings.js':
+ source =>
'puppet:///modules/testreduce/parsoid-rt-client.rttest.localsettings.js',
+ owner => 'root',
+ group => 'wikidev',
+ mode => '0444',
+ before => Service['parsoid'],
+ }
+
+ service { 'parsoid':
+ hasstatus => true,
+ hasrestart => true,
+ subscribe => [
+ File['/lib/systemd/system/parsoid.service'],
+ ],
+ }
+
+ # mysql client and configuration to provide command line access to
+ # parsoid testing database
+ include ::passwords::testreduce::mysql
+ $parsoid_cli_password = $passwords::testreduce::mysql::mysql_client_pass
+ $parsoid_test_db_host = 'm5-master.eqiad.wmnet'
+
+ package { [
+ 'mysql-client',
+ ]: ensure => present,
+ }
+
+ file { '/etc/my.cnf':
+ content => template('mariadb/parsoid_testing.my.cnf'),
+ owner => 'root',
+ group => 'parsoid-test-roots',
+ mode => '0440',
+ }
+
+
+}
--
To view, visit https://gerrit.wikimedia.org/r/269603
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I379220a4f2c6d41874b75b253d22f2c08100f762
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Dzahn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
