Giuseppe Lavagetto has uploaded a new change for review.
https://gerrit.wikimedia.org/r/231500
Change subject: puppet_compiler: refresh module/role
......................................................................
puppet_compiler: refresh module/role
Since we rebuilt the puppet compiler, we refresh the puppet module as
well.
Bug: T96802
Change-Id: Ie3fc79eec44667f3bee4f5cbb9130f3826e94964
---
M manifests/role/puppet_compiler.pp
D modules/puppet_compiler/manifests/bundle.pp
D modules/puppet_compiler/manifests/differ.pp
M modules/puppet_compiler/manifests/init.pp
M modules/puppet_compiler/manifests/packages.pp
A modules/puppet_compiler/manifests/setup.pp
A modules/puppet_compiler/manifests/web.pp
D modules/puppet_compiler/templates/mysql_queries.erb
M modules/puppet_compiler/templates/nginx_site.erb
A modules/puppet_compiler/templates/puppet-compiler.conf.erb
D modules/puppet_compiler/templates/run_wrapper.erb
11 files changed, 126 insertions(+), 198 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/00/231500/1
diff --git a/manifests/role/puppet_compiler.pp
b/manifests/role/puppet_compiler.pp
index 7e240a8..238b430 100644
--- a/manifests/role/puppet_compiler.pp
+++ b/manifests/role/puppet_compiler.pp
@@ -16,48 +16,6 @@
default : { fail("Realm ${::realm} NOT supported by this role.") }
}
- class {'::puppet_compiler':
- ensure => 'present',
- version => '0.3',
- user => 'jenkins-deploy',
- }
-
- file { '/mnt/puppet-compiler-output':
- ensure => directory,
- owner => 'jenkins-deploy',
- require => $::role::ci::slave::labs::common::slash_mnt_require
- }
-
- mount { "${::puppet_compiler::program_dir}/output":
- ensure => mounted,
- fstype => 'auto',
- options => 'bind',
- device => '/mnt/puppet-compiler-output'
- }
-
- file { '/mnt/puppet-compiler-external':
- ensure => directory,
- owner => 'jenkins-deploy',
- require => $::role::ci::slave::labs::common::slash_mnt_require
- }
-
- mount { "${::puppet_compiler::program_dir}/external/change":
- ensure => mounted,
- fstype => 'auto',
- options => 'bind',
- device => '/mnt/puppet-compiler-external'
- }
-
- cron { 'check_compiler_stale_repositories':
- command => '/usr/bin/find /mnt/puppet-compiler-external -mindepth 1
-maxdepth 1 -ctime +1 -type d | /usr/bin/xargs rm -rf',
- user => 'jenkins-deploy',
- weekday => 0,
- }
-
- cron { 'check_compiler_stale_output':
- command => '/usr/bin/find /mnt/puppet-compiler-output -mindepth 1
-maxdepth 1 -ctime +10 -type d | /usr/bin/xargs rm -rf',
- user => 'jenkins-deploy',
- weekday => 0,
- }
+ include ::puppet_compiler
}
diff --git a/modules/puppet_compiler/manifests/bundle.pp
b/modules/puppet_compiler/manifests/bundle.pp
deleted file mode 100644
index f724818..0000000
--- a/modules/puppet_compiler/manifests/bundle.pp
+++ /dev/null
@@ -1,12 +0,0 @@
-# Installs the deployment bundle needed by puppet
-define puppet_compiler::bundle(
- $program_dir = $puppet_compiler::program_dir
-) {
- $installer = "${program_dir}/shell/installer"
- exec { "install_puppet_bundle_${title}":
- command => "${installer} ${title}",
- user => $puppet_compiler::user,
- creates => "${program_dir}/shell/env_puppet_${title}/vendor",
- require => Git::Install['operations/software'],
- }
-}
diff --git a/modules/puppet_compiler/manifests/differ.pp
b/modules/puppet_compiler/manifests/differ.pp
deleted file mode 100644
index 2b07c74..0000000
--- a/modules/puppet_compiler/manifests/differ.pp
+++ /dev/null
@@ -1,12 +0,0 @@
-class puppet_compiler::differ(
- $envdir = "${::puppet_compiler::program_dir}/shell/env_puppet_3",
- $modulepath = "${::puppet_compiler::puppetdir}/modules",
- $user = $::puppet_compiler::user
-) {
- exec { 'Install catalog diff module':
- command => "/usr/bin/bundle exec puppet module install
zack-catalog_diff --modulepath=${modulepath}",
- cwd => $envdir,
- user => $user,
- creates => "${modulepath}/catalog_diff",
- }
-}
diff --git a/modules/puppet_compiler/manifests/init.pp
b/modules/puppet_compiler/manifests/init.pp
index 1b0e89f..d37c573 100644
--- a/modules/puppet_compiler/manifests/init.pp
+++ b/modules/puppet_compiler/manifests/init.pp
@@ -1,105 +1,81 @@
# Installs the puppet compiler and all the other software we need.
class puppet_compiler(
- $version = '0.1.0',
- $rootdir = '/opt/wmf',
+ $version = '0.0.1',
+ $workdir = '/mnt/jenkins-workspace/puppet-compiler',
+ $libdir = '/var/lib/catalog-differ',
$ensure = 'present',
- $user = 'www-data'
- ) {
+ $user = 'jenkins-deploy',
+ $homedir = '/mnt/home/jenkins-deploy',
+) {
- include puppet_compiler::packages
+ require puppet_compiler::packages
- $install_dir = "${rootdir}/software"
- $program_dir = "${install_dir}/compare-puppet-catalogs"
- $puppetdir = "${program_dir}/external/puppet"
+ $vardir = "${libdir}/puppet"
- nginx::site {'puppet-compiler':
- ensure => $ensure,
- content => template('puppet_compiler/nginx_site.erb'),
+ file { [$libdir, $vardir]:
+ ensure => ensure_directory($ensure),
+ owner => $user,
+ mode => '0755',
}
- file_line { 'modify_nginx_magic_types':
- path => '/etc/nginx/mime.types',
- line => "\ttext/plain\t\t\t\ttxt pson warnings out diff formatted;",
- match => "\ttext/plain\t\t\t\ttxt",
- require => Nginx::Site['puppet-compiler'],
- notify => Service['nginx']
- }
-
- # This wrapper defines the env variables for running.
- file { 'run_wrapper':
- ensure => $ensure,
- path => '/usr/local/bin/puppet-compiler',
- content => template('puppet_compiler/run_wrapper.erb'),
- mode => '0555'
- }
-
-
- if $ensure != 'present' {
- file { 'root_dir':
- ensure => absent,
- path => $rootdir,
- owner => $user,
- recurse => true,
- force => true,
- }
- } else {
- file { 'root_dir':
- ensure => directory,
- path => $rootdir,
- owner => $user,
- before => Git::Install['operations/software'],
- }
-
- git::install { 'operations/software':
- ensure => present,
- directory => $install_dir,
- owner => $user,
- git_tag => "compare-puppet-catalogs-${version}",
- require => Nginx::Site['puppet-compiler'],
- }
-
- exec { 'install_puppet_compare_requirements':
- command => '/usr/bin/pip install requests simplediff',
- user => 'root',
- require => Git::Install['operations/software'],
- }
-
- puppet_compiler::bundle {['3']: }
-
- # Now install the puppet repo
-
- exec { 'install_puppet_repositories':
- command => "${program_dir}/shell/helper install",
+ if $ensure == 'present' {
+ class { 'puppet_compiler::setup':
user => $user,
- creates => $puppetdir,
- require => Git::Install['operations/software'],
- notify => Class['puppet_compiler::differ']
+ vardir => $vardir,
+ homedir => $homedir,
}
+ }
- class { '::puppetmaster::scripts':
- require => Exec['install_puppet_repositories']
- }
+ # We don't really need sshknowngen and naggen, link them to /bin/true
+ file { ['/usr/local/bin/sshknowngen', '/usr/local/bin/naggen2']:
+ ensure => ensure_link($ensure),
+ target => '/bin/true',
+ }
- class { 'puppet_compiler::differ':
- require =>
[Exec['install_puppet_repositories'],Class['::puppetmaster::scripts']]
- }
+ include ::puppet_compiler::web
- file {[ "${program_dir}/output",
- "${program_dir}/output/html",
- "${program_dir}/output/diff",
- "${program_dir}/output/compiled",
- ]:
- ensure => directory,
- owner => $user,
- mode => '0775',
- require => Exec['install_puppet_repositories']
- }
+ ## Git cloning
- $mysql_query = template('puppet_compiler/mysql_queries.erb')
- exec { 'mysql queries':
- command => "/usr/bin/mysql -NBe ${mysql_query}",
- unless => "/usr/bin/mysql puppet -NBe 'SELECT 1' ",
- require => Package['mysql-server']
- }
+ # Git clone of the puppet repo
+ git::clone { 'operations/puppet':
+ ensure => $ensure,
+ directory => "${libdir}/puppet",
+ owner => $user,
+ mode => '0755',
+ require => File[$libdir],
+ }
+
+ # Git clone labs/private
+ git::clone { 'labs/puppet':
+ ensure => $ensure,
+ directory => "${libdir}/puppet",
+ owner => $user,
+ mode => '0755',
+ require => File[$libdir],
+ }
+
+ $compiler_dir = "${libdir}/compiler"
+ # Git clone the puppet compiler, install it
+ git::install { 'operations/software/puppet-compiler':
+ ensure => $ensure,
+ git_tag => $version,
+ directory => $compiler_dir,
+ owner => $user,
+ notify => Exec['install compiler']
+ }
+
+ # Install the compiler
+ exec { 'install compiler':
+ command => '/usr/bin/python setup.py install',
+ owner => $user,
+ cwd => $compiler_dir,
+ refreshonly => true,
+ }
+
+ # configuration file
+ file { '/etc/puppet-compiler.conf':
+ ensure => $ensure,
+ owner => $user,
+ content => template('puppet_compiler/puppet-compiler.conf.erb')
}
}
diff --git a/modules/puppet_compiler/manifests/packages.pp
b/modules/puppet_compiler/manifests/packages.pp
index 56a6088..2620da4 100644
--- a/modules/puppet_compiler/manifests/packages.pp
+++ b/modules/puppet_compiler/manifests/packages.pp
@@ -1,16 +1,8 @@
-class puppet_compiler::packages($ensure = $puppet_compiler::ensure) {
- include stdlib
- $list = [
- 'curl',
- 'python-dev',
- 'rubygems',
- 'ruby-bundler',
- 'ruby1.8-dev',
- 'mysql-server',
- 'mysql-client',
- 'ruby-mysql',
- 'ruby-bcrypt',
- 'nginx'
- ]
- ensure_packages([$list])
+# === Class puppet_compiler::packages
+#
+# Installs all the needed packages
+class puppet_compiler::packages {
+
+ require_package 'python-yaml', 'python-requests', 'python-jinja2', 'nginx'
+
}
diff --git a/modules/puppet_compiler/manifests/setup.pp
b/modules/puppet_compiler/manifests/setup.pp
new file mode 100644
index 0000000..bdabdac
--- /dev/null
+++ b/modules/puppet_compiler/manifests/setup.pp
@@ -0,0 +1,29 @@
+# === Class puppet_compiler::setup
+#
+# Sets up the puppet environment
+
+class puppet_compiler::setup($vardir, $user, $homedir) {
+ # Install the puppet var dir files
+ exec { 'create puppet directories':
+ command => "/usr/bin/puppet master --compile test --vardir ${vardir}",
+ creates => "${vardir}/yaml",
+ user => $user,
+ require => File[$vardir]
+ }
+
+ # Create the ssl directory, and the puppet ca
+ exec { 'create puppet ssl dir':
+ command => "/usr/bin/puppet cert --ssldir ${vardir}/ssl --vardir
${vardir} list -a",
+ creates => "${vardir}/ssl/ca/inventory.txt",
+ user => $user,
+ require => Exec['create puppet directories'],
+ }
+
+ # Install the puppet catalog diff face
+ exect { 'install puppet catalog diff':
+ command => "/usr/bin/puppet module install zack-catalog_diff",
+ creates => "${homedir}/.puppet/modules/catalog_diff",
+ user => $user,
+ }
+
+}
diff --git a/modules/puppet_compiler/manifests/web.pp
b/modules/puppet_compiler/manifests/web.pp
new file mode 100644
index 0000000..94a1770
--- /dev/null
+++ b/modules/puppet_compiler/manifests/web.pp
@@ -0,0 +1,16 @@
+# === Class puppet_compiler::web
+#
+class puppet_compiler::web($ensure='present') {
+ nginx::site {'puppet-compiler':
+ ensure => $ensure,
+ content => template('puppet_compiler/nginx_site.erb'),
+ }
+
+ file_line { 'modify_nginx_magic_types':
+ path => '/etc/nginx/mime.types',
+ line => "\ttext/plain\t\t\t\ttxt pson warnings out diff formatted;",
+ match => "\ttext/plain\t\t\t\ttxt",
+ require => Nginx::Site['puppet-compiler'],
+ notify => Service['nginx']
+ }
+}
diff --git a/modules/puppet_compiler/templates/mysql_queries.erb
b/modules/puppet_compiler/templates/mysql_queries.erb
deleted file mode 100644
index c01d63b..0000000
--- a/modules/puppet_compiler/templates/mysql_queries.erb
+++ /dev/null
@@ -1 +0,0 @@
-"CREATE DATABASE puppet; GRANT ALL PRIVILEGES ON puppet.* TO
'puppet'@'localhost' IDENTIFIED BY '<%=
scope.lookupvar('passwords::puppet_compiler::mysql_password') -%>';"
diff --git a/modules/puppet_compiler/templates/nginx_site.erb
b/modules/puppet_compiler/templates/nginx_site.erb
index e93dc66..9cd5ece 100644
--- a/modules/puppet_compiler/templates/nginx_site.erb
+++ b/modules/puppet_compiler/templates/nginx_site.erb
@@ -2,7 +2,7 @@
listen 80;
server_name localhost puppet-compiler.wmflabs.org <%= @fqdn -%>;
- root <%= scope.lookupvar('::puppet_compiler::program_dir') -%>/output;
+ root <%= scope.lookupvar('::puppet_compiler::workdir') -%>/output;
index index.html index.htm;
location / {
diff --git a/modules/puppet_compiler/templates/puppet-compiler.conf.erb
b/modules/puppet_compiler/templates/puppet-compiler.conf.erb
new file mode 100644
index 0000000..0974d0b
--- /dev/null
+++ b/modules/puppet_compiler/templates/puppet-compiler.conf.erb
@@ -0,0 +1,7 @@
+# This file is managed by puppet, so heads up: modify it manually and you lose
+# everything eventually.
+base: "<%= @workdir %>"
+puppet_src: "<%= @libdir %>/production"
+puppet_private: "<%= @libdir %>/private"
+http_url: "https://puppet-compiler.wmflabs.org";
+puppet_var: "<%= @vardir %>"
diff --git a/modules/puppet_compiler/templates/run_wrapper.erb
b/modules/puppet_compiler/templates/run_wrapper.erb
deleted file mode 100755
index a426c7f..0000000
--- a/modules/puppet_compiler/templates/run_wrapper.erb
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-PUPPET_COMPILER_BASEDIR="<%= @program_dir -%>"
-PUPPET_COMPILER_MYSQL_PWD="<%=
scope.lookupvar('passwords::puppet_compiler::mysql_password') -%>"
-export PUPPET_COMPILER_BASEDIR PUPPET_COMPILER_MYSQL_PWD
-
-# Now support jenkins. Jenkins is not very flexible so we need to support it :)
-if [ $1 == "--run-with-envvars" ]; then
- OPTS=""
- if [ "x${CHANGE}" != "x" ]; then
- OPTS="${OPTS} --change ${CHANGE}"
- if [ "x${TRANSITION}" != "x" ]; then
- OPTS="${OPTS} --transition"
- fi;
- fi;
- if [ "x${NODES}" != "x" ]; then
- OPTS="${OPTS} --nodes ${NODES}"
- fi;
- if [ "x${NUM_THREADS}" != "x" ]; then
- OPTS="${OPTS} --numthreads ${NUM_THREADS}"
- fi;
-else
- OPTS=$@
-fi;
-
-${PUPPET_COMPILER_BASEDIR}/comparator $OPTS
--
To view, visit https://gerrit.wikimedia.org/r/231500
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie3fc79eec44667f3bee4f5cbb9130f3826e94964
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
