Alexandros Kosiaris has submitted this change and it was merged.
Change subject: puppetmaster: servermon report handler
......................................................................
puppetmaster: servermon report handler
Use(and abuse) puppet report handlers to get servermon database updated
Reuse the settings from puppet to get the database since it is designed
to be the same anyway. For this, also change the conditionals so we
always got that in puppet.conf on puppetmasters.
Also stop sending reports to specific hosts
Not really benchmarked
Change-Id: Id18971df1586883abeae8915b39a61cd43dfd2b4
---
M .rubocop_todo.yml
A modules/puppetmaster/lib/puppet/reports/servermon.rb
M modules/puppetmaster/templates/web-frontend.conf.erb
M modules/role/manifests/puppetmaster/common.pp
4 files changed, 129 insertions(+), 6 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, but someone else must approve
Alexandros Kosiaris: Verified; Looks good to me, approved
diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml
index 42e8e96..d96ce2c 100644
--- a/.rubocop_todo.yml
+++ b/.rubocop_todo.yml
@@ -24,6 +24,7 @@
Lint/RescueException:
Exclude:
- 'modules/puppetmaster/lib/puppet/reports/logstash.rb'
+ - 'modules/puppetmaster/lib/puppet/reports/servermon.rb'
# Offense count: 2
# Cop supports --auto-correct.
@@ -52,6 +53,7 @@
Style/AsciiComments:
Exclude:
- 'modules/puppetmaster/lib/puppet/reports/logstash.rb'
+ - 'modules/puppetmaster/lib/puppet/reports/servermon.rb'
# Offense count: 58
# Cop supports --auto-correct.
@@ -267,6 +269,7 @@
Exclude:
- 'modules/puppet_statsd/lib/puppet/reports/statsd.rb'
- 'modules/puppetmaster/lib/puppet/reports/logstash.rb'
+ - 'modules/puppetmaster/lib/puppet/reports/servermon.rb'
- 'modules/wmflib/lib/hiera/backend/proxy_backend.rb'
# Offense count: 23
diff --git a/modules/puppetmaster/lib/puppet/reports/servermon.rb
b/modules/puppetmaster/lib/puppet/reports/servermon.rb
new file mode 100644
index 0000000..8ccb994
--- /dev/null
+++ b/modules/puppetmaster/lib/puppet/reports/servermon.rb
@@ -0,0 +1,117 @@
+# Puppet reporter to update servermon database
+#
+# Based on
https://serverfault.com/questions/515455/how-do-i-make-puppet-post-facts-after-run
+#
+# This report handler uses the puppet config from puppet.conf to get the
+# active_record database settings and connect to it and issue commands to
update
+# it in case the used storeconfigs_backend is puppetdb. This is useful for the
+# servermon software (https://github.com/servermon/servermon), at least until
it
+# is updated to support puppetdb directly. The manual SQL command approach was
+# chosen instead of reusing the active record approach supported by the rails
+# framework of puppet, since it would not be initialized when
+# storeconfig_backend = puppetdb
+# NOTE: This is probably quite inefficient, but let's evaluate first
+#
+# Copyright © 2016 Alexandros Kosiaris and Wikimedia Foundation.
+# License http://www.apache.org/licenses/LICENSE-2.0
+
+require 'puppet'
+require 'mysql'
+
+Puppet::Reports.register_report(:servermon) do
+ desc 'Update facts of a servermon database'
+
+ def process
+ # Get our users from the configuration
+ dbserver = Puppet[:dbserver]
+ dbuser = Puppet[:dbuser]
+ dbpassword = Puppet[:dbpassword]
+ log_level = Puppet[:log_level]
+ begin
+ con = Mysql.new dbserver, dbuser, dbpassword, 'puppet'
+ # First we try to update the host, if it fails, insert it
+ update_host = "UPDATE hosts SET \
+ environment = '#{self.environment}', \
+ updated_at = '#{self.time}', \
+ last_compile = '#{self.time}' \
+ WHERE name='#{self.host}'"
+ con.query(update_host)
+ if con.affected_rows == 0
+ insert_host = "INSERT INTO hosts(
+ name, environment, last_compile, updated_at, created_at) \
+ VALUES('#{self.host}', #{self.environment}', '#{self.time}',
'#{self.time}', #{self.time}')"
+ con.query(insert_host)
+ end
+ # Now we know the host is there, get the id
+ query = "SELECT id from hosts \
+ WHERE name='#{self.host}'"
+ rs = con.query(query)
+ host_id = rs.fetch_row[0]
+ if log_level == 'debug'
+ puts "Got host: #{self.host} with id: #{host_id}"
+ end
+
+ # if facts file found, read it and update facts for host:
+ if File.exists?("#{Puppet[:vardir]}/yaml/facts/#{self.host}.yaml")
+ node_facts =
YAML.load_file("#{Puppet[:vardir]}/yaml/facts/#{self.host}.yaml")
+ # We got a Ruby object, get the values attributes and walk it
+ node_facts.values.each do |key, value|
+ # First update the fact_names table, if it fails, insert
the
+ # fact name
+ update_fact_name = "UPDATE fact_names SET \
+ updated_at = '#{self.time}' \
+ WHERE name='#{key}'"
+ if log_level == 'debug'
+ puts update_fact_name
+ end
+ con.query(update_fact_name)
+ if con.affected_rows == 0
+ insert_fact_name = "INSERT INTO fact_names(name,
updated_at, created_at) \
+ VALUES('#{key}', '#{self.time}', '#{self.time}'"
+ if log_level == 'debug'
+ puts insert_fact_name
+ end
+ con.query(insert_fact_name)
+ end
+ # We now can be sure the fact exists if the code reaches
+ # this point, so just get the id
+ query = "SELECT id from fact_names \
+ WHERE name='#{key}'"
+ rs = con.query(query)
+ fact_id = rs.fetch_row[0]
+ if log_level == 'debug'
+ puts "Got fact: #{key} with id: #{fact_id}"
+ end
+ # Now try to update the fact_value, it is fails, insert it
+ update_fact_value = "UPDATE fact_values SET \
+ updated_at = '#{self.time}', \
+ value = '#{value}' \
+ WHERE fact_name_id=#{fact_id} AND host_id=#{host_id}"
+ if log_level == 'debug'
+ puts(update_fact_value)
+ end
+ con.query(update_fact_value)
+ # rubocop:disable Style/Next
+ if con.affected_rows == 0
+ insert_fact_value = "INSERT INTO fact_values( \
+ value,fact_name_id,host_id,updated_at,created_at) \
+ VALUES('#{value}', #{fact_id}, #{host_id},
'#{self.time}', '#{self.time}')"
+ if log_level == 'debug'
+ puts(insert_fact_value)
+ end
+ con.query(insert_fact_value)
+ end
+ # rubocop:enable Style/Next
+ end
+ end
+ rescue Mysql::Error => e
+ puts "Mysql error: #{e.errno}, #{e.error}"
+ puts e.errno
+ puts e.error
+ rescue Exception => e
+ puts "Exception caught: #{e.errno}, #{e.error}"
+ ensure
+ con.close if con
+ end
+ end
+end
diff --git a/modules/puppetmaster/templates/web-frontend.conf.erb
b/modules/puppetmaster/templates/web-frontend.conf.erb
index 5a53f63..728c99e 100644
--- a/modules/puppetmaster/templates/web-frontend.conf.erb
+++ b/modules/puppetmaster/templates/web-frontend.conf.erb
@@ -25,11 +25,9 @@
SSLProxyEngine on
# Tries to avoid the "error reading status line from server" error
- SetEnv proxy-initial-not-pooled 1
+ SetEnv proxy-initial-not-pooled 1
# Certificates requests should only be in the master.
ProxyPassMatch ^/([^/]+/certificate.*)$ https://<%= @master %>:8141
- # Reports should only be in one place. Choosing the master by default for
simplicity
- ProxyPassMatch ^/([^/]+/report/.*)$ https://<%= @master %>:8141
# File buckets should only be in one place. Choosing the frontend by
default for simplicity
ProxyPassMatch ^/([^/]+/file_bucket_file/md5/.*)$ https://<%= @fqdn %>:8141
# Volatile should only be in one place. Choosing the master by default for
simplicity
diff --git a/modules/role/manifests/puppetmaster/common.pp
b/modules/role/manifests/puppetmaster/common.pp
index c0509f5..d52aad3 100644
--- a/modules/role/manifests/puppetmaster/common.pp
+++ b/modules/role/manifests/puppetmaster/common.pp
@@ -3,6 +3,11 @@
$activerecord_config = {
'storeconfigs' => true,
'thin_storeconfigs' => true,
+ }
+ # Note: We are going to need this anyway regardless of
+ # puppetdb/active_record use for the configuration of servermon report
+ # handler
+ $active_record_db = {
'dbadapter' => 'mysql',
'dbuser' => 'puppet',
'dbpassword' =>
$passwords::puppet::database::puppet_production_db_pass,
@@ -13,7 +18,7 @@
$puppetdb_config = {
storeconfigs => true,
storeconfigs_backend => 'puppetdb',
- reports => 'store,puppetdb',
+ reports => 'puppetdb,servermon',
}
$use_puppetdb = hiera('puppetmaster::config::use_puppetdb', false)
@@ -23,9 +28,9 @@
class { 'puppetmaster::puppetdb::client':
host => $puppetdb_host,
}
- $config = merge($base_config, $puppetdb_config)
+ $config = merge($base_config, $puppetdb_config, $active_record_db)
}
else {
- $config = merge($base_config, $activerecord_config)
+ $config = merge($base_config, $activerecord_config, $active_record_db)
}
}
--
To view, visit https://gerrit.wikimedia.org/r/311738
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id18971df1586883abeae8915b39a61cd43dfd2b4
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris <akosia...@wikimedia.org>
Gerrit-Reviewer: Alexandros Kosiaris <akosia...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
