Jcrespo has uploaded a new change for review.
https://gerrit.wikimedia.org/r/301076
Change subject: Delete coredb_mysql module
......................................................................
Delete coredb_mysql module
After all MySQL 5.5 instances have been upgraded to Mariadb 10,
delete the coredb_mysql module.
It had some interesting things that have not been ported but:
* Snapshots: we do not do those anymore; a delayed slave +
xtrabackup is more useful
* Query digest functionality has been transferred to performance_
schema
* Other things such as extra icinga checks may be unnecesary or
can be ported later from git history
Change-Id: Id43776eb209f8984b501c3f9a4361f1cf9290720
---
D manifests/role/coredb.pp
D modules/coredb_mysql/files/utils/pt-heartbeat.init
D modules/coredb_mysql/files/utils/snaprotate.pl
D modules/coredb_mysql/manifests/base.pp
D modules/coredb_mysql/manifests/heartbeat.pp
D modules/coredb_mysql/manifests/init.pp
D modules/coredb_mysql/manifests/packages.pp
D modules/coredb_mysql/manifests/purge_slow_digest.pp
D modules/coredb_mysql/manifests/slow_digest.pp
D modules/coredb_mysql/manifests/snapshot.pp
D modules/coredb_mysql/manifests/utils.pp
D modules/coredb_mysql/templates/master_id.py.erb
D modules/coredb_mysql/templates/prod.my.cnf.erb
D modules/coredb_mysql/templates/purge_query_digest.sh.erb
D modules/coredb_mysql/templates/root.my.cnf.erb
D modules/coredb_mysql/templates/send_query_digest.sh.erb
16 files changed, 0 insertions(+), 1,156 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/76/301076/1
diff --git a/manifests/role/coredb.pp b/manifests/role/coredb.pp
deleted file mode 100644
index a7d7357..0000000
--- a/manifests/role/coredb.pp
+++ /dev/null
@@ -1,335 +0,0 @@
-## DEPRECATED: to be deleted once all hosts have been migrated to the
-## mariadb class; topology will be handled by orchestration
-## for describing replication topology
-## hosts must be added here in addition to site.pp
-class role::coredb::config {
- $topology = {
- 's1' => {
- 'hosts' => { 'eqiad' => [ 'db1050', 'db1051', 'db1052', 'db1055',
'db1061', 'db1062', 'db1065', 'db1066', 'db1070', 'db1071' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1052' },
- 'snapshot' => [ 'db1050' ],
- 'no_master' => [ 'db1050', 'db1055' ]
- },
- 's2' => {
- 'hosts' => { 'eqiad' => [ 'db1009', 'db1018', 'db1024', 'db1036',
'db1060', 'db1063', 'db1067' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1024' },
- 'snapshot' => [ 'db1018' ],
- 'no_master' => [ 'db1018' ]
- },
- 's3' => {
- 'hosts' => { 'eqiad' => [ 'db1019', 'db1027', 'db1035', 'db1038' ]
},
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1038' },
- 'snapshot' => [ 'db1035' ],
- 'no_master' => [ 'db1035' ]
- },
- 's4' => {
- 'hosts' => { 'eqiad' => [ 'db1040', 'db1042', 'db1056', 'db1059',
'db1064', 'db1068' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1040' },
- 'snapshot' => [ 'db1042' ],
- 'no_master' => [ 'db1042' ]
- },
- 's5' => {
- 'hosts' => { 'eqiad' => [ 'db1021', 'db1026', 'db1037', 'db1045',
'db1049', 'db1058' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1058' },
- 'snapshot' => [ ],
- 'no_master' => [ 'db1026' ]
- },
- 's6' => {
- 'hosts' => { 'eqiad' => [ 'db1010', 'db1015', 'db1022', 'db1023',
'db1030' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1023' },
- 'snapshot' => [ 'db1022' ],
- 'no_master' => [ 'db1022', 'db1010' ]
- },
- 's7' => {
- 'hosts' => { 'eqiad' => [ 'db1028', 'db1033', 'db1034', 'db1039',
'db1041' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1033' },
- 'snapshot' => [ ],
- 'no_master' => [ 'db1041' ]
- },
- 'x1' => {
- 'hosts' => {
- 'eqiad' => [ 'db1029', 'db1031' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1029' },
- 'snapshot' => [ 'db1031' ],
- 'no_master' => []
- },
- 'm1' => {
- 'hosts' => {
- 'eqiad' => ['db1001', 'db1016'] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'db1001' },
- 'snapshot' => ['db1016' ],
- 'no_master' => []
- },
- # m2 role::mariadb::misc
- # m3 role::mariadb::misc::phabricator
- 'es1' => {
- 'hosts' => { 'eqiad' => [ 'es1001', 'es1002', 'es1003', 'es1004' ]
},
- 'primary_site' => false,
- 'masters' => {},
- 'snapshot' => [],
- 'no_master' => []
- },
- 'es2' => {
- 'hosts' => { 'eqiad' => [ 'es1005', 'es1006', 'es1007' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'es1006' },
- 'snapshot' => [],
- 'no_master' => []
- },
- 'es3' => {
- 'hosts' => { 'eqiad' => [ 'es1008', 'es1009', 'es1010' ] },
- 'primary_site' => $::mw_primary,
- 'masters' => { 'eqiad' => 'es1009' },
- 'snapshot' => [],
- 'no_master' => []
- },
- }
-}
-
-class role::coredb::s1( $mariadb = false, $innodb_file_per_table = false ) {
- class { 'role::coredb::common':
- shard => 's1',
- slow_query_digest => false,
- mariadb => $mariadb,
- innodb_file_per_table => $innodb_file_per_table,
- innodb_log_file_size => '2000M'
- }
-}
-
-class role::coredb::s2( $mariadb = false, $innodb_file_per_table = false ) {
- class { 'role::coredb::common':
- shard => 's2',
- slow_query_digest => false,
- mariadb => $mariadb,
- innodb_file_per_table => $innodb_file_per_table,
- innodb_log_file_size => '2000M'
- }
-}
-
-class role::coredb::s3( $mariadb = false, $innodb_file_per_table = false ) {
- class { 'role::coredb::common':
- shard => 's3',
- slow_query_digest => false,
- mariadb => $mariadb,
- innodb_file_per_table => $innodb_file_per_table,
- }
-}
-
-class role::coredb::s4( $mariadb = false, $innodb_file_per_table = false ) {
- class { 'role::coredb::common':
- shard => 's4',
- slow_query_digest => false,
- mariadb => $mariadb,
- innodb_file_per_table => $innodb_file_per_table,
- innodb_log_file_size => '2000M'
- }
-}
-
-class role::coredb::s5( $mariadb = false, $innodb_file_per_table = false ) {
- class { 'role::coredb::common':
- shard => 's5',
- slow_query_digest => false,
- mariadb => $mariadb,
- innodb_file_per_table => $innodb_file_per_table,
- innodb_log_file_size => '1000M'
- }
-}
-
-class role::coredb::s6( $mariadb = false, $innodb_file_per_table = false ) {
- class { 'role::coredb::common':
- shard => 's6',
- slow_query_digest => false,
- mariadb => $mariadb,
- innodb_file_per_table => $innodb_file_per_table,
- }
-}
-
-class role::coredb::s7( $mariadb = false, $innodb_file_per_table = false ) {
- class { 'role::coredb::common':
- shard => 's7',
- slow_query_digest => false,
- mariadb => $mariadb,
- innodb_file_per_table => $innodb_file_per_table,
- }
-}
-
-class role::coredb::x1( $mariadb = true ) {
- class { 'role::coredb::common':
- shard => 'x1',
- mariadb => $mariadb,
- innodb_file_per_table => true,
- }
-}
-
-class role::coredb::m1( $mariadb = false ) {
- class { 'role::coredb::common':
- shard => 'm1',
- mariadb => $mariadb,
- innodb_file_per_table => true,
- }
-}
-
-# m2 role::mariadb::misc
-# m3 role::mariadb::misc::phabricator
-
-class role::coredb::es1( $mariadb = false ) {
- class { 'role::coredb::common':
- shard => 'es1',
- mariadb => $mariadb,
- innodb_file_per_table => true,
- slow_query_digest => false,
- heartbeat_enabled => false,
- }
-}
-
-class role::coredb::es2( $mariadb = false ) {
- class { 'role::coredb::common':
- shard => 'es2',
- mariadb => $mariadb,
- innodb_file_per_table => true,
- slow_query_digest => false,
- }
-}
-
-class role::coredb::es3( $mariadb = false ) {
- class { 'role::coredb::common':
- shard => 'es3',
- mariadb => $mariadb,
- innodb_file_per_table => true,
- slow_query_digest => false,
- }
-}
-
-class role::coredb::researchdb(
- $shard='s1',
- $innodb_log_file_size = '2000M',
- $mariadb = false,
- $innodb_file_per_table = false
-){
- class { 'role::coredb::common':
- shard => $shard,
- mariadb => $mariadb,
- innodb_log_file_size => $innodb_log_file_size,
- read_only => false,
- disable_binlogs => true,
- long_timeouts => true,
- enable_unsafe_locks => true,
- large_slave_trans_retries => true,
- innodb_file_per_table => $innodb_file_per_table,
- # send researchdb icinga alerts to admins
- # and analytics icinga contact groups.
- contact_group => 'admins,analytics',
- }
-}
-
-class role::coredb::fundraising( $mariadb = true ) {
- class { 'role::coredb::common':
- shard => 'fundraisingdb',
- logical_cluster => 'fundraising',
- mariadb => $mariadb,
- innodb_file_per_table => true,
- slow_query_digest => false,
- heartbeat_enabled => false
- }
-}
-
-class role::coredb::common(
- $shard,
- $logical_cluster = 'mysql',
- $mariadb,
- $read_only = true,
- $skip_name_resolve = true,
- $mysql_myisam = false,
- $mysql_max_allowed_packet = '16M',
- $disable_binlogs = false,
- $innodb_log_file_size = '500M',
- $innodb_file_per_table = false,
- $long_timeouts = false,
- $enable_unsafe_locks = false,
- $large_slave_trans_retries = false,
- $slow_query_digest = true,
- $heartbeat_enabled = true,
- $contact_group = 'admins',
- ) inherits role::coredb::config {
-
- $primary_site = $topology[$shard]['primary_site']
- $masters = $topology[$shard]['masters']
- $snapshots = $topology[$shard]['snapshot']
-
- system::role { 'dbcore': description => "Shard ${shard} Core Database
server" }
-
- ::base::expose_puppet_certs { '/etc/mysql':
- ensure => present,
- provide_private => true,
- user => 'mysql',
- group => 'mysql',
- }
-
- include standard,
- mha::node,
- cpufrequtils
- class { 'mysql_wmf::coredb::ganglia' : mariadb => $mariadb; }
-
- if $masters[$::site] == $::hostname
- and ( $primary_site == $::site or $primary_site == 'both' ){
- class { 'coredb_mysql':
- shard => $shard,
- mariadb => $mariadb,
- read_only => false,
- skip_name_resolve => $skip_name_resolve,
- mysql_myisam => $mysql_myisam,
- mysql_max_allowed_packet => $mysql_max_allowed_packet,
- disable_binlogs => $disable_binlogs,
- innodb_log_file_size => $innodb_log_file_size,
- innodb_file_per_table => $innodb_file_per_table,
- long_timeouts => $long_timeouts,
- enable_unsafe_locks => $enable_unsafe_locks,
- large_slave_trans_retries => $large_slave_trans_retries,
- slow_query_digest => $slow_query_digest,
- heartbeat_enabled => $heartbeat_enabled,
- }
-
- class { 'mysql_wmf::coredb::monitoring':
- crit => true,
- contact_group => $contact_group,
- }
-
- }
- else {
- class { 'coredb_mysql':
- shard => $shard,
- mariadb => $mariadb,
- read_only => $read_only,
- skip_name_resolve => $skip_name_resolve,
- mysql_myisam => $mysql_myisam,
- mysql_max_allowed_packet => $mysql_max_allowed_packet,
- disable_binlogs => $disable_binlogs,
- innodb_log_file_size => $innodb_log_file_size,
- innodb_file_per_table => $innodb_file_per_table,
- long_timeouts => $long_timeouts,
- enable_unsafe_locks => $enable_unsafe_locks,
- large_slave_trans_retries => $large_slave_trans_retries,
- slow_query_digest => $slow_query_digest,
- heartbeat_enabled => $heartbeat_enabled,
- }
-
- if $primary_site {
- class { 'mysql_wmf::coredb::monitoring': crit => false }
- } else {
- class { 'mysql_wmf::coredb::monitoring': crit => false, no_slave
=> true }
- }
- }
-
- if $::hostname in $snapshots {
- include coredb_mysql::snapshot
- }
-}
diff --git a/modules/coredb_mysql/files/utils/pt-heartbeat.init
b/modules/coredb_mysql/files/utils/pt-heartbeat.init
deleted file mode 100755
index 0b990be..0000000
--- a/modules/coredb_mysql/files/utils/pt-heartbeat.init
+++ /dev/null
@@ -1,49 +0,0 @@
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides: pt-heartbeat
-# Required-Start: $remote_fs $syslog
-# Required-Stop: $remote_fs $syslog
-# Default-Start: 2 3 4 5
-# Default-Stop: 0 1 6
-# Short-Description: pt-heartbeat
-# Description: pt-heartbeat
-### END INIT INFO
-
-# Start/stop the pt-heartbeat daemon.
-#
-
-. /lib/init/vars.sh
-. /lib/lsb/init-functions
-
-do_start() {
- start-stop-daemon --start --quiet --pidfile /var/run/pt-heartbeat.pid
--name pt-heartbeat \
- --startas /usr/bin/pt-heartbeat --test || return 1
- start-stop-daemon --start --quiet --pidfile
/var/run/pt-heartbeat.pid --name pt-heartbeat \
- --startas /usr/bin/pt-heartbeat -- -D heartbeat --update
--replace --interval=0.5 \
- --daemon --pid /var/run/pt-heartbeat.pid -S /tmp/mysql.sock ||
return 2
-}
-
-do_stop() {
- start-stop-daemon --stop --quiet --retry 5 --pidfile
/var/run/pt-heartbeat.pid --startas /usr/bin/pt-heartbeat \
- || return 1
-}
-
-case "$1" in
-start) log_daemon_msg "Starting pt-heartbeat" "pt-heartbeat"
- do_start
- log_end_msg $?
- ;;
-stop) log_daemon_msg "Stopping pt-heartbeat" "pt-heartbeat"
- do_stop
- log_end_msg $?
- ;;
-restart) log_daemon_msg "Restarting pt-heartbeat" "upd2log"
- do_stop
- do_start
- log_end_msg $?
- ;;
-*) log_action_msg "Usage: /etc/init.d/pt-heartbeat {start|stop|restart}"
- exit 2
- ;;
-esac
-exit 0
diff --git a/modules/coredb_mysql/files/utils/snaprotate.pl
b/modules/coredb_mysql/files/utils/snaprotate.pl
deleted file mode 100755
index 29347b1..0000000
--- a/modules/coredb_mysql/files/utils/snaprotate.pl
+++ /dev/null
@@ -1,315 +0,0 @@
-#!/usr/bin/perl -w
-# vim:ai:filetype=perl:sta:sw=4:et:
-
-use strict;
-use Getopt::Long;
-use Pod::Usage;
-
-Getopt::Long::Configure('gnu_getopt',
- 'prefix_pattern=(--|-)');
-
-my $DEBUG=0;
-
-# Declarations
-#
-my $TRESHOLD=90;
-my $SNAPCOUNT=2;
-
-my ($ACTION, $VOLGRP, $SRCLV, $SNSIZE)=();
-my ($SHOWALL, $SNAME, $MAILTO, $OPTIONS, $HELP)=();
-my @snaps=();
-my $outstr;
-my @tmparr;
-
-# main
-GetOptions ('option|o' => \$OPTIONS,
- 'action|a=s' => \$ACTION,
- 'all|A' => \$SHOWALL,
- 'volgroup|V=s' => \$VOLGRP,
- 'sourcelv|s=s' => \$SRCLV,
- 'mailto|m=s' => \$MAILTO,
- 'snapname|n=s' => \$SNAME,
- 'size|L=s' => \$SNSIZE,
- 'treshold|t=i' => \$TRESHOLD,
- 'snapcount|c=i' => \$SNAPCOUNT,
- 'help|h' => \$HELP)
- or pod2usage(-verbose => 0, -exitval => 2);
-
-pod2usage(-verbose => 1) if ($OPTIONS);
-pod2usage(-verbose => 2) if ($HELP);
-
-
-if ( not defined $ACTION or ($ACTION !~ /^s(wap)?$/ and $ACTION !~ /^m(ail)?$/
- and $ACTION !~ /^r(eport)?$/)) {
- print STDERR "Error: missing or invalid action type.\n";
- pod2usage(-verbose => 0, -exitval => 2);
-}
-elsif (not defined $VOLGRP or not defined $SRCLV or (not defined $SNSIZE and
- $ACTION =~ /^s/)) {
- print STDERR "Error: missing mandatory option.\n";
- pod2usage(-verbose => 0, -exitval => 2);
-}
-elsif ($ACTION =~ /^s/ and $SNSIZE !~ /^\d+[kmgt]?$/i) {
- print STDERR "Error: Invalid size value.\n";
- pod2usage(-verbose => 0, -exitval => 2);
-}
-elsif ($ACTION =~ /^s/ and $SNAPCOUNT < 1) {
- print STDERR "Error: use a positive integer for
\"snapcount\".\n";
- pod2usage(-verbose => 0, -exitval => 2);
-}
-elsif ($ACTION =~ /^m/ and $MAILTO !~ /^[\w\.-]+(@[\w-]+(\.[\w-]+)*)?$/) {
- print STDERR "Error: Missing or invalid mail address.\n";
- pod2usage(-verbose => 0, -exitval => 2);
-}
-
-# More declarations
-my @lvscmd=("/sbin/lvs", "--noheadings", "--options",
- "lv_name,lv_attr,vg_name,origin,snap_percent");
-my @lvcrt=("/sbin/lvcreate", "--snapshot", "--size", $SNSIZE, "--name",
- "__FOO__", "--permission", "rw", "/dev/$VOLGRP/$SRCLV");
-my @lvrm=("/sbin/lvremove", "-f", "__FOO__");
-my @dfcmd=("/bin/df", "-k");
-my @lsofcmd=("/usr/bin/lsof","+D");
-my $mailprog="/usr/bin/Mail";
-
-#get lvs info
-$outstr=&Backticks(@lvscmd);
-chomp $outstr;
-@tmparr=split("\n",$outstr);
-foreach my $lvsline (@tmparr) {
- my @lvarr=split(" ", $lvsline);
- next if (scalar @lvarr != 5 or $lvarr[2] ne $VOLGRP or
$lvarr[3] ne $SRCLV);
- push @snaps, \@lvarr;
-}
-
-if ($ACTION =~ /^s/) {
- while (scalar(@snaps) >= $SNAPCOUNT) {
- # Delete oldest
- my $oldest="";
- my $percent=0;
- foreach my $lvline (@snaps) {
- if ($oldest eq "") {
- $oldest=$lvline->[0];
- $percent=$lvline->[4];
- }
- else {
- if ($lvline->[4] > $percent) {
- $oldest=$lvline->[0];
- $percent=$lvline->[4];
- }
- else {
- }
- }
- }
- # remove the entry from from @snaps
- @tmparr=();
- foreach my $lvline (@snaps) {
- push @tmparr, $lvline if
($lvline->[0] ne $oldest);
- }
- @snaps=@tmparr;
- # Check if the snapshot is mounted
- my @filesys=&Unfold(@dfcmd);
- foreach my $dfline(@filesys) {
- my @dfres=split(" ", $dfline, 6);
- next if ($dfres[0] ne
"/dev/mapper/$VOLGRP-$oldest");
- my @pids;
- my $lsofoutp=&Backticks(@lsofcmd,
$dfres[5]);
- foreach my $line (split "\n",
$lsofoutp) {
- chomp $line;
- next if ($line =~ /^COMMAND/);
- my @elems=split(" ",$line);
- unshift @pids, $elems[1];
- }
- # kill processes on the mounted
snapshot
- my
$dummy=&Backticks("/bin/kill","-KILL",@pids) if (scalar(@pids) > 0);
- # umount the snapshot
-
$dummy=&Backticks("/bin/umount",$dfres[5]);
- die "Could not umount snapshot
\"$oldest\".\n" if ($?);
- }
- $lvrm[2]="/dev/$VOLGRP/$oldest";
- # remove the snapshot
- my $dummy=&Backticks(@lvrm);
- die "Could not remove old snapshot
\"$oldest\".\n" if ($?);
- }
- if (not defined $SNAME or $SNAME eq "") {
- my ($min,$hour,$day,$mon);
-
($_,$min,$hour,$day,$mon,$_,$_,$_,$_)=localtime();
- $mon=sprintf("%02d",++$mon);
- $day=sprintf("%02d",$day);
- $hour=sprintf("%02d",$hour);
- $min=sprintf("%02d",$min);
- $SNAME="snap$mon$day$hour$min";
- }
- $lvcrt[5]=$SNAME;
- my $dummy=&Backticks(@lvcrt);
- die "Could not create new snapshot.\n" if ($?);
- exit 0;
-}
-else {
- #ACTION eq mail|report
- my $mailbody="";
- foreach my $snap (@snaps) {
- if ($SHOWALL) {
- $mailbody .= "Snapshot percentage for
$snap->[0] is $snap->[4]\%\n";
- }
- if ($snap->[1] =~ /^S/) {
- $mailbody .= "ERROR: snapshot
$snap->[0] is broken!\n";
- }
- elsif ($snap->[4] >= $TRESHOLD) {
- $mailbody .= "WARNING: snapshot
$snap->[0] is filled " .
- "above treshold ($snap->[4]\%)\n";
- }
- }
- if ($mailbody ne "") {
- if ($ACTION =~ /^m/) {
- die "No mailaddress specified\n" if
(not defined $MAILTO);
- my $pid=undef;
- my @mailcmd=($mailprog,"-s","Snapshot
report for /dev/$VOLGRP/$SRCLV",
- $MAILTO);
- if ($pid = open (CHILD,"|-")) {
- print CHILD $mailbody;
- }
- else {
- die "Cannot fork.\n" unless
defined $pid;
- exec @mailcmd;
- }
- }
- else {
- print STDERR $mailbody;
- }
- }
-}
-
-# Start of subs
-sub Unfold {
- my $result=&Backticks(@_);
- chomp $result;
- $result=~s/\n\s+/ /g;
- my @res=split("\n",$result);
- return @res;
-}
-
-sub Backticks {
- my @arr=@_;
- my ($result,$pid);
- if ($pid = open (CHILD,"-|")) {
- local $/;
- return <CHILD>;
- }
- else {
- if ( ! defined $pid ) {
- die "Cannot fork.\n";
- }
- if ( ! -x $arr[0] ) {
- die "Cannot execute $arr[0]\n";
- }
- print STDERR "executing ".join(" ",@arr)."\n"
if ($DEBUG);
- exec @arr;
- }
-}
-
-
-__END__
-
-=head1 NAME
-
-snaprotate.pl - Use rotating snapshots as a backup means.
-
-=head1 SYNOPSIS
-
-B<snaprotate.pl> B<-a>|B<--action> I<s[wap] | m[ail] | r[eport]>
-[B<-A>|B<--all>] B<-V>|B<--volgroup> I<vg>
-B<-s>|B<--sourcelv> I<lv> [B<-L>|B<--size> I<snapsize>]
-[B<-n>|B<--snapname> I<snapvol>] [B<-m>|B<--mailto> I<user@fqdn>]
-[B<-t>|B<--treshold> I<value>] [B<-c>|B<--snapcount> I<value>]
-
-B<snaprotate.pl> B<-h>|B<--help>
-
-B<snaprotate.pl> B<-o>|B<--options>
-
-=head1 DESCRIPTION
-
-This script will either rotate between snapshots of a logical volume or
-report if a snapshot volume is used more than a treshold value (default 90%).
-When rotating, the snapshot volume with most usage is deleted and a new one
-is created.
-
-=head1 OPTIONS
-
-=over 4
-
-=item B<-a, --action> I<s[wap] | m[ail] | r[eport]>
-
-Mandatory option. The I<swap> option will delete the oldest snapshot if
-all snapshots are used (or more oldest if more snapshots are used).
-WARNING: if a snapshot to be deleted is mounted, it will first be unmounted.
-If the filesystem is busy, all processed on the filesystem are killed with a
-SIGKILL.
-
-The <report> and I<mail> options check if a snapshot is used more than
-the treshold value or if a snapshot became invalid. I<report> prints the
-message to stderr and I<mail> sends the message to a mail address.
-
-=item B<-A, --all>
-
-Show usage info for all snapshots. This option is only used with the I<mail>
-and I<report> action types.
-
-=item B<-V, --volgroup> I<vg>
-
-The volume group that the logical volumes are members of. Mandatory option.
-
-=item B<-s, --sourcelv> I<lv>
-
-The logical volume that is snapped. Also mandatory.
-
-=item B<-n, --snapname> I<snapvol>
-
-Use this name as the name for the newly created snapshot (optional).
-
-=item B<-L, --size> I<snapsize>
-
-The size of the logical volume (use a suffix of K, M or G for Kibi, Mebi or
-Gibibytes). This option is mandatory for the I<swap> action, else ignored.
-
-=item B<-m, --mailto> I<user@fqdn>
-
-The user to where a warning of the snap utilization is mailed to. This option
-is ignored unless the I<mail> action is used but is mandatory when it is.
-
-=item B<-t, --treshold> I<value>
-
-The percentage of snap usage where the report option will generate an warning.
-The default value is 90. This value is only used by the I<mail> and
-I<report> actions.
-
-=item B<-c, --snapcount> I<value>
-
-The number of snapshots to cycle through. When fewer snapshots are available,
-nothing is deleted. If more than this number are available, the excess plus one
-more are deleted. The default number is 2. This option is only used by the
-I<swap> action.
-
-=item B<-o, --options>
-
-Print the list of command line options.
-
-=item B<-h, --help>
-
-Show the manpage.
-
-=head1 AUTHOR
-
-Rob S. Wolfram E<lt>[email protected]<gt>
-
-=head1 LICENSE
-
-This program is licensed according to the GNU General Public License
-(GPL) Version 2, or at your discretion, any later version. A copy of the
-license text can be obtained from
E<lt>http://www.gnu.org/licenses/gpl.htmlE<gt>
-or by mailing the author. In short it means that there are no restrictions on
-its use, but distributing the program or derivative works is only allowed
-according to the terms of the GPL.
-
-=cut
-
diff --git a/modules/coredb_mysql/manifests/base.pp
b/modules/coredb_mysql/manifests/base.pp
deleted file mode 100644
index 487fb81..0000000
--- a/modules/coredb_mysql/manifests/base.pp
+++ /dev/null
@@ -1,33 +0,0 @@
-# coredb_mysql required directories
-class coredb_mysql::base {
- require coredb_mysql::packages
-
- group { 'mysql':
- ensure => present,
- name => 'mysql',
- system => true,
- }
-
- user { 'mysql':
- shell => '/bin/sh',
- home => '/home/mysql',
- managehome => true,
- system => true,
- }
-
- file { '/a/sqldata':
- ensure => directory,
- owner => 'mysql',
- group => 'mysql',
- mode => '0755',
- require => User['mysql'],
- }
-
- file { '/a/tmp':
- ensure => directory,
- owner => 'mysql',
- group => 'mysql',
- mode => '0755',
- require => User['mysql'],
- }
-}
diff --git a/modules/coredb_mysql/manifests/heartbeat.pp
b/modules/coredb_mysql/manifests/heartbeat.pp
deleted file mode 100644
index c3b515f..0000000
--- a/modules/coredb_mysql/manifests/heartbeat.pp
+++ /dev/null
@@ -1,26 +0,0 @@
-# coredb_mysql heartbeat capability
-class coredb_mysql::heartbeat {
- require coredb_mysql::packages
- include passwords::misc::scripts
-
- file { '/root/.my.cnf':
- owner => 'root',
- group => 'root',
- mode => '0400',
- content => template('coredb_mysql/root.my.cnf.erb'),
- }
-
- file { '/etc/init.d/pt-heartbeat':
- owner => 'root',
- group => 'root',
- mode => '0555',
- source => 'puppet:///modules/coredb_mysql/utils/pt-heartbeat.init',
- }
-
- service { 'pt-heartbeat':
- ensure => running,
- require => File['/etc/init.d/pt-heartbeat'],
- subscribe => File['/etc/init.d/pt-heartbeat'],
- hasstatus => false,
- }
-}
diff --git a/modules/coredb_mysql/manifests/init.pp
b/modules/coredb_mysql/manifests/init.pp
deleted file mode 100644
index 6c82c3c..0000000
--- a/modules/coredb_mysql/manifests/init.pp
+++ /dev/null
@@ -1,45 +0,0 @@
-class coredb_mysql(
- $shard,
- $mariadb,
- $read_only,
- $skip_name_resolve,
- $mysql_myisam,
- $mysql_max_allowed_packet,
- $disable_binlogs,
- $innodb_log_file_size,
- $innodb_file_per_table,
- $long_timeouts,
- $enable_unsafe_locks,
- $large_slave_trans_retries,
- $slow_query_digest,
- $heartbeat_enabled
-) {
-
- include coredb_mysql::base
- include coredb_mysql::packages
- include coredb_mysql::utils
-
- if $slow_query_digest == true {
- include coredb_mysql::slow_digest
- }
-
- if $heartbeat_enabled == true {
- include coredb_mysql::heartbeat
- }
-
- file { '/etc/db.cluster':
- content => $shard,
- }
-
- file { '/etc/my.cnf':
- content => template('coredb_mysql/prod.my.cnf.erb'),
- }
-
- file { '/etc/mysql/my.cnf':
- ensure => link,
- target => '/etc/my.cnf',
- }
- Class['coredb_mysql'] -> Class['coredb_mysql::packages']
-}
-
-
diff --git a/modules/coredb_mysql/manifests/packages.pp
b/modules/coredb_mysql/manifests/packages.pp
deleted file mode 100644
index 08bc79e..0000000
--- a/modules/coredb_mysql/manifests/packages.pp
+++ /dev/null
@@ -1,31 +0,0 @@
-# coredb_mysql required packages
-class coredb_mysql::packages {
- if $::lsbdistid == 'Ubuntu' {
- apt::repository { 'wikimedia-mariadb':
- uri => 'http://apt.wikimedia.org/wikimedia',
- dist => 'precise-wikimedia',
- components => 'mariadb',
- }
-
- package { [
- 'libmariadbclient18',
- 'mariadb-client-5.5',
- 'mariadb-server-5.5',
- 'mariadb-server-core-5.5',
- ]:
- ensure => present,
- require => Apt::Repository['wikimedia-mariadb'],
- }
- } else {
- fail("coredb_mysql is deprecated and does not support ${::lsbdistid}.
Please use the 'mariadb' module")
- }
-
- package { [
- 'libaio1',
- 'lvm2',
- 'percona-toolkit',
- 'percona-xtrabackup',
- ]:
- ensure => latest,
- }
-}
diff --git a/modules/coredb_mysql/manifests/purge_slow_digest.pp
b/modules/coredb_mysql/manifests/purge_slow_digest.pp
deleted file mode 100644
index 1d1ce84..0000000
--- a/modules/coredb_mysql/manifests/purge_slow_digest.pp
+++ /dev/null
@@ -1,27 +0,0 @@
-# class for purging slow query digest log history
-class coredb_mysql::purge_slow_digest {
-
- include passwords::mysql::querydigest
-
- $mysql_user = 'ops'
- $digest_host = 'm1-master.eqiad.wmnet'
- $digest_slave = 'm1-slave.eqiad.wmnet'
- $digest_db = 'query_digests'
-
- file { '/usr/local/bin/purge_query_digest.sh':
- owner => 'root',
- group => 'root',
- mode => '0500',
- content => template('coredb_mysql/purge_query_digest.sh.erb'),
- }
-
- cron { 'purge_slow_digest':
- ensure => present,
- command => '/usr/local/bin/purge_query_digest.sh >/dev/null 2>&1',
- require => File['/usr/local/bin/purge_query_digest.sh'],
- user => 'root',
- weekday => 0,
- hour => 3,
- minute => 30,
- }
-}
diff --git a/modules/coredb_mysql/manifests/slow_digest.pp
b/modules/coredb_mysql/manifests/slow_digest.pp
deleted file mode 100644
index a8106c8..0000000
--- a/modules/coredb_mysql/manifests/slow_digest.pp
+++ /dev/null
@@ -1,33 +0,0 @@
-# class for sending sending slow query digest logs
-class coredb_mysql::slow_digest {
- include passwords::mysql::querydigest
-
- $mysql_user = 'ops'
- $digest_host = 'm1-master.eqiad.wmnet'
- $digest_db = 'query_digests'
-
- file { '/usr/local/bin/send_query_digest.sh':
- owner => 'root',
- group => 'root',
- mode => '0500',
- content => template('coredb_mysql/send_query_digest.sh.erb'),
- }
-
- cron { 'slow_digest':
- ensure => present,
- command => '/usr/local/bin/send_query_digest.sh >/dev/null 2>&1',
- require => File['/usr/local/bin/send_query_digest.sh'],
- user => 'root',
- minute => '*/20',
- hour => '*',
- }
-
- cron { 'tcp_query_digest':
- ensure => present,
- command => '/usr/local/bin/send_query_digest.sh tcpdump >/dev/null
2>&1',
- require => File['/usr/local/bin/send_query_digest.sh'],
- user => 'root',
- minute => [ 5, 25, 45 ],
- hour => '*',
- }
-}
diff --git a/modules/coredb_mysql/manifests/snapshot.pp
b/modules/coredb_mysql/manifests/snapshot.pp
deleted file mode 100644
index ebf68bd..0000000
--- a/modules/coredb_mysql/manifests/snapshot.pp
+++ /dev/null
@@ -1,17 +0,0 @@
-class coredb_mysql::snapshot {
- file { '/usr/local/sbin/snaprotate.pl':
- owner => 'root',
- group => 'root',
- mode => '0555',
- source => 'puppet:///modules/coredb_mysql/utils/snaprotate.pl',
- }
-
- cron { 'snaprotate':
- ensure => present,
- command => '/usr/local/sbin/snaprotate.pl -a swap -V tank -s data -L
100G',
- require => File['/usr/local/sbin/snaprotate.pl'],
- user => 'root',
- minute => 15,
- hour => '*/8',
- }
-}
diff --git a/modules/coredb_mysql/manifests/utils.pp
b/modules/coredb_mysql/manifests/utils.pp
deleted file mode 100644
index b70764d..0000000
--- a/modules/coredb_mysql/manifests/utils.pp
+++ /dev/null
@@ -1,9 +0,0 @@
-# various utility scripts for core dbs
-class coredb_mysql::utils {
- file { '/usr/local/bin/master_id.py':
- owner => 'root',
- group => 'root',
- mode => '0555',
- content => template('coredb_mysql/master_id.py.erb'),
- }
-}
diff --git a/modules/coredb_mysql/templates/master_id.py.erb
b/modules/coredb_mysql/templates/master_id.py.erb
deleted file mode 100755
index 3565d04..0000000
--- a/modules/coredb_mysql/templates/master_id.py.erb
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-#
-# this script returns the server-id of the current master
-# cluster name must be in /etc/db.cluster, set by puppet
-# and $cluster-master should be an up-to-date CNAME to
-# the master.
-#
-
-from socket import gethostbyname
-masterdom = '.<%= @mw_primary %>.wmnet'
-
-f = open('/etc/db.cluster', 'r')
-c = f.readline()
-mip = gethostbyname(c.split()[0] + '-master' + masterdom)
-octets = mip.split('.')
-print octets[0] + octets[2] + octets[3]
diff --git a/modules/coredb_mysql/templates/prod.my.cnf.erb
b/modules/coredb_mysql/templates/prod.my.cnf.erb
deleted file mode 100644
index deba8fc..0000000
--- a/modules/coredb_mysql/templates/prod.my.cnf.erb
+++ /dev/null
@@ -1,156 +0,0 @@
-[client]
-port = 3306
-socket = /tmp/mysql.sock
-
-[mysqld]
-
-<% ia = @ipaddress.split('.'); server_id = ia[0] + ia[2] + ia[3]; %>
-server_id=<%= server_id %>
-<% if @read_only == false then -%>
-# writes are enabled
-read_only = 0
-<% else %>
-read_only = 1
-<% end %>
-
-user = mysql
-socket = /tmp/mysql.sock
-port = 3306
-datadir = /a/sqldata/
-tmpdir = /a/tmp/
-
-skip-external-locking
-
-<% if @skip_name_resolve == true then -%>
-skip-name-resolve
-<% end -%>
-
-<% if @mysql_myisam == true then -%>
-myisam-recover = QUICK
-key_buffer = 1024M
-<% else %>
-key_buffer = 1M
-<% end -%>
-max_allowed_packet = <%= @mysql_max_allowed_packet %>
-thread_stack = 192K
-thread_cache_size = 300
-
-max_connections = 2500
-table_open_cache = 50000
-table_definition_cache = 40000
-
-
-query_cache_size = 0
-
-#log_slow_queries
-#log_queries_not_using_indexes
-#long_query_time = 0.45
-
-<% if @disable_binlogs != true then -%>
-log_bin
-log_slave_updates
-sync_binlog = 1
-binlog_cache_size = 1M
-max_binlog_size = 1000M
-binlog_format=statement
-expire_logs_days = 30
-<% end -%>
-
-connect_timeout=3
-back_log=1000
-max_connect_errors=1000000000
-
-temp-pool
-
-<% if @mariadb == true then -%>
-query_cache_type=0
-
-# these were set as compile time defaults in our old 5.1 mysql builds
-character_set_server=binary
-character_set_filesystem=binary
-collation_server=binary
-
-log_slow_verbosity=Query_plan
-optimizer_switch='mrr=on,mrr_cost_based=on,mrr_sort_keys=on,optimize_join_buffer_size=on,extended_keys=off'
-#thread_handling=pool-of-threads
-#extra_port=3308
-
-# https://mariadb.com/kb/en/user-statistics/
-userstat = 1
-
-# until we switch to RBR
-log-warnings = 0
-
-# MariaDB uses ARIA instead of MyISAM for implicit temporary tables that
-# hit disk for whever reason. The default 128M page cache is a bottleneck
-# particularly on slaves in 'vslow' load balancing group.
-aria_pagecache_buffer_size = 256M
-
-<% else %>
-rpl_transaction_enabled
-reserved_super_connections=20
-log_slow_extra
-innodb-thread-lifo=1
-<% end -%>
-
-<% ram = @memorysize.split[0]; ram = Float(ram) * 0.75;
-ram = ram.round; ram = String(ram) + "G"; %>
-
-innodb-adaptive-flushing=1
-innodb-buffer-pool-size=<%= ram %>
-innodb-flush-method=O_DIRECT
-innodb-io-capacity=1000
-innodb-log-file-size=<%= @innodb_log_file_size %>
-innodb-old-blocks-pct=80
-innodb-old-blocks-time=1000
-innodb-read-io-threads=16
-innodb-thread-concurrency=0
-innodb-use-sys-malloc
-innodb-write-io-threads=8
-innodb-checksums=1
-
-# try to improve innodb index cardinality accuracy on large tables
-# might need to be even higher and innodb_stats_on_metadata=OFF
-innodb_stats_sample_pages = 16
-
-<% if @mariadb == true then -%>
-# encourage more ref lookups when joining tables with nullable fields
-innodb_stats_method = nulls_unequal
-<% end -%>
-
-<% if @innodb_file_per_table == true then -%>
-innodb_file_per_table
-<% end -%>
-
-# Can't use without RBR
-#innodb-autoinc-lock-mode=2
-#innodb-locks-unsafe-for-binlog
-
-<% if @long_timeouts == true then -%>
-interactive_timeout = 129600
-wait_timeout = 129600
-<% end -%>
-
-<% if @enable_unsafe_locks == true then -%>
-# since no binlogs on this slave, and writes only to intern db's
-innodb-locks-unsafe-for-binlog
-<% end -%>
-
-<% if @large_slave_trans_retries == true then -%>
-slave_transaction_retries = 4294967295
-<% end -%>
-
-
-[mysqldump]
-quick
-quote-names
-max_allowed_packet = 16M
-
-[mysql]
-
-[isamchk]
-key_buffer = 16M
-
-
-#!includedir /etc/mysql/conf.d/
-
diff --git a/modules/coredb_mysql/templates/purge_query_digest.sh.erb
b/modules/coredb_mysql/templates/purge_query_digest.sh.erb
deleted file mode 100644
index 6a2ab81..0000000
--- a/modules/coredb_mysql/templates/purge_query_digest.sh.erb
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-# Remove query digest history older than 4 weeks
-
-pw=<%= scope.lookupvar('passwords::mysql::querydigest::mysql_ops_db') %>
-mysql_user=<%= mysql_user %>
-digest_host=<%= digest_host %>
-digest_slave=<%= digest_slave %>
-digest_db=<%= digest_db %>
-mysql_args="-h ${digest_host} -u ${mysql_user} -p${pw} -P 3306
--skip-column-names"
-
-for table in $(mysql ${mysql_args} -e 'show tables like "%history"'
${digest_db}); do
- echo $table
- pt-archiver --no-check-charset --primary-key-only --purge \
- --check-slave-lag="h=${digest_slave},P=3306,u=${mysql_user},p=${pw}"
--limit=1000 \
-
--source="h=${digest_host},P=3306,u=${mysql_user},p=${pw},D=${digest_db},t=${table}"
\
- --where="ts_max < DATE_SUB(NOW(), INTERVAL 4 WEEK)"
-done
diff --git a/modules/coredb_mysql/templates/root.my.cnf.erb
b/modules/coredb_mysql/templates/root.my.cnf.erb
deleted file mode 100644
index 9dea78d..0000000
--- a/modules/coredb_mysql/templates/root.my.cnf.erb
+++ /dev/null
@@ -1,4 +0,0 @@
-[client]
-user=root
-password=<%= scope.lookupvar('passwords::misc::scripts::mysql_root_pass') %>
-socket=/tmp/mysql.sock
diff --git a/modules/coredb_mysql/templates/send_query_digest.sh.erb
b/modules/coredb_mysql/templates/send_query_digest.sh.erb
deleted file mode 100755
index e436435..0000000
--- a/modules/coredb_mysql/templates/send_query_digest.sh.erb
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-#
-# Track slow queries and their history in a db using pt-query-digest
-# Filters out SELECT MASTER_POS_WAIT queries which would otherwise
-# rise to the top.
-
-pw=<%= scope.lookupvar('passwords::mysql::querydigest::mysql_ops_db') %>
-mysql_user=<%= @mysql_user %>
-digest_host=<%= @digest_host %>
-digest_db=<%= @digest_db %>
-hostname=$(hostname)
-log=/a/sqldata/${hostname}-slow.log
-runtime=8m
-table="query_review"
-
-export PERCONA_TOOLKIT_TCP_ERRORS_FILE=/dev/null
-
-digestif() {
- /usr/bin/pt-query-digest --review
h=${digest_host},D=${digest_db},t=${hostname}_${table},u=${mysql_user},p=${pw} \
- --create-review-table \
- --history
h=${digest_host},D=${digest_db},t=${hostname}_${table}_history,u=${mysql_user},p=${pw}
\
- --create-history-table \
- --noreport \
- --run-time $runtime \
- --filter '$event->{fingerprint} !~ m/^select master_pos_wait/' \
- $log
-}
-
-# optionally analayze all query traffic via tcpdump for a limited time
-if [ "$1" = "tcpdump" ] ; then
- # safeguard in case of --run-time failure
- /usr/bin/pgrep -f "pt-query-digest.*tcpdump" >/dev/null &&
/usr/bin/pkill -f "pt-query-digest.*tcpdump"
- log="--type tcpdump --limit 100%"
- runtime=30s
- table="tcpquery_review"
- /usr/sbin/tcpdump -s 65535 -x -nn -q -tttt -i any port 3306 | digestif
-else
- [ -s $log ] || exit 0
- # safeguard in case of --run-time failure
- /usr/bin/pgrep -f "pt-query-digest.*${log}" >/dev/null &&
/usr/bin/pkill -9 -f "pt-query-digest.*${log}"
- digestif
- > $log
-fi
--
To view, visit https://gerrit.wikimedia.org/r/301076
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id43776eb209f8984b501c3f9a4361f1cf9290720
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Jcrespo <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
