Faidon has uploaded a new change for review.
https://gerrit.wikimedia.org/r/74119
Change subject: Add an authdns module & associated role classes
......................................................................
Add an authdns module & associated role classes
This adds an authdns module. The module is an uncommon for our tree
mixture of a role module, a package replacement and internal logic to
handle the configuration of the authdns cluster. It serves as a
replacement to both the dns.pp authdns parts and for the
wikimedia-task-dns-auth package (the scripts included are basically
rewritten, though).
A gdnsd module is not provided here, as the software is simple enough to
not warrant it (a single Package & Service definition).
This also adds role classes for each of ns0/ns1/ns2 (with their current
IPs) and a testns class for playing with in labs. These are all UNUSED
for now.
This is the initial work. Pending work includes adding IPv6 addresses,
switching to the commercial GeoIP databases and providing scripts for CI
integration. Plus, bug fixes :)
Change-Id: Ice119ead68bd7cb5a232f31c778b03791a917012
---
A manifests/role/authdns.pp
A modules/authdns/files/authdns-gen-zones.py
A modules/authdns/files/authdns-git-pull
A modules/authdns/files/authdns-local-update
A modules/authdns/files/authdns-update
A modules/authdns/manifests/init.pp
A modules/authdns/manifests/monitoring.pp
A modules/authdns/manifests/scripts.pp
A modules/authdns/templates/config-head.erb
A modules/authdns/templates/wikimedia-authdns.conf.erb
10 files changed, 563 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/19/74119/1
diff --git a/manifests/role/authdns.pp b/manifests/role/authdns.pp
new file mode 100644
index 0000000..5ae6c75
--- /dev/null
+++ b/manifests/role/authdns.pp
@@ -0,0 +1,54 @@
+# authdns role classes, heavily relying on the authdns module
+
+class role::authdns::base {
+ include standard
+
+ system_role { 'authdns': description => 'Authoritative DNS server' }
+
+ $master = 'ns0.wikimedia.org'
+ $nameservers = [
+ 'ns0.wikimedia.org',
+ 'ns1.wikimedia.org',
+ 'ns2.wikimedia.org',
+ ]
+ $gitrepo = 'https://gerrit.wikimedia.org/r/p/operations/dns.git'
+
+ include authdns::monitoring
+}
+
+class role::authdns::ns0 inherits role::authdns::base {
+ class { 'authdns':
+ soa_name => 'ns0.wikimedia.org',
+ ipaddress => '208.80.152.130',
+ managed_iface => 'eth0',
+ nameservers => $nameservers,
+ gitrepo => $gitrepo,
+ }
+}
+
+class role::authdns::ns1 inherits role::authdns::base {
+ class { 'authdns':
+ soa_name => 'ns1.wikimedia.org',
+ ipaddress => '208.80.152.142',
+ managed_iface => 'eth0',
+ nameservers => $nameservers,
+ gitrepo => $gitrepo,
+ }
+}
+
+class role::authdns::ns2 inherits role::authdns::base {
+ class { 'authdns':
+ soa_name => 'ns2.wikimedia.org',
+ ipaddress => '91.198.174.4',
+ managed_iface => 'eth0',
+ nameservers => $nameservers,
+ gitrepo => $gitrepo,
+ }
+}
+
+class role::authdns::testns {
+ $gitrepo = 'https://gerrit.wikimedia.org/r/p/operations/dns.git'
+ class { 'authdns':
+ gitrepo => $gitrepo,
+ }
+}
diff --git a/modules/authdns/files/authdns-gen-zones.py
b/modules/authdns/files/authdns-gen-zones.py
new file mode 100644
index 0000000..56f317c
--- /dev/null
+++ b/modules/authdns/files/authdns-gen-zones.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+
+import argparse
+import sys
+import os
+import time
+import jinja2
+from filecmp import dircmp
+
+HEADER = '''; WARNING!
+; This file was automatically generated from a template
+; Do NOT edit this file directly!
+
+'''
+
+
+def parse_args():
+ """Sets ups argument parser and its arguments, returns args"""
+ parser = argparse.ArgumentParser()
+ parser.add_argument("templatedir",
+ help="the directory containing the zone templates")
+ parser.add_argument("zonedir",
+ help="the directory containing the formatted zones")
+ parser.add_argument("zones",
+ help="zone names to regenerate (optional, implies --force --keep)",
+ nargs="*")
+ parser.add_argument("-v", "--verbose",
+ help="increase output verbosity",
+ action="store_true",
+ default=0)
+ parser.add_argument("-f", "--force",
+ help="force regeneration of all zones",
+ action="store_true",
+ default=0)
+ parser.add_argument("-k", "--keep",
+ help="don't clean up zones for which no template exists",
+ action="store_true",
+ default=0)
+
+ return parser.parse_args()
+
+
+def main():
+ """main"""
+ args = parse_args()
+
+ template_env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(args.templatedir),
+ undefined=jinja2.StrictUndefined,
+ cache_size=0,
+ )
+
+ errors = False
+ context = {}
+ context['serial'] = time.strftime('%Y%m%d%H', time.gmtime())
+
+ if args.zones:
+ zones = args.zones
+ args.force = True
+ args.keep = True
+ else:
+ zones = os.listdir(args.templatedir)
+
+ for filename in zones:
+ filepath = os.path.join(args.templatedir, filename)
+ zonefilepath = os.path.join(args.zonedir, filename)
+ context['zonename'] = filename
+
+ # only process regular files and symlinks
+ if not os.path.isfile(filepath) or filename.startswith('.'):
+ if args.zones:
+ print "Skipping non-existent zone", filename
+ continue
+
+ try:
+ if not args.force and \
+ (os.path.getmtime(filepath) <= os.path.getmtime(zonefilepath)):
+ continue
+ except OSError:
+ # destination file not found
+ pass
+
+ if args.verbose:
+ print 'Processing zone', filename
+
+ try:
+ template = template_env.get_template(filename)
+ output = template.render(context)
+ except jinja2.exceptions.TemplateSyntaxError, exc:
+ print 'Skipping zone %s, syntax error on line %d: %s' % \
+ (filename, exc.lineno, exc.message)
+ errors = True
+ continue
+ except jinja2.exceptions.TemplateError, exc:
+ print 'Skipping zone %s, could not parse: %s' % \
+ (filename, str(exc))
+ errors = True
+ continue
+
+ # Write zonefile
+ open(zonefilepath, 'w').write(HEADER + output)
+
+ if not args.keep:
+ # cleanup removed zones
+ dcmp = dircmp(args.templatedir, args.zonedir)
+ for filename in dcmp.right_only:
+ if filename.startswith('.'):
+ continue
+ if args.verbose:
+ print "Cleaning up zone", filename
+ os.unlink(os.path.join(args.zonedir, filename))
+
+ if errors:
+ sys.exit(1)
+
+if __name__ == '__main__':
+ main()
+
+# vim: ts=4 sts=4 et ai shiftwidth=4 fileencoding=utf-8
diff --git a/modules/authdns/files/authdns-git-pull
b/modules/authdns/files/authdns-git-pull
new file mode 100644
index 0000000..cf59ba9
--- /dev/null
+++ b/modules/authdns/files/authdns-git-pull
@@ -0,0 +1,63 @@
+#!/bin/sh
+
+# Simple script that substitutes "git pull" but making sure that:
+# - the working tree has no untracked files
+# - the working tree has no unstaged changes
+# - the working tree has no staged but uncommited changes
+# - the working tree has no commits that are not present in FETCH_HEAD
+#
+# This is basically estabilishing that the repository is being used as a
+# replica and that a "pull" would only resync with remote
+#
+# Created by Faidon Liambotis, Jul 2013
+
+REMOTE=$1
+BRANCH=$2
+
+die() { echo >&2 "E: $*"; exit 1; }
+
+if [ -z "$REMOTE" ]; then
+ # shall we use 'origin' here?
+ die "no remote specified"
+elif [ -z "$BRANCH" ]; then
+ BRANCH="master"
+fi
+
+
+if test "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = false; then
+ die "not inside a working tree"
+fi
+
+if ! git rev-parse --verify $BRANCH >/dev/null; then
+ die "could not verify $BRANCH"
+fi
+
+if [ $(git rev-parse HEAD) != $(git rev-parse $BRANCH) ]; then
+ cur=$(git rev-parse --abbrev-ref HEAD)
+ die "working tree HEAD is pointed to '$cur', not '$BRANCH'"
+fi
+
+untracked=$(git ls-files --exclude-standard --others)
+if [ "$untracked" != "" ]; then
+ die "untracked files present: $untracked"
+fi
+
+if ! git diff-files --quiet --ignore-submodules; then
+ die "unstaged changes present"
+fi
+
+if ! git diff-index --cached --quiet --ignore-submodules HEAD --; then
+ die "staged but uncommited changes present"
+fi
+
+if ! git fetch $REMOTE $BRANCH 2>/dev/null; then
+ die "could not fetch $REMOTE $BRANCH"
+fi
+
+revlist=$(git rev-list -1 $BRANCH --not FETCH_HEAD)
+if [ "$revlist" != "" ]; then
+ echo $revlist
+ die "$BRANCH has diverged from $REMOTE, please reconcile first"
+fi
+
+git merge FETCH_HEAD
diff --git a/modules/authdns/files/authdns-local-update
b/modules/authdns/files/authdns-local-update
new file mode 100644
index 0000000..8c40eeb
--- /dev/null
+++ b/modules/authdns/files/authdns-local-update
@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Shell script that pulls zone templates from the origin or master DNS server,
+# regenerate zones & configuration and reload the DNS server.
+#
+# Written by Faidon Liambotis, Jul 2013
+
+set -e
+
+CONFFILE=/etc/wikimedia-authdns.conf
+
+# Source the configuration file
+[ -f $CONFFILE ] && . $CONFFILE
+
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+die() { echo >&2 "E: $*"; exit 1; }
+
+
+if [ "$(id -u)" -ne "0" ]; then
+ die "this script needs root"
+fi
+
+# setup locking; only one copy of this may be running at the same time
+LOCKFILE=/var/lock/authdns-local-update
+LOCKFD=9
+lock() { flock -xn $LOCKFD; }
+unlock() { rm -f $LOCKFILE; }
+eval "exec $LOCKFD>\"$LOCKFILE\""; trap unlock EXIT
+
+if ! flock -xn $LOCKFD; then
+ die "failed to lock, another update running?"
+fi
+
+if [ "$1" = "--skip-reload" ]; then
+ SKIP_RELOAD="true"
+ shift
+fi
+
+REMOTE=""
+if [ -z "$1" ]; then
+ if [ -z "$ORIGIN" ]; then
+ die "no master given and no origin defined in config"
+ fi
+ REMOTE=$ORIGIN
+else
+ REMOTE="ssh://$1/etc/gdnsd"
+fi
+
+echo "Pulling the current revision"
+(cd /etc/gdnsd; authdns-git-pull $REMOTE)
+
+echo "Generating zonefiles from zone templates"
+authdns-gen-zones /etc/gdnsd/templates /etc/gdnsd/zones
+
+echo "Generating gdnsd config"
+if [ ! -e "/etc/gdnsd/config-head" ]; then
+ die "config-head not found, system misconfigured?"
+fi
+if [ ! -e "/etc/gdnsd/config-geo" ]; then
+ die "config-geo not found, system misconfigured?"
+fi
+cp /etc/gdnsd/config /etc/gdnsd/config~ 2>/dev/null || true
+cat /etc/gdnsd/config-head /etc/gdnsd/config-geo > /etc/gdnsd/config
+
+echo "Doing sanity checks"
+if [ ! -s "/etc/gdnsd/config" ]; then
+ die "config seems empty, aborting"
+elif [ `ls /etc/gdnsd/zones |wc -l` -le 10 ]; then
+ die "less than 10 zones, probably something's wrong, aborting";
+fi
+
+if ! which gdnsd > /dev/null || [ "$SKIP_RELOAD" = "true" ]; then
+ rm -f /etc/gdnsd/config~
+ exit 0
+fi
+
+if ! gdnsd checkconf 2>/dev/null; then
+ mv /etc/gdnsd/config~ /etc/gdnsd/config
+ die "gdnsd checkconf failed, aborting"
+fi
+
+if ! cmp --quiet /etc/gdnsd/config~ /etc/gdnsd/config; then
+ rm -f /etc/gdnsd/config~
+ echo "Reloading zones & config"
+ gdnsd force-reload
+else
+ rm -f /etc/gdnsd/config~
+ echo "Reloading zones"
+ gdnsd reload
+fi
diff --git a/modules/authdns/files/authdns-update
b/modules/authdns/files/authdns-update
new file mode 100644
index 0000000..9e6a4af
--- /dev/null
+++ b/modules/authdns/files/authdns-update
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Shell script that takes care of running authdns-local-update for each
+# nameserver via SSH, optionally skipping failed ones.
+#
+# Written by Faidon Liambotis, Jul 2013 based on previous work by Mark Bergsma
+
+
+set -e
+
+CONFFILE=/etc/wikimedia-authdns.conf
+
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# Source the configuration file
+[ -f $CONFFILE ] && . $CONFFILE
+
+if [ -z "$NAMESERVERS" -o -z "$SKIP" ]; then
+ echo "Missing config file options, system misconfigured"
+ exit 1
+fi
+
+while [ -n "$1" ]; do
+ if [ "$1" = "-s" ]; then
+ # Skip the following slaves
+ SKIP="$SKIP $2"
+ fi
+ shift
+done
+
+if ssh-add -l>/dev/null 2>&1
+then
+ echo "Found your ssh agent."
+else
+ echo "Can't contact your ssh agent. You must forward your ssh agent to
use this script."
+ exit 2
+fi
+
+# update the local instance first
+authdns-local-update
+
+echo "Will skip slave(s) $SKIP."
+
+for slave in $NAMESERVERS; do
+ for skip in $SKIP; do
+ if [ $skip = $slave ]; then
+ echo "Skipping $slave."
+ continue 2
+ fi
+ done
+
+ echo "Updating $slave..."
+ ssh $slave authdns-local-update
+done
+
+echo "Done!"
diff --git a/modules/authdns/manifests/init.pp
b/modules/authdns/manifests/init.pp
new file mode 100644
index 0000000..3584d94
--- /dev/null
+++ b/modules/authdns/manifests/init.pp
@@ -0,0 +1,87 @@
+# == Class authdns
+# A class to implement Wikimedia's authoritative DNS system
+#
+class authdns(
+ $soa_name = $::fqdn,
+ $nameservers = [ $::fqdn ],
+ $ipaddress = undef,
+ $ipaddress6 = undef,
+ $gitrepo = undef,
+ $managed_iface = 'eth0',
+) {
+ Class['authdns::scripts'] -> Class['authdns']
+ Class['::geoip'] -> Class['authdns']
+
+ include authdns::scripts
+ class { '::geoip':
+ data_provider => 'package',
+ }
+
+ package { 'gdnsd':
+ ensure => installed,
+ }
+
+ service { 'gdnsd':
+ ensure => running,
+ hasrestart => true,
+ hasstatus => true,
+ require => Package['gdnsd'],
+ }
+
+ # do the initial clone via puppet; subsequent ones happen manually
+ git::clone{ '/etc/gdnsd':
+ directory => '/etc/gdnsd',
+ origin => $gitrepo,
+ branch => 'master',
+ before => Package['gdnsd'],
+ notify => Exec['authdns-local-update'],
+ }
+
+ file { '/etc/gdnsd/config-head':
+ ensure => present,
+ mode => '0444',
+ owner => 'root',
+ group => 'root',
+ content => template('authdns/config-head.erb'),
+ # this feels weird but git::clone is the one to create the dir
+ require => Git::Clone['/etc/gdnsd'],
+ }
+
+ file { '/etc/wikimedia-authdns.conf':
+ ensure => present,
+ mode => '0444',
+ owner => 'root',
+ group => 'root',
+ content => template('authdns/wikimedia-authdns.conf.erb'),
+ }
+
+ exec { 'authdns-local-update':
+ command => '/usr/local/sbin/authdns-local-update',
+ user => root,
+ refreshonly => true,
+ timeout => 60,
+ require => [
+ File['/etc/wikimedia-authdns.conf'],
+ File['/etc/gdnsd/config-head'],
+ ],
+ # we prepare the config even before the package gets installed, leaving
+ # no window where service would be started and answer with REFUSED
+ before => Package['gdnsd'],
+ }
+
+ if $ipaddress and $managed_iface {
+ interface_ip { 'authdns_ipv4':
+ interface => $managed_iface,
+ address => $ipaddress,
+ before => Package['gdnsd'],
+ }
+ }
+ if $ipaddress6 and $managed_iface {
+ interface_ip { 'authdns_ipv6':
+ interface => $managed_iface,
+ address => $ipaddress6,
+ prefixlen => 64,
+ before => Package['gdnsd'],
+ }
+ }
+}
diff --git a/modules/authdns/manifests/monitoring.pp
b/modules/authdns/manifests/monitoring.pp
new file mode 100644
index 0000000..03d28bc
--- /dev/null
+++ b/modules/authdns/manifests/monitoring.pp
@@ -0,0 +1,22 @@
+# == Class authdns::monitoring
+# Monitoring checks for authdns, specific to Wikimedia setup
+#
+class authdns::monitoring {
+ Class['authdns'] -> Class['authdns-monitoring']
+
+ if $authdns::ipaddress {
+ $monitor_ip = $authdns::ipaddress
+ } else {
+ $monitor_ip = $::ipaddress
+ }
+
+ monitor_host { $authdns::soa_name:
+ ip_address => $monitor_ip,
+ }
+
+ monitor_service { 'auth dns':
+ host => $authdns::soa_name,
+ description => 'Auth DNS',
+ check_command => 'check_dns!www.wikipedia.org'
+ }
+}
diff --git a/modules/authdns/manifests/scripts.pp
b/modules/authdns/manifests/scripts.pp
new file mode 100644
index 0000000..f29d9f8
--- /dev/null
+++ b/modules/authdns/manifests/scripts.pp
@@ -0,0 +1,48 @@
+# == Class authdns::monitoring
+# Scripts used by the authdns system. These used to be in a package,
+# but we don't do that anymore and provisioning them here instead.
+#
+class authdns::scripts {
+ if ! defined(Package['python-jinja2']){
+ package { 'python-jinja2':
+ ensure => present,
+ }
+ }
+
+ if ! defined(Package['git-core']){
+ package { 'git-core':
+ ensure => present,
+ }
+ }
+
+ file { '/usr/local/bin/authdns-gen-zones':
+ ensure => present,
+ mode => '0555',
+ owner => 'root',
+ group => 'root',
+ source => 'puppet:///modules/authdns/authdns-gen-zones.py',
+ }
+ file { '/usr/local/sbin/authdns-update':
+ ensure => present,
+ mode => '0555',
+ owner => 'root',
+ group => 'root',
+ source => 'puppet:///modules/authdns/authdns-update',
+ }
+
+ file { '/usr/local/sbin/authdns-local-update':
+ ensure => present,
+ mode => '0555',
+ owner => 'root',
+ group => 'root',
+ source => 'puppet:///modules/authdns/authdns-local-update',
+ }
+
+ file { '/usr/local/sbin/authdns-git-pull':
+ ensure => present,
+ mode => '0555',
+ owner => 'root',
+ group => 'root',
+ source => 'puppet:///modules/authdns/authdns-git-pull',
+ }
+}
diff --git a/modules/authdns/templates/config-head.erb
b/modules/authdns/templates/config-head.erb
new file mode 100644
index 0000000..b105c65
--- /dev/null
+++ b/modules/authdns/templates/config-head.erb
@@ -0,0 +1,20 @@
+options => {
+<% if (! @ipaddress.nil? || ! @ipaddress6.nil?) -%>
+ listen = [
+<% if (! @ipaddress.nil?) -%>
+ <%= @ipaddress %>,
+<% end -%>
+<% if (! @ipaddress6.nil?) -%>
+ <%= @ipaddress6 %>,
+<% end -%>
+ ],
+<% end -%>
+ http_listen = [
+ 127.0.0.1,
+ ::1,
+ ],
+ zones_default_ttl = 43200,
+ include_optional_ns = true,
+ # don't inotify on zonefiles but wait for HUP
+ zones_rfc1035_auto = false,
+}
diff --git a/modules/authdns/templates/wikimedia-authdns.conf.erb
b/modules/authdns/templates/wikimedia-authdns.conf.erb
new file mode 100644
index 0000000..c670b5f
--- /dev/null
+++ b/modules/authdns/templates/wikimedia-authdns.conf.erb
@@ -0,0 +1,3 @@
+NAMESERVERS="<%= @nameservers.join(' ') %>"
+SKIP="<%= @soa_name %>"
+ORIGIN="<%= @gitrepo %>"
--
To view, visit https://gerrit.wikimedia.org/r/74119
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ice119ead68bd7cb5a232f31c778b03791a917012
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Faidon <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
