Faidon Liambotis has uploaded a new change for review. https://gerrit.wikimedia.org/r/175571
Change subject: geoip: kill geoliteupdate in favor of geoipupdate ...................................................................... geoip: kill geoliteupdate in favor of geoipupdate MaxMind's geoipupdate mechanism has a "hidden" feature that MaxMind themselves pointed me to: GeoLite databases have their own updates.maxmind.com product codes and there is a special UserID of 999999 with a LicenseKey of 000000000000 that has privileges to download them. Kill geoliteupdate in favor of using geoipupdate across the board. This brings us a similar update mechanism for production & Labs, plus a better program to fetch updates, as this one also does MD5 checks etc. Change-Id: I34fb5b2d5253a9161d3c86c2e92375049c241775 --- D modules/geoip/files/geoliteupdate D modules/geoip/manifests/data/lite.pp M modules/geoip/manifests/data/maxmind.pp M modules/puppet/manifests/self/geoip.pp M modules/puppetmaster/manifests/geoip.pp 5 files changed, 42 insertions(+), 144 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/71/175571/1 diff --git a/modules/geoip/files/geoliteupdate b/modules/geoip/files/geoliteupdate deleted file mode 100644 index a5e1792..0000000 --- a/modules/geoip/files/geoliteupdate +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh - -# This is based on geoip-database-contrib_update from Debian's -# geoip-database-contrib package. The original source can be found at -# http://git.debian.org/?p=collab-maint/geoip-database-contrib.git -# and is -# Copyright: 2010/2013, Ludovico Cavedon <cave...@debian.org> -# Patrick Matthäi <pmatth...@debian.org> -# License: GPL-2+ -# -# It was modified by Faidon Liambotis for use by the Wikimedia Foundation - -DESTDIR=$1 -if [ ! -d "$DESTDIR" ]; then - echo "Usage: $0 <destdir>" - exit 1 -fi - -GEOIP_URL="http://geolite.maxmind.com/download/geoip/database"; - -FAILED=0 - -for url in \ - "$GEOIP_URL/GeoLiteCountry/GeoIP.dat.gz" \ - "$GEOIP_URL/GeoIPv6.dat.gz" \ - "$GEOIP_URL/GeoLiteCity.dat.gz" \ - "$GEOIP_URL/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz" \ - "$GEOIP_URL/asnum/GeoIPASNum.dat.gz" \ - "$GEOIP_URL/asnum/GeoIPASNumv6.dat.gz" \ - "$GEOIP_URL/GeoLite2-Country" \ - "$GEOIP_URL/GeoLite2-City.mmdb.gz" -do - echo "Downloading: $url" - - # Download file in the same directory as the final one so that the "mv" - # below can be atomic. - TEMPGZ=$(mktemp --tmpdir=$DESTDIR/ --suffix=.gz) - TEMP=${TEMPGZ%.gz} - FILEGZ=$(basename $url) - FILE=${FILEGZ%.gz} - - # MaxMind is being totally inconsistent and names both GeoIP Country and - # GeoLite Country with the same file. Explicitly rename this to - # GeoLite.dat, so that we can happily coexist with geoipupdate. - case "$FILE" in - GeoIP.dat) - FILE="GeoLite.dat" - ;; - esac - - /usr/bin/wget -q -t3 -T15 "$url" -O $TEMPGZ - - if [ "$?" != "0" ] - then - echo "Failed to download $url" - else - /bin/gunzip -f $TEMPGZ - - if [ "$?" != "0" ] - then - echo "Failed to decompress $FILEGZ" - else - rm -f $DESTDIR/$FILE - mv $TEMP $DESTDIR/$FILE - chmod 644 $DESTDIR/$FILE - fi - fi - - rm -f $TEMP $TEMPGZ -done - -exit 0 diff --git a/modules/geoip/manifests/data/lite.pp b/modules/geoip/manifests/data/lite.pp deleted file mode 100644 index e78ad21..0000000 --- a/modules/geoip/manifests/data/lite.pp +++ /dev/null @@ -1,55 +0,0 @@ -# == Class geoip::data::lite -# Installs Maxmind GeoLite database files by downloading them from Maxmind with -# a wget wrapper script. This also installs a cron job to do this weekly. -# -# == Parameters -# $data_directory - Where the data files should live. -# $environment - The environment parameter to pass to exec and cron for the -# geoliteupdate download command. default: undef - -class geoip::data::lite( - $data_directory = '/usr/share/GeoIP', - $environment = undef) -{ - if ! defined(File[$data_directory]) { - file { $data_directory: - ensure => directory, - } - } - - file { '/usr/local/bin/geoliteupdate': - ensure => present, - mode => '0555', - owner => 'root', - group => 'root', - source => 'puppet:///modules/geoip/geoliteupdate', - } - - $geoliteupdate_command = "/usr/local/bin/geoliteupdate ${data_directory}" - - # run once on the first instantiation of this class - exec { 'geoliteupdate': - command => $geoliteupdate_command, - refreshonly => true, - subscribe => File['/usr/local/bin/geoliteupdate'], - require => File[$data_directory], - } - - # Set up a cron to run geoliteupdate weekly. - cron { 'geoliteupdate': - ensure => present, - command => "${geoliteupdate_command} &> /dev/null", - user => 'root', - weekday => 0, - hour => 3, - minute => 30, - require => File[$data_directory], - } - - # if $environment was passed in, - # set it on the geoliteupdate commands - if ($environment != undef) { - Exec['geoliteupdate'] { environment => $environment } - Cron['geoliteupdate'] { environment => $environment } - } -} diff --git a/modules/geoip/manifests/data/maxmind.pp b/modules/geoip/manifests/data/maxmind.pp index 70f0055..66102ed 100644 --- a/modules/geoip/manifests/data/maxmind.pp +++ b/modules/geoip/manifests/data/maxmind.pp @@ -28,8 +28,8 @@ # class geoip::data::maxmind( $data_directory = '/usr/share/GeoIP', - $license_key = false, - $user_id = false, + $user_id = '999999', + $license_key = '000000000000', $product_ids = [106], $proxy = undef ) { diff --git a/modules/puppet/manifests/self/geoip.pp b/modules/puppet/manifests/self/geoip.pp index 476ed5a..3d5ce46 100644 --- a/modules/puppet/manifests/self/geoip.pp +++ b/modules/puppet/manifests/self/geoip.pp @@ -1,7 +1,5 @@ class puppet::self::geoip { - # Fetch the GeoIP databases into puppet's volatile dir, so that other hosts - # can then just sync that directory into their own /usr/share/GeoIP via a - # normal puppet File resource (see the geoip module for more) + # FIXME: this a partial duplicate of puppetmaster::geoip $geoip_destdir = '/var/lib/puppet/volatile/GeoIP' @@ -10,16 +8,30 @@ ensure => directory, } - # fetch the GeoLite databases - class { 'geoip::data::lite': + # legacy; remove eventually + file { '/usr/local/bin/geoliteupdate': + ensure => absent, + } + cron { 'geoliteupdate': + ensure => absent, + } + + class { 'geoip::data::maxmind': data_directory => $geoip_destdir, + product_ids => [ + '506', # GeoLite Legacy Country + '517', # GeoLite ASN + '533', # GeoLite Legacy City + 'GeoLite2-Country', + 'GeoLite2-City', + ], } # compatibility symlinks, so that users can use the stable paths # GeoIP.dat/GeoIPCity.dat between labs and production file { "$geoip_destdir/GeoIP.dat": ensure => link, - target => 'GeoLite.dat', + target => 'GeoLiteCountry.dat', } file { "$geoip_destdir/GeoIPCity.dat": ensure => link, @@ -34,4 +46,3 @@ target => 'GeoLite2-City.mmdb', } } - diff --git a/modules/puppetmaster/manifests/geoip.pp b/modules/puppetmaster/manifests/geoip.pp index 61ec6c9..23a33a1 100644 --- a/modules/puppetmaster/manifests/geoip.pp +++ b/modules/puppetmaster/manifests/geoip.pp @@ -11,18 +11,32 @@ ensure => directory, } - # fetch the GeoLite databases - class { 'geoip::data::lite': - data_directory => $geoip_destdir, - environment => "http_proxy=$webproxy", - } - if $is_labs_puppet_master { + # legacy; remove eventually + file { '/usr/local/bin/geoliteupdate': + ensure => absent, + } + cron { 'geoliteupdate': + ensure => absent, + } + + class { 'geoip::data::maxmind': + data_directory => $geoip_destdir, + proxy => $webproxy, + product_ids => [ + '506', # GeoLite Legacy Country + '517', # GeoLite ASN + '533', # GeoLite Legacy City + 'GeoLite2-Country', + 'GeoLite2-City', + ], + } + # compatibility symlinks, so that users can use the stable paths # GeoIP.dat/GeoIPCity.dat between labs and production file { "$geoip_destdir/GeoIP.dat": ensure => link, - target => 'GeoLite.dat', + target => 'GeoLiteCountry.dat', } file { "$geoip_destdir/GeoIPCity.dat": ensure => link, @@ -43,8 +57,8 @@ class { 'geoip::data::maxmind': data_directory => $geoip_destdir, proxy => $webproxy, - license_key => $passwords::geoip::license_key, user_id => $passwords::geoip::user_id, + license_key => $passwords::geoip::license_key, product_ids => [ '106', # GeoIP.dat '115', # GeoIPRegion.dat -- To view, visit https://gerrit.wikimedia.org/r/175571 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I34fb5b2d5253a9161d3c86c2e92375049c241775 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Faidon Liambotis <fai...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
