Faidon Liambotis has uploaded a new change for review. https://gerrit.wikimedia.org/r/93683
Change subject: geoip: add a geoip::data::lite class ...................................................................... geoip: add a geoip::data::lite class Add a Lite data class that reuses a script from Debian to download the GeoLite databases from MaxMind. This is useful for both databases that have no paid-for equivalent (e.g. GeoIPv6, GeoLiteCityv6, etc.; basically all but GeoIP.dat & GeoIPCity.dat) as well as for Labs, where we can't distribute the proprietary databases. Note that normally, the GeoIP.dat name is shared among both GeoIP Country and GeoLite Country and hence would render this class in direct conflict with the other data methods (while in practice, sometimes you *need* to coinstall them). We diverge from what others do here, and rename the Lite GeoIP.dat to GeoLite.dat. The naming conventions and file paths under maxmind.com are really a mess anyway :) Change-Id: I402d5dc56522f43c7f1c2c76fe04f98078b3f902 --- A modules/geoip/files/geoliteupdate A modules/geoip/manifests/data/lite.pp 2 files changed, 134 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/83/93683/1 diff --git a/modules/geoip/files/geoliteupdate b/modules/geoip/files/geoliteupdate new file mode 100644 index 0000000..470f136 --- /dev/null +++ b/modules/geoip/files/geoliteupdate @@ -0,0 +1,88 @@ +#!/bin/sh + +# This is based on geoip-database-contrib_update from Debian's +# geoip-database-contrib package. The original source can be found at +# http://git.debian.org/?p=collab-maint/geoip-database-contrib.git +# and is +# Copyright: 2010/2013, Ludovico Cavedon <cave...@debian.org> +# Patrick Matthäi <pmatth...@debian.org> +# License: GPL-2+ +# +# It was modified by Faidon Liambotis for use by the Wikimedia Foundation + +DESTDIR=$1 +if [ ! -d "$DESTDIR" ]; then + echo "Usage: $0 <destdir>" + exit 1 +fi + +GEOIP_URL="http://geolite.maxmind.com/download/geoip/database/"; + +GEOLITE_COUNTRY_PATH="GeoLiteCountry/" +GEOLITE_COUNTRY_FILE="GeoIP.dat.gz" + +GEOLITE_COUNTRY_IPV6_PATH="" +GEOLITE_COUNTRY_IPV6_FILE="GeoIPv6.dat.gz" + +GEOLITE_CITY_PATH="" +GEOLITE_CITY_FILE="GeoLiteCity.dat.gz" + +GEOLITE_CITY_IPV6_PATH="GeoLiteCityv6-beta/" +GEOLITE_CITY_IPV6_FILE="GeoLiteCityv6.dat.gz" + +GEOLITE_ASNUM_PATH="asnum/" +GEOLITE_ASNUM_FILE="GeoIPASNum.dat.gz" + +GEOLITE_ASNUM_IPV6_PATH="asnum/" +GEOLITE_ASNUM_IPV6_FILE="GeoIPASNumv6.dat.gz" + +FAILED=0 + +for url in \ + "$GEOIP_URL$GEOLITE_COUNTRY_PATH$GEOLITE_COUNTRY_FILE" \ + "$GEOIP_URL$GEOLITE_COUNTRY_IPV6_PATH$GEOLITE_COUNTRY_IPV6_FILE" \ + "$GEOIP_URL$GEOLITE_CITY_PATH$GEOLITE_CITY_FILE" \ + "$GEOIP_URL$GEOLITE_CITY_IPV6_PATH$GEOLITE_CITY_IPV6_FILE" \ + "$GEOIP_URL$GEOLITE_ASNUM_PATH$GEOLITE_ASNUM_FILE" \ + "$GEOIP_URL$GEOLITE_ASNUM_IPV6_PATH$GEOLITE_ASNUM_IPV6_FILE" +do + echo "Downloading: $url" + + # Download file in the same directory as the final one so that the "mv" + # below can be atomic. + TEMPGZ=$(mktemp --tmpdir=$DESTDIR/ --suffix=.gz) + TEMP=${TEMPGZ%.gz} + FILEGZ=$(basename $url) + FILE=${FILEGZ%.gz} + + # MaxMind is being totally inconsistent and names both GeoIP Country and + # GeoLite Country with the same file. Explicitly rename this to + # GeoLite.dat, so that we can happily coexist with geoipupdate. + case "$FILE" in + GeoIP.dat) + FILE="GeoLite.dat" + ;; + esac + + /usr/bin/wget -q -t3 -T15 "$url" -O $TEMPGZ + + if [ "$?" != "0" ] + then + echo "Failed to download $url" + else + /bin/gunzip -f $TEMPGZ + + if [ "$?" != "0" ] + then + echo "Failed to decompress $FILEGZ" + else + rm -f $DESTDIR/$FILE + mv $TEMP $DESTDIR/$FILE + chmod 644 $DESTDIR/$FILE + fi + fi + + rm -f $TEMP $TEMPGZ +done + +exit 0 diff --git a/modules/geoip/manifests/data/lite.pp b/modules/geoip/manifests/data/lite.pp new file mode 100644 index 0000000..787ebda --- /dev/null +++ b/modules/geoip/manifests/data/lite.pp @@ -0,0 +1,46 @@ +# == Class geoip::data::lite +# Installs Maxmind GeoLite database files by downloading +# them from Maxmind with a wget wrapper script. +# This also installs a cron job to do this weekly. +# +# == Parameters +# $data_directory - Where the data files should live. default: $geoip::data::data_directory +# $environment - The environment parameter to pass to exec and cron for the +# geoliteupdate download command. default: undef + +class geoip::data::lite( + $data_directory = $geoip::data::data_directory, + $environment = undef) inherits geoip::data +{ + file { '/usr/local/bin/geoliteupdate': + source => "puppet:///${module_name}/geoliteupdate", + } + + $geoliteupdate_command = "/usr/local/bin/geoliteupdate ${data_directory}" + + # run once on the first instantiation of this class + exec { 'geoliteupdate': + command => $geoliteupdate_command, + refreshonly => true, + subscribe => File['/usr/local/bin/geoliteupdate'], + require => File[$data_directory], + } + + # Set up a cron to run geoliteupdate weekly. + cron { 'geoliteupdate': + ensure => present, + command => "${geoliteupdate_command} &> /dev/null", + user => root, + weekday => 0, + hour => 3, + minute => 30, + require => File[$data_directory], + } + + # if $environment was passed in, + # set it on the geoliteupdate commands + if ($environment != undef) { + Exec['geoliteupdate'] { environment => $environment } + Cron['geoliteupdate'] { environment => $environment } + } +} -- To view, visit https://gerrit.wikimedia.org/r/93683 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I402d5dc56522f43c7f1c2c76fe04f98078b3f902 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Faidon Liambotis <fai...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
