From: Martin Langhoff <[EMAIL PROTECTED]> --- client/ds_backup.py | 136 ++++++++++++++++++++++++++++++++++++++++++++++++ client/ds_backup.sh | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++ ds_backup.py | 136 ------------------------------------------------ ds_backup.sh | 144 --------------------------------------------------- 4 files changed, 280 insertions(+), 280 deletions(-) create mode 100755 client/ds_backup.py create mode 100755 client/ds_backup.sh delete mode 100755 ds_backup.py delete mode 100755 ds_backup.sh
diff --git a/client/ds_backup.py b/client/ds_backup.py new file mode 100755 index 0000000..dabcf39 --- /dev/null +++ b/client/ds_backup.py @@ -0,0 +1,136 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (C) 2007 Ivan KrstiÄ +# Copyright (C) 2007 Tomeu Vizoso +# Copyright (C) 2007 One Laptop per Child +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License (and +# no other version) as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import sha +import urllib +import os.path +import tempfile +import time +import glob +import popen2 +import re + +from sugar import env +from sugar import profile + +class BackupError(Exception): pass +class ProtocolVersionError(BackupError): pass +class RefusedByServerError(BackupError): pass +class ServerTooBusyError(BackupError): pass +class TransferError(BackupError): pass +class NoPriorBackups(BackupError): pass +class BulkRestoreUnavailable(BackupError): pass + +def find_last_backup(server, xo_serial): + try: + ret = urllib.urlopen(server + '/last/%s' % xo_serial).read() + return ret.split(',', 1) + except IOError, e: + if e[1] == 404: + raise ProtocolVersionError(server) + elif e[1] == 403: + raise RefusedByServerError(server) + elif e[1] == 503: + raise ServerTooBusyError(server) + +def find_restore_path(server, xo_serial): + try: + ret = urllib.urlopen(server + '/restore/%s' % xo_serial).read() + if ret == '0': + raise NoPriorBackups(server) + else: + return ret + except IOError, e: + if e[1] == 500: + raise BulkRestoreUnavailable(server) + elif e[1] == 503: + raise ServerTooBusyError(server) + +def rsync_to_xs(from_path, to_path, keyfile, user): + + # add a trailing slash to ensure + # that we don't generate a subdir + # at the remote end. rsync oddities... + if not re.compile('/$').search(from_path): + from_path = from_path + '/' + + ssh = '/usr/bin/ssh -F /dev/null -o "PasswordAuthentication no" -i "%s" -l "%s"' \ + % (keyfile, user) + rsync = "/usr/bin/rsync -az --partial --delete --timeout=160 -e '%s' '%s' '%s' " % \ + (ssh, from_path, to_path) + print rsync + rsync_p = popen2.Popen3(rsync, True) + + # here we could track progress with a + # for line in pipe: + # (an earlier version had it) + + # wait() returns a DWORD, we want the lower + # byte of that. + rsync_exit = os.WEXITSTATUS(rsync_p.wait()) + if rsync_exit != 0: + # TODO: retry a couple of times + # if rsync_exit is 30 (Timeout in data send/receive) + raise TransferError('rsync error code %s, message:' + % rsync_exit, rsync_p.childerr.read()) + + # Transfer an empty file marking completion + # so the XS can see we are done. + tmpfile = tempfile.mkstemp() + rsync = ("/usr/bin/rsync --timeout 10 -e '%s' '%s' '%s' " + % (ssh, tmpfile[1], to_path+'/.transfer_complete')) + rsync_p = popen2.Popen3(rsync, True) + rsync_exit = os.WEXITSTATUS(rsync_p.wait()) + if rsync_exit != 0: + # TODO: retry a couple ofd times + # if rsync_exit is 30 (Timeout in data send/receive) + raise TransferError('rsync error code %s, message:' + % rsync_exit, rsync_p.childerr.read()) + +def have_ofw_tree(): + return os.path.exists('/ofw') + +def read_ofw(path): + path = os.path.join('/ofw', path) + if not os.path.exists(path): + return None + fh = open(path, 'r') + data = fh.read().rstrip('\0\n') + fh.close() + return data + +# if run directly as script +if __name__ == "__main__": + + backup_url = 'http://schoolserver/backup/1' + + if have_ofw_tree(): + sn = read_ofw('mfg-data/SN') + else: + sn = 'SHF00000000' + + ds_path = env.get_profile_path('datastore') + pk_path = os.path.join(env.get_profile_path(), 'owner.key') + + # TODO: Check backup server availability + # if ping_xs(): + rsync_to_xs(ds_path, 'schoolserver:datastore', pk_path, sn) + # this marks success to the controlling script... + os.system('touch ~/.sugar/default/ds_backup-done') diff --git a/client/ds_backup.sh b/client/ds_backup.sh new file mode 100755 index 0000000..9916334 --- /dev/null +++ b/client/ds_backup.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Wrapper around ds_backup - will be called in 2 situations +# +# - On cron, every 30 minutes during waking/school hours +# If you are calling this from cron, pass 'cron' as +# the first parameter. +# +# - from a NetworkManager event when we +# associate to the school network. In that case, we get +# 2 parameters - if, action +# +# Note: this wrapper _must_ be cheap to execute to avoid burning +# battery. +# +# Author: Martin Langhoff <[EMAIL PROTECTED]> +# + +## +## Are we on a School server network? +## skip if we aren't! +## +## Note: this is simplistic on purpose - as we may be +## in one of many network topologies. +## +function skip_noschoolnet { + + # no DNS, no XS + grep -c '^nameserver ' /etc/resolv.conf 1>&/dev/null || exit + + # can't resolve & ping? outta here + ping -c1 schoolserver 1>&/dev/null || exit + + # TODO: if we are on a mesh, count the hops to + # the MPP - as the MPP will be the XS _or_ will provide + # access to it. Only continue to backup if the hopcount + # is low... + +} + +# If we have backed up recently, leave it for later. Use +# -mtime 0 for "today" +# -mtime -1 for "since yesterday" +# -mtime -10 for in the last 10 days +# +# Using -daystart means that the script is more eager to backup +# from early each day. Without -daystart, backups tend to happen +# later and later everyday, as they only start trying after 24hs... +# +# Another tack could be to try -mmin -1200 (20hs) - +# +function skip_ifrecent { + RECENT_CHECK='-daystart -mtime 0' + if [ `find ~/.sugar/default/ds_backup-done $RECENT_CHECK 2>/dev/null` ] + then + exit 0 + fi +} + + +# Will skip if we are on low batt +function skip_onlowbatt { + + if [ -e /sys/class/power_supply/olpc-battery/capacity \ + -a -e /sys/class/power_supply/olpc-ac/online ] + then + # OLPC HW + B_LEVEL=`cat /sys/class/power_supply/olpc-battery/capacity` + AC_STAT=`cat /sys/class/power_supply/olpc-ac/online` + else + # Portable, but 100ms slower on XO-1 + # Note - we read the 1st battery, and the 1st AC + # TODO: Smarter support for >1 battery + B_HAL=`hal-find-by-capability --capability battery | head -n1` + AC_HAL=`hal-find-by-capability --capability ac_adapter` + if [ -z $B_HAL -o -z $AC_HAL ] + then + # We do expect a battery & AC + exit 1; + fi + + B_LEVEL=`hal-get-property --udi $B_HAL --key battery.charge_level.percentage` + AC_STAT=`hal-get-property --udi $AC_HAL --key ac_adapter.present` + + # hal reports ac adapter presence as 'true' + # ... translate... + if [ "$AC_STAT" = 'true' ] + then + AC_STAT=1 + else + AC_STAT=0 + fi + fi + + # If we are on battery, and below 30%, leave it for later + if [ $AC_STAT == "0" -a $B_LEVEL -lt 30 ] + then + exit 0 + fi +} +## +## TODO: +## - Handle being called from NM + +## These checks are ordered cheapest first +skip_ifrecent; +skip_onlowbatt; +skip_noschoolnet; + +### Ok, we are going to attempt a backup + +# make the lock dir if needed +# we will keep the (empty) file around +if [ ! -d ~/.sugar/default/lock ] +then + mkdir ~/.sugar/default/lock || exit 1; +fi + +# +# Sleep a random amount, not greater than 20 minutes +# We use this to stagger client machines in the 30 minute +# slots between cron invocations... +# (yes we need all the parenthesys) +(sleep $(($RANDOM % 1200))); + +# After the sleep, check again. Perhaps something triggered +# another invokation that got the job done while we slept +skip_ifrecent; + +# Execute ds_backup.py from the same +# directory where we are. Use a flock +# to prevent concurrent runs. If the +# flock does not succeed immediately, +# we quit. +LOCKFILE=~/.sugar/default/lock/ds_backup.run +flock -n $LOCKFILE `dirname $0 `/ds_backup.py +EXITCODE=$? + +# Note: we keep the lockfile around to save +# NAND cycles. + +# Propagate the exit code of the flock/ds_backup invocation +exit $EXITCODE + diff --git a/ds_backup.py b/ds_backup.py deleted file mode 100755 index dabcf39..0000000 --- a/ds_backup.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# Copyright (C) 2007 Ivan KrstiÄ -# Copyright (C) 2007 Tomeu Vizoso -# Copyright (C) 2007 One Laptop per Child -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of version 2 of the GNU General Public License (and -# no other version) as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -import os -import sha -import urllib -import os.path -import tempfile -import time -import glob -import popen2 -import re - -from sugar import env -from sugar import profile - -class BackupError(Exception): pass -class ProtocolVersionError(BackupError): pass -class RefusedByServerError(BackupError): pass -class ServerTooBusyError(BackupError): pass -class TransferError(BackupError): pass -class NoPriorBackups(BackupError): pass -class BulkRestoreUnavailable(BackupError): pass - -def find_last_backup(server, xo_serial): - try: - ret = urllib.urlopen(server + '/last/%s' % xo_serial).read() - return ret.split(',', 1) - except IOError, e: - if e[1] == 404: - raise ProtocolVersionError(server) - elif e[1] == 403: - raise RefusedByServerError(server) - elif e[1] == 503: - raise ServerTooBusyError(server) - -def find_restore_path(server, xo_serial): - try: - ret = urllib.urlopen(server + '/restore/%s' % xo_serial).read() - if ret == '0': - raise NoPriorBackups(server) - else: - return ret - except IOError, e: - if e[1] == 500: - raise BulkRestoreUnavailable(server) - elif e[1] == 503: - raise ServerTooBusyError(server) - -def rsync_to_xs(from_path, to_path, keyfile, user): - - # add a trailing slash to ensure - # that we don't generate a subdir - # at the remote end. rsync oddities... - if not re.compile('/$').search(from_path): - from_path = from_path + '/' - - ssh = '/usr/bin/ssh -F /dev/null -o "PasswordAuthentication no" -i "%s" -l "%s"' \ - % (keyfile, user) - rsync = "/usr/bin/rsync -az --partial --delete --timeout=160 -e '%s' '%s' '%s' " % \ - (ssh, from_path, to_path) - print rsync - rsync_p = popen2.Popen3(rsync, True) - - # here we could track progress with a - # for line in pipe: - # (an earlier version had it) - - # wait() returns a DWORD, we want the lower - # byte of that. - rsync_exit = os.WEXITSTATUS(rsync_p.wait()) - if rsync_exit != 0: - # TODO: retry a couple of times - # if rsync_exit is 30 (Timeout in data send/receive) - raise TransferError('rsync error code %s, message:' - % rsync_exit, rsync_p.childerr.read()) - - # Transfer an empty file marking completion - # so the XS can see we are done. - tmpfile = tempfile.mkstemp() - rsync = ("/usr/bin/rsync --timeout 10 -e '%s' '%s' '%s' " - % (ssh, tmpfile[1], to_path+'/.transfer_complete')) - rsync_p = popen2.Popen3(rsync, True) - rsync_exit = os.WEXITSTATUS(rsync_p.wait()) - if rsync_exit != 0: - # TODO: retry a couple ofd times - # if rsync_exit is 30 (Timeout in data send/receive) - raise TransferError('rsync error code %s, message:' - % rsync_exit, rsync_p.childerr.read()) - -def have_ofw_tree(): - return os.path.exists('/ofw') - -def read_ofw(path): - path = os.path.join('/ofw', path) - if not os.path.exists(path): - return None - fh = open(path, 'r') - data = fh.read().rstrip('\0\n') - fh.close() - return data - -# if run directly as script -if __name__ == "__main__": - - backup_url = 'http://schoolserver/backup/1' - - if have_ofw_tree(): - sn = read_ofw('mfg-data/SN') - else: - sn = 'SHF00000000' - - ds_path = env.get_profile_path('datastore') - pk_path = os.path.join(env.get_profile_path(), 'owner.key') - - # TODO: Check backup server availability - # if ping_xs(): - rsync_to_xs(ds_path, 'schoolserver:datastore', pk_path, sn) - # this marks success to the controlling script... - os.system('touch ~/.sugar/default/ds_backup-done') diff --git a/ds_backup.sh b/ds_backup.sh deleted file mode 100755 index 9916334..0000000 --- a/ds_backup.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# -# Wrapper around ds_backup - will be called in 2 situations -# -# - On cron, every 30 minutes during waking/school hours -# If you are calling this from cron, pass 'cron' as -# the first parameter. -# -# - from a NetworkManager event when we -# associate to the school network. In that case, we get -# 2 parameters - if, action -# -# Note: this wrapper _must_ be cheap to execute to avoid burning -# battery. -# -# Author: Martin Langhoff <[EMAIL PROTECTED]> -# - -## -## Are we on a School server network? -## skip if we aren't! -## -## Note: this is simplistic on purpose - as we may be -## in one of many network topologies. -## -function skip_noschoolnet { - - # no DNS, no XS - grep -c '^nameserver ' /etc/resolv.conf 1>&/dev/null || exit - - # can't resolve & ping? outta here - ping -c1 schoolserver 1>&/dev/null || exit - - # TODO: if we are on a mesh, count the hops to - # the MPP - as the MPP will be the XS _or_ will provide - # access to it. Only continue to backup if the hopcount - # is low... - -} - -# If we have backed up recently, leave it for later. Use -# -mtime 0 for "today" -# -mtime -1 for "since yesterday" -# -mtime -10 for in the last 10 days -# -# Using -daystart means that the script is more eager to backup -# from early each day. Without -daystart, backups tend to happen -# later and later everyday, as they only start trying after 24hs... -# -# Another tack could be to try -mmin -1200 (20hs) - -# -function skip_ifrecent { - RECENT_CHECK='-daystart -mtime 0' - if [ `find ~/.sugar/default/ds_backup-done $RECENT_CHECK 2>/dev/null` ] - then - exit 0 - fi -} - - -# Will skip if we are on low batt -function skip_onlowbatt { - - if [ -e /sys/class/power_supply/olpc-battery/capacity \ - -a -e /sys/class/power_supply/olpc-ac/online ] - then - # OLPC HW - B_LEVEL=`cat /sys/class/power_supply/olpc-battery/capacity` - AC_STAT=`cat /sys/class/power_supply/olpc-ac/online` - else - # Portable, but 100ms slower on XO-1 - # Note - we read the 1st battery, and the 1st AC - # TODO: Smarter support for >1 battery - B_HAL=`hal-find-by-capability --capability battery | head -n1` - AC_HAL=`hal-find-by-capability --capability ac_adapter` - if [ -z $B_HAL -o -z $AC_HAL ] - then - # We do expect a battery & AC - exit 1; - fi - - B_LEVEL=`hal-get-property --udi $B_HAL --key battery.charge_level.percentage` - AC_STAT=`hal-get-property --udi $AC_HAL --key ac_adapter.present` - - # hal reports ac adapter presence as 'true' - # ... translate... - if [ "$AC_STAT" = 'true' ] - then - AC_STAT=1 - else - AC_STAT=0 - fi - fi - - # If we are on battery, and below 30%, leave it for later - if [ $AC_STAT == "0" -a $B_LEVEL -lt 30 ] - then - exit 0 - fi -} -## -## TODO: -## - Handle being called from NM - -## These checks are ordered cheapest first -skip_ifrecent; -skip_onlowbatt; -skip_noschoolnet; - -### Ok, we are going to attempt a backup - -# make the lock dir if needed -# we will keep the (empty) file around -if [ ! -d ~/.sugar/default/lock ] -then - mkdir ~/.sugar/default/lock || exit 1; -fi - -# -# Sleep a random amount, not greater than 20 minutes -# We use this to stagger client machines in the 30 minute -# slots between cron invocations... -# (yes we need all the parenthesys) -(sleep $(($RANDOM % 1200))); - -# After the sleep, check again. Perhaps something triggered -# another invokation that got the job done while we slept -skip_ifrecent; - -# Execute ds_backup.py from the same -# directory where we are. Use a flock -# to prevent concurrent runs. If the -# flock does not succeed immediately, -# we quit. -LOCKFILE=~/.sugar/default/lock/ds_backup.run -flock -n $LOCKFILE `dirname $0 `/ds_backup.py -EXITCODE=$? - -# Note: we keep the lockfile around to save -# NAND cycles. - -# Propagate the exit code of the flock/ds_backup invocation -exit $EXITCODE - -- 1.5.4.34.g053d9
_______________________________________________ Server-devel mailing list Server-devel@lists.laptop.org http://lists.laptop.org/listinfo/server-devel