Last changes: - splitted scripts and torrents: added the parameter AUTOINSTALL_TORRENT_DIR in /etc/systemimager/systemimager.conf to define the place where torrents resides on the image server (default: /var/lib/systemimager/torrents/) - added the function bittorrent_get_file() that performs all the generic commands to download a file by bittorrent
Regards, -Andrea
Index: patches/bittorrent.freezehack.patch =================================================================== --- patches/bittorrent.freezehack.patch (revision 0) +++ patches/bittorrent.freezehack.patch (revision 0) @@ -0,0 +1,125 @@ +--- bittorrent-console.py.orig 2006-01-08 18:44:54.374123448 +0100 ++++ bittorrent-console.py 2006-01-08 18:46:56.425568808 +0100 +@@ -14,6 +14,122 @@ + + from __future__ import division + ++try: ++ import re ++ import encodings ++ import encodings.ascii ++ import encodings.base64_codec ++ import encodings.big5hkscs ++ import encodings.big5 ++ import encodings.bz2_codec ++ import encodings.charmap ++ import encodings.cp037 ++ import encodings.cp1006 ++ import encodings.cp1026 ++ import encodings.cp1140 ++ import encodings.cp1250 ++ import encodings.cp1251 ++ import encodings.cp1252 ++ import encodings.cp1253 ++ import encodings.cp1254 ++ import encodings.cp1255 ++ import encodings.cp1256 ++ import encodings.cp1257 ++ import encodings.cp1258 ++ import encodings.cp424 ++ import encodings.cp437 ++ import encodings.cp500 ++ import encodings.cp737 ++ import encodings.cp775 ++ import encodings.cp850 ++ import encodings.cp852 ++ import encodings.cp855 ++ import encodings.cp856 ++ import encodings.cp857 ++ import encodings.cp860 ++ import encodings.cp861 ++ import encodings.cp862 ++ import encodings.cp863 ++ import encodings.cp864 ++ import encodings.cp865 ++ import encodings.cp866 ++ import encodings.cp869 ++ import encodings.cp874 ++ import encodings.cp875 ++ import encodings.cp932 ++ import encodings.cp949 ++ import encodings.cp950 ++ import encodings.euc_jis_2004 ++ import encodings.euc_jisx0213 ++ import encodings.euc_jp ++ import encodings.euc_kr ++ import encodings.gb18030 ++ import encodings.gb2312 ++ import encodings.gbk ++ import encodings.hex_codec ++ import encodings.hp_roman8 ++ import encodings.hz ++ import encodings.idna ++ import encodings.iso2022_jp_1 ++ import encodings.iso2022_jp_2004 ++ import encodings.iso2022_jp_2 ++ import encodings.iso2022_jp_3 ++ import encodings.iso2022_jp_ext ++ import encodings.iso2022_jp ++ import encodings.iso2022_kr ++ import encodings.iso8859_10 ++ import encodings.iso8859_11 ++ import encodings.iso8859_13 ++ import encodings.iso8859_14 ++ import encodings.iso8859_15 ++ import encodings.iso8859_16 ++ import encodings.iso8859_1 ++ import encodings.iso8859_2 ++ import encodings.iso8859_3 ++ import encodings.iso8859_4 ++ import encodings.iso8859_5 ++ import encodings.iso8859_6 ++ import encodings.iso8859_7 ++ import encodings.iso8859_8 ++ import encodings.iso8859_9 ++ import encodings.johab ++ import encodings.koi8_r ++ import encodings.koi8_u ++ import encodings.latin_1 ++ import encodings.mac_cyrillic ++ import encodings.mac_greek ++ import encodings.mac_iceland ++ import encodings.mac_latin2 ++ import encodings.mac_roman ++ import encodings.mac_turkish ++ import encodings.mbcs ++ import encodings.palmos ++ import encodings.ptcp154 ++ import encodings.punycode ++ import encodings.quopri_codec ++ import encodings.raw_unicode_escape ++ import encodings.rot_13 ++ import encodings.shift_jis_2004 ++ import encodings.shift_jis ++ import encodings.shift_jisx0213 ++ import encodings.string_escape ++ import encodings.tis_620 ++ import encodings.undefined ++ import encodings.unicode_escape ++ import encodings.unicode_internal ++ import encodings.utf_16_be ++ import encodings.utf_16_le ++ import encodings.utf_16 ++ import encodings.utf_7 ++ import encodings.utf_8 ++ import encodings.uu_codec ++ import encodings.zlib_codec ++ import cStringIO ++ import gc ++ import xml.dom ++except: ++ pass ++ + from BitTorrent.platform import install_translation + install_translation() + Index: sbin/si_installbtimage =================================================================== --- sbin/si_installbtimage (revision 0) +++ sbin/si_installbtimage (revision 0) @@ -0,0 +1,299 @@ +#!/usr/bin/perl -w +# +# "SystemImager" +# +# Copyright (C) 2005 Andrea Righi <[EMAIL PROTECTED]> + +use lib "USR_PREFIX/lib/systemimager/perl"; +use strict; +use POSIX; +use Getopt::Long; +use SystemImager::Config; +use vars qw($config $VERSION); + +my $config_dir = "/etc/systemimager"; + +my $VERSION = "SYSTEMIMAGER_VERSION_STRING"; +my $program_name = "si_installbtimage"; +my $version_info = << "EOF"; +$program_name (part of SystemImager) v$VERSION + +Copyright (C) 1999-2001 Andrea Righi <[EMAIL PROTECTED]> +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +EOF + +my $get_help = "\n Try \"--help\" for more options."; + +my $help_info = $version_info . <<"EOF"; + +Usage: $program_name --images IMAGE1,IMAGE2,...IMAGEn [OPTION]... + +Options: (options can be presented in any order and may be abbreviated) + --help Display this output. + + --version Display version and copyright information. + + --images IMAGE1,IMAGE2,...,IMAGEn + Specify a comma separated list of the images to + distribute. + + --update Update bittorrent files. + + --compress Compress the image tar file with gzip. + (Default: don't compress) + + --quiet Run image server seeder in background. + +Download, report bugs, and make suggestions at: +http://systemimager.org/ + +EOF + +my ($help, $version, $quiet, $compress, $update, $images); +GetOptions( + "help" => \$help, + "version" => \$version, + "quiet" => \$quiet, + "compress" => \$compress, + "update" => \$update, + "images=s" => \$images, +) or die "$help_info"; + +### BEGIN evaluate commad line options ### +if ($help) { + print "$help_info"; + exit(0); +} +if ($version) { + print "$version_info"; + exit(0); +} + +# Get SystemImager directories. +my $IMAGE_DIR = $config->default_image_dir(); +my $TORRENTS_DIR = $config->autoinstall_torrent_dir(); + +# Get tracker port and interface from the configuration file. +my ($tracker_port, $image_server); +my $config_file = $config_dir . '/bittorrent.conf'; +open(IN, '<', $config_file) or + die "error: cannot open configuration file $config_file!\n"; +while(<IN>) { + if (m/BT_TRACKER_PORT=([0-9]+)/) { + $tracker_port = $1; + } elsif (m/BT_INTERFACE=(.+)/) { + ($image_server) = (`/sbin/ifconfig $1`)[1] =~ /inet addr:(\S+)/; + } +} +unless ($tracker_port) { + die "error in $config_file: BT_TRACKER_PORT not specified!\n"; +} +unless ($image_server) { + die "error in $config_file: BT_INTERFACE not specified or not valid!\n"; +} + +# Check the status of the tracker. +my $PIDFILE='/var/run/systemimager-server-bttracker.pid'; +system "ps -p `cat $PIDFILE 2>/dev/null` >/dev/null 2>&1"; +if ($?) { + print "error: systemimager tracker seems to be down...\n" . + "Try with:\n" . + "\t# /etc/init.d/systemimager-server-bittorrent restart\n"; + exit(1); +} + +# Check to use tarball or tar.gz. +if ($compress) { + $compress = 'z'; +} else { + $compress = ''; +} + +# Prepare all images for seeding. +foreach my $image_name (split(',', $images)) { + unless (-d "$IMAGE_DIR/$image_name") { + print "warning: \"$image_name\" is not a valid image!\n"; + next; + } + + # Create the tarball of the image. + my $tarball_file; + print "Preparing tar file for $image_name...\n"; + if ($update) { + unlink("$IMAGE_DIR/$image_name.tar", "$IMAGE_DIR/$image_name.tar.gz"); + } + if ($compress eq 'z') { + unlink ("$IMAGE_DIR/$image_name.tar") + if (-f "$IMAGE_DIR/$image_name.tar"); + $tarball_file = "$IMAGE_DIR/$image_name.tar.gz" + } else { + unlink ("$IMAGE_DIR/$image_name.tar.gz") + if (-f "$IMAGE_DIR/$image_name.tar.gz"); + $tarball_file = "$IMAGE_DIR/$image_name.tar" + } + unless (-f "$tarball_file") { + system "cd $IMAGE_DIR/$image_name && tar -c${compress}f $tarball_file ."; + if ($?) { + die "error: cannot create image tar file!\n"; + } + } + print "done.\n"; + + # Prepare torrent file. + my $torrent_file; + print "Preparing torrent file for $image_name...\n"; + if ($update) { + unlink("$TORRENTS_DIR/$image_name.tar.torrent", + "$TORRENTS_DIR/$image_name.tar.gz.torrent"); + } + if ($compress eq 'z') { + unlink ("$TORRENTS_DIR/$image_name.tar.torrent") + if (-f "$TORRENTS_DIR/$image_name.tar.torrent"); + $torrent_file = "$TORRENTS_DIR/$image_name.tar.gz.torrent" + } else { + unlink ("$TORRENTS_DIR/$image_name.tar.gz.torrent") + if (-f "$TORRENTS_DIR/$image_name.tar.gz.torrent"); + $torrent_file = "$TORRENTS_DIR/$image_name.tar.torrent" + } + unless (-f "$torrent_file") { + system "maketorrent-console --target $torrent_file http://$image_server:$tracker_port/announce $tarball_file"; + if ($?) { + die "error: cannot create torrent file!\n"; + } + } + print "done.\n"; +} + +# Prepare torrent file for the BOEL binaries. +my $ARCH = (uname())[4]; +$ARCH =~ s/i.86/i386/; + +my $boel_binaries = "$IMAGE_DIR/boel_binaries.tar.gz"; +print "Preparing torrent file for BOEL binaries...\n"; +# Create the symbolic link into the image directory. +symlink($config->autoinstall_boot_dir() . '/' . $ARCH . + '/standard/boel_binaries.tar.gz', $boel_binaries) + if (! -e $boel_binaries); +# Make .torrent file. +system "maketorrent-console --target $TORRENTS_DIR/boel_binaries.tar.gz.torrent http://$image_server:$tracker_port/announce $boel_binaries"; +if ($?) { + die "error: cannot create torrent file!\n"; +} +print "done.\n"; + +# Starting first image seeder. +print "Starting first image server seeder...\n"; +if ($quiet) { + system "cd $IMAGE_DIR && launchmany-console --no_start_trackerless_client --max_upload_rate 0 --rerequest_interval 1 --bind $image_server --save_in $IMAGE_DIR $TORRENTS_DIR>/dev/null 2>&1 &"; +} else { + # Evaluate if python-curses is installed. + `/usr/bin/env python << EOF +import sys, os; +try: + import curses +except: + sys.exit(1) +sys.exit(0); +EOF +`; + my $bt_ui = ($?) ? 'console' : 'curses'; + system "cd $IMAGE_DIR && launchmany-$bt_ui --no_start_trackerless_client --max_upload_rate 0 --rerequest_interval 1 --bind $image_server --save_in $IMAGE_DIR $TORRENTS_DIR"; +} +print "done.\n"; + +# Well done. +exit(0); + +__END__ + +=head1 NAME + +si_installbtimage - systemimager bittorrent seeder + +=head1 SYNOPSIS + +si_installbtimage --images IMAGE1,IMAGE2,...IMAGEn [OPTIONS]... + +=head1 DESCRIPTION + +B<si_installbtimage> is a program that configures the image server +to distribute an image using the bittorrent protocol as transport. + +Before using this program the tracker must be running on the image +server (see B</etc/init.d/systemimager-server-bittorrent>). + +The program provides: + - to create a tarball of the whole image, + - to create a .torrent file associated to the tarball, + - to start a "first seeder" on the image server. + +To install a client using the bittorrent transport you must define +in the B<BITTORRENT_STAGING=E<lt>pathE<gt>> parameter in the kernel boot +options (see /etc/systemimager/pxelinux.cfg/syslinux.cfg for a network +installation). + +If you think the clients have sufficient memory to host the whole +image in RAM you can specify B<BITTORRENT_STAGING=/tmp> to improve +performance. + +Otherwise you can choose to deploy the image tarball using the client +disk as a staging area, for example: B<BITTORRENT_STAGING=/a/tmp> +(where /a/ is the root of the file system of the client). + +=head1 OPTIONS + +=over 8 + +=item B<--help> + +Display a short help. + +=item B<--version> + +Display version and copyright information. + +=item B<--images IMAGE1,IMAGE2,...,IMAGEn> + +Specify a comma separated list of the images to deploy. + +=item B<--update> + +Rebuild the image tarball and the .torrent file. This option is +needed when the image has been modified (for example by the +B<si_getimage(8)> command or by direct changes in the image chrooted +file system). + +=item B<--compress> + +Compress the image tar file (with gzip). + +=item B<--quiet> + +Run the "first seeder" process in background. In this way you cannot +see the upload informations during the installtion of the clients. + +=head1 SEE ALSO + +systemimager(8), si_lsimage(8), si_getimage(8) + +=head1 AUTHOR + +Andrea Righi <[EMAIL PROTECTED]>. + +=head1 COPYRIGHT AND LICENSE + +Copyright 2003 by Andrea Righi <[EMAIL PROTECTED]>. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +=cut + Property changes on: sbin/si_installbtimage ___________________________________________________________________ Name: svn:executable + * Index: systemimager.spec =================================================================== --- systemimager.spec (revision 3410) +++ systemimager.spec (working copy) @@ -54,7 +54,7 @@ Packager: dann frazier <[EMAIL PROTECTED]> URL: http://systemimager.org/ Distribution: System Installation Suite -Requires: rsync >= 2.4.6, systemimager-common = %{version}, perl-AppConfig, dosfstools, /sbin/chkconfig, perl, perl(XML::Simple) >= 2.08 +Requires: rsync >= 2.4.6, systemimager-common = %{version}, perl-AppConfig, dosfstools, /sbin/chkconfig, perl, perl(XML::Simple) >= 2.08, python AutoReqProv: no %description server @@ -181,6 +181,7 @@ URL: http://systemimager.org/ Distribution: System Installation Suite Obsoletes: systemimager-%{_build_arch}boot +BuildRequires: python, python-devel, python-xml Requires: systemimager-server >= %{version} AutoReqProv: no @@ -214,6 +215,7 @@ URL: http://systemimager.org/ Distribution: System Installation Suite Obsoletes: systemimager-%{_build_arch}initrd_template +BuildRequires: python, python-devel, python-xml Requires: systemimager-client >= %{version} AutoReqProv: no Index: lib/SystemImager/Config.pm =================================================================== --- lib/SystemImager/Config.pm (revision 3410) +++ lib/SystemImager/Config.pm (working copy) @@ -21,6 +21,7 @@ 'rsync_stub_dir' => { ARGCOUNT => 1 }, 'tftp_dir' => { ARGCOUNT => 1 }, 'net_boot_default' => { ARGCOUNT => 1 }, + 'autoinstall_torrent_dir' => { ARGCOUNT => 1 }, ); $config->file('/etc/systemimager/systemimager.conf'); Index: initrd_source/make.d/cx_freeze.rul =================================================================== --- initrd_source/make.d/cx_freeze.rul (revision 0) +++ initrd_source/make.d/cx_freeze.rul (revision 0) @@ -0,0 +1,34 @@ +# +# 2005-12-28 Andrea Righi +# - Tool to freeze python script into binary +# + + +CX_FREEZE_VERSION := 3.0.2 +CX_FREEZE_DIR := cx_Freeze-$(CX_FREEZE_VERSION) +CX_FREEZE_TARBALL := cx_Freeze-$(CX_FREEZE_VERSION)-source.tgz +CX_FREEZE_URL := http://switch.dl.sourceforge.net/sourceforge/cx-freeze/$(CX_FREEZE_TARBALL) +#CX_FREEZE_URL := http://download.systemimager.org/pub/cx-freeze/$(CX_FREEZE_TARBALL) +CX_FREEZE_BINARY := $(INITRD_SRC_DIR)/$(CX_FREEZE_DIR)/FreezePython + +PHONY += cx_freeze + +cx_freeze: $(CX_FREEZE_BINARY) + +$(CX_FREEZE_BINARY): $(INITRD_SRC_DIR)/$(CX_FREEZE_TARBALL) + rm -rf $(INITRD_SRC_DIR)/$(CX_FREEZE_DIR) + cd $(INITRD_SRC_DIR) && tar -xvzf $(CX_FREEZE_TARBALL) + cd $(INITRD_SRC_DIR)/$(CX_FREEZE_DIR) && $(PYTHON) MakeFrozenBases.py && \ + $(PYTHON) FreezePython.py --no-copy-deps FreezePython.py + +$(INITRD_SRC_DIR)/$(CX_FREEZE_TARBALL): + [ -d $(INITRD_SRC_DIR) ] || mkdir -p $(INITRD_SRC_DIR) + $(GETSOURCE) $(CX_FREEZE_URL) $(INITRD_SRC_DIR) + +ALL_SOURCE += $(INITRD_SRC_DIR)/$(CX_FREEZE_TARBALL) + +PHONY += cx_freeze_clean +cx_freeze_clean: + rm -rf $(INITRD_SRC_DIR)/$(CX_FREEZE_DIR) + +# /* vi: set noet ts=4: */ Index: initrd_source/make.d/bittorrent.rul =================================================================== --- initrd_source/make.d/bittorrent.rul (revision 0) +++ initrd_source/make.d/bittorrent.rul (revision 0) @@ -0,0 +1,36 @@ +# +# 2005-12-28 Andrea Righi +# - Bittorrent binaries +# + + +BITTORRENT_VERSION := 4.4.0 +BITTORRENT_DIR := BitTorrent-$(BITTORRENT_VERSION) +BITTORRENT_TARBALL := BitTorrent-$(BITTORRENT_VERSION).tar.gz +BITTORRENT_URL := http://www.bittorrent.com/dl/$(BITTORRENT_TARBALL) +#BITTORRENT_URL := http://download.systemimager.org/pub/bittorrent/$(BITTORRENT_TARBALL) +BITTORRENT_BINARY := $(INITRD_SRC_DIR)/$(BITTORRENT_DIR)/bittorrent.py +BITTORRENT_PATCHES := $(shell ls $(PATCH_DIR)/bittorrent.*.patch 2>/dev/null | sort) + + +PHONY += bittorrent + +bittorrent: $(BITTORRENT_BINARY) + +$(BITTORRENT_BINARY): $(INITRD_SRC_DIR)/$(BITTORRENT_TARBALL) + rm -rf $(INITRD_SRC_DIR)/$(BITTORRENT_DIR) + cd $(INITRD_SRC_DIR) && tar -xvzf $(BITTORRENT_TARBALL) + cd $(INITRD_SRC_DIR)/$(BITTORRENT_DIR) && \ + patch -p0 < $(BITTORRENT_PATCHES) + +$(INITRD_SRC_DIR)/$(BITTORRENT_TARBALL): + [ -d $(INITRD_SRC_DIR) ] || mkdir -p $(INITRD_SRC_DIR) + $(GETSOURCE) $(BITTORRENT_URL) $(INITRD_SRC_DIR) + +ALL_SOURCE += $(INITRD_SRC_DIR)/$(BITTORRENT_TARBALL) + +PHONY += bittorrent_clean +bittorrent_clean: + rm -rf $(INITRD_SRC_DIR)/$(BITTORRENT_DIR) + +# /* vi: set noet ts=4: */ Index: initrd_source/initrd.rul =================================================================== --- initrd_source/initrd.rul (revision 3410) +++ initrd_source/initrd.rul (working copy) @@ -116,6 +116,8 @@ $(UCLIBC_TARGET) \ $(INSMOD_BINARY) \ $(MODPROBE_BINARY) \ + $(BITTORRENT_BINARY) \ + $(CX_FREEZE_BINARY) \ $(UDPCAST_BINARY) # Create directory structure. rm -fr $(INITRD_BUILD_DIR) @@ -138,7 +140,16 @@ cp -a $(DHCLIENT_BINARY) $(INITRD_BUILD_DIR)/sbin cp -a $(RSYNC_BINARY) $(INITRD_BUILD_DIR)/bin cp -a $(UDPCAST_BINARY) $(INITRD_BUILD_DIR)/bin - + + # Bittorrent installation. + $(CX_FREEZE_BINARY) --install-dir=$(INITRD_BUILD_DIR)/lib/bittorrent --exclude-modules=gobject,pango,gtk $(INITRD_SRC_DIR)/$(BITTORRENT_DIR)/torrentinfo-console.py + $(CX_FREEZE_BINARY) --install-dir=$(INITRD_BUILD_DIR)/lib/bittorrent --exclude-modules=gobject,pango,gtk $(INITRD_SRC_DIR)/$(BITTORRENT_DIR)/bittorrent-console.py + + # Explicit copy shared libraries needed by bittorrent binaries. + ( for l in $$(ldd $(INITRD_BUILD_DIR)/lib/bittorrent/* 2>/dev/null | sed -ne 's/.*=> \(.*\) (.*)/\1/p' | sort -u); do cp -pf $$l $(INITRD_BUILD_DIR)/lib; done ) + # Make sure the dynamic linker is present. + (for l in $$(readelf --program-headers $(INITRD_BUILD_DIR)/lib/bittorrent/* | sed -ne 's/^.*\[Requesting program interpreter: \(.*\)\]$$/\1/p' | sort -u); do cp -pf $$l $(INITRD_BUILD_DIR)/lib; done ) + ifdef INSMOD_BINARY ( [ ! -e $(INITRD_BUILD_DIR)/sbin/insmod ] && \ cp -a $(INSMOD_BINARY) $(INITRD_BUILD_DIR)/sbin/insmod ) || \ @@ -179,6 +190,10 @@ strip $(INITRD_BUILD_DIR)/usr/bin/* strip $(INITRD_BUILD_DIR)/usr/sbin/* + # Create the bittorrent links. + cd $(INITRD_BUILD_DIR)/bin && ln -s ../lib/bittorrent/torrentinfo-console + cd $(INITRD_BUILD_DIR)/bin && ln -s ../lib/bittorrent/bittorrent-console + # Copy over text files from the skel directory. cp -a $(INITRD_DIR)/skel/* $(INITRD_BUILD_DIR) @@ -200,8 +215,8 @@ cp -a $(INITRD_DIR)/my_modules $(INITRD_BUILD_DIR) # Clear out CVS cruft. - find $(INITRD_BUILD_DIR) -type d -name CVS -depth | xargs rm -rf - find $(INITRD_BUILD_DIR) -type d -name .svn -depth | xargs rm -rf + find $(INITRD_BUILD_DIR) -depth -type d -name CVS | xargs rm -rf + find $(INITRD_BUILD_DIR) -depth -type d -name .svn | xargs rm -rf mkdir $(INITRD_BUILD_DIR)/new_root touch $(INITRD_DIR)/build_dir-stamp Index: initrd_source/skel/etc/init.d/rcS =================================================================== --- initrd_source/skel/etc/init.d/rcS (revision 3410) +++ initrd_source/skel/etc/init.d/rcS (working copy) @@ -82,6 +82,8 @@ get_flamethrower_directory fi +get_torrents_directory + get_boel_binaries_tarball beep Index: initrd_source/skel/etc/init.d/functions =================================================================== --- initrd_source/skel/etc/init.d/functions (revision 3410) +++ initrd_source/skel/etc/init.d/functions (working copy) @@ -54,6 +54,8 @@ PATH=/sbin:/bin:/usr/bin:/usr/sbin:/tmp SCRIPTS=scripts SCRIPTS_DIR=/scripts +TORRENTS=torrents +TORRENTS_DIR=/torrents FLAMETHROWER_DIRECTORY_DIR=/var/lib/systemimager/flamethrower BOEL_BINARIES_DIR=/tmp/boel_binaries VERSION="SYSTEMIMAGER_VERSION_STRING" @@ -193,6 +195,8 @@ echo "MONITOR_SERVER=$MONITOR_SERVER" >> /tmp/variables.txt echo "MONITOR_PORT=$MONITOR_PORT" >> /tmp/variables.txt echo "MONITOR_CONSOLE=$MONITOR_CONSOLE" >> /tmp/variables.txt + + echo "BITTORRENT_STAGING=$BITTORRENT_STAGING" >> /tmp/variables.txt } # ################################################################################ @@ -309,6 +313,37 @@ # ################################################################################ # +# Usage: get_torrents_directory +# +get_torrents_directory() { + if [ -z $BITTORRENT_STAGING ]; then + return + fi + + logmsg + logmsg get_torrents_directory + + if [ ! -z $FLAMETHROWER_DIRECTORY_PORTBASE ]; then + # + # We're using Multicast, so we should already have a directory + # full of scripts. Break out here, so that we don't try to pull + # the scripts dir again (that would be redundant). + # + MODULE_NAME="autoinstall_torrents" + DIR="${SCRIPTS_DIR}" + RETRY=7 + flamethrower_client + + else + mkdir -p ${TORRENTS_DIR} + CMD="rsync -a ${IMAGESERVER}::${TORRENTS}/ ${TORRENTS_DIR}/" + logmsg "$CMD" + $CMD + fi +} +# +################################################################################ +# # Usage: get_scripts_directory # get_scripts_directory() { @@ -487,7 +522,57 @@ unset SUCCESS unset FLAMETHROWER_TARPIPE } +# +################################################################################ +# +# Download a file using bittorrent. +# +# Usage: bittorrent_get_file torrent destination +# +bittorrent_get_file() { + torrent=$1 + destination=$2 + # Bittorrent log file + bittorrent_log=/tmp/bittorrent.log + # Time to poll bittorrent events + bittorrent_polling_time=30 + # Minimum upload rate threshold (in KB/s), if lesser stop seeding + bittorrent_upload_min=50 + + # Start downloading. + bittorrent-console --no_start_trackerless_client --max_upload_rate 0 --display_interval 1 --rerequest_interval 1 --bind ${IPADDR} --save_in ${destination} ${torrent} > $bittorrent_log & + pid=$! + while :; do + while :; do + status=`grep 'percent done:' $bittorrent_log | sed -ne '$p' | sed 's/percent done: *//' | sed -ne '/^[0-9]*\.[0-9]*$/p'` + [ ! -z "$status" ] && break + done + logmsg "percent done: $status %" + if [ "$status" = "100.0" ]; then + # Sleep until upload rate reaches the minimum threshold + while :; do + sleep $bittorrent_polling_time + while :; do + upload_rate=`grep 'upload rate:' $bittorrent_log | sed -ne '$p' | sed 's/upload rate: *\([0-9]*\)\.[0-9]* .*$/\1/' | sed -ne '/^\([0-9]*\)$/p'` + [ ! -z $upload_rate ] && break + done + logmsg "upload rate: $upload_rate KB/s" + [ $upload_rate -lt $bittorrent_upload_min ] && break + done + logmsg "Stop seeding" + kill -9 $pid + unset bittorrent_log pid upload_rate + break + fi + sleep $bittorrent_polling_time + done + + unset bittorrent_polling_time + unset bittorrent_upload_min + unset torrent + unset destination +} # ################################################################################ # @@ -516,6 +601,17 @@ logmsg "$CMD" $CMD || shellout + elif [ ! -z $BITTORRENT_STAGING ]; then + + # Download BOEL binaries from peers + bittorrent_tarball="boel_binaries.tar.gz" + logmsg "Start downloading ${bittorrent_tarball} using bitorrent" + logmsg "" + logmsg "--> INFO: remember to run si_installbtimage on the image server!" + logmsg "" + bittorrent_get_file ${TORRENTS_DIR}/${bittorrent_tarball}.torrent ${BOEL_BINARIES_DIR} + unset bittorrent_tarball + elif [ ! -z $FLAMETHROWER_DIRECTORY_PORTBASE ]; then MODULE_NAME="boot-${ARCH}-${FLAVOR}" @@ -1423,7 +1519,19 @@ # Evaluate image size. logmsg "Evaluating image size." - IMAGESIZE=`rsync -av $IMAGESERVER::$IMAGENAME | grep "total size" | sed -e "s/total size is \([0-9]*\).*/\1/"` + if [ -z $BITTORRENT_STAGING ]; then + IMAGESIZE=`rsync -av $IMAGESERVER::$IMAGENAME | grep "total size" | sed -e "s/total size is \([0-9]*\).*/\1/"` + else + if [ -f "${TORRENTS_DIR}/$IMAGENAME.tar.torrent" ]; then + torrent_file="${TORRENTS_DIR}/$IMAGENAME.tar.torrent" + elif [ -f "${TORRENTS_DIR}/$IMAGENAME.tar.gz.torrent" ]; then + torrent_file="${TORRENTS_DIR}/$IMAGENAME.tar.gz.torrent" + else + logmsg "error: cannot find a valid torrent file for image ${IMAGENAME}" + shellout + fi + IMAGESIZE=`torrentinfo-console $torrent_file | sed -ne "s/file size\.*: \([0-9]*\) .*$/\1/p"` + fi IMAGESIZE=`expr $IMAGESIZE / 1024` # Evaluate disks size. Index: etc/bittorrent.conf =================================================================== --- etc/bittorrent.conf (revision 0) +++ etc/bittorrent.conf (revision 0) @@ -0,0 +1,19 @@ +# +# "SystemImager" +# +# Copyright (C) 2006 Andrea Righi <[EMAIL PROTECTED]> +# +# $Id$ +# + +# The bittorrent tracker port. +BT_TRACKER_PORT=6969 + +# Tracker state file. +BT_TRACKER_STATE=/tmp/dstate + +# Tracker log file. +BT_TRACKER_LOG=/var/log/systemimager/bittorrent-tracker.log + +# Interface used to seed files with bittorrent. +BT_INTERFACE=eth0 Index: etc/flamethrower.conf =================================================================== --- etc/flamethrower.conf (revision 3410) +++ etc/flamethrower.conf (working copy) @@ -344,3 +344,6 @@ [autoinstall_scripts] DIR = /var/lib/systemimager/scripts +[autoinstall_torrents] +DIR = /var/lib/systemimager/torrents + Index: etc/init.d/systemimager-server-bittorrent =================================================================== --- etc/init.d/systemimager-server-bittorrent (revision 0) +++ etc/init.d/systemimager-server-bittorrent (revision 0) @@ -0,0 +1,81 @@ +#!/bin/sh +# +# "SystemImager" +# +# Copyright (C) 2005 Andrea Righi +# +# Support for IRIX style chkconfig: +# chkconfig: 2345 20 20 +# description: The SystemImager tracker daemon. +# +# +# Support for LSB compliant init system: +### BEGIN INIT INFO +# Provides: systemimager +# Required-Start: $network +# Required-Stop: +# Default-Start: 3 5 +# Default-Stop: 0 1 2 6 +# Short-Description: SystemImager's tracker daemon. +# Description: This daemon is needed to image clients using the +# bittorrent protocol as transport. +# +### END INIT INFO + +. /etc/systemimager/bittorrent.conf + +PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin: +PIDFILE=/var/run/systemimager-server-bttracker.pid + +# Check configuration +[ -z $BT_TRACKER_PORT ] && + echo "error in /etc/systemimager/bittorrent.conf: BT_TRACKER_PORT not specified!" && + exit 1 +[ -z $BT_TRACKER_STATE ] && + echo "error in /etc/systemimager/bittorrent.conf: BT_TRACKER_STATE not specified!" && + exit 1 +[ -z $BT_TRACKER_LOG ] && + echo "error in /etc/systemimager/bittorrent.conf: BT_TRACKER_LOG not specified!" && + exit 1 + +case "$1" in + start) + echo -n "Starting SystemImager's tracker: systemimager-server-bttracker... " + if [ -e $PIDFILE ]; then + echo -e "failed.\nPID file $PIDFILE exists. Must be already running." + exit 1 + fi + # Remove the previous state file (if present). + rm -f $BT_TRACKER_STATE + # Start tracker in background. + bittorrent-tracker --port $BT_TRACKER_PORT --dfile $BT_TRACKER_STATE --logfile $BT_TRACKER_LOG & + if [ $? -ne 0 ]; then + echo failed. + exit 1 + else + echo $! > $PIDFILE + echo ok. + fi + ;; + stop) + echo -n "Stopping SystemImager's tracker: systemimager-server-bittorrent... " + [ -f $PIDFILE ] && kill -9 `cat $PIDFILE` >/dev/null 2>&1 + rm -f $PIDFILE + rm -f $BT_TRACKER_STATE + echo "stopped." + ;; + status) + echo -n "Status of SystemImager's tracker: systemimager-server-bittorrent... " + ([ -f $PIDFILE ] && ps -p `cat $PIDFILE 2>/dev/null` >/dev/null 2>&1 && echo running.) || echo not running. + ;; + force-reload|restart) + sh $0 stop + sh $0 start + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac + +exit 0 Property changes on: etc/init.d/systemimager-server-bittorrent ___________________________________________________________________ Name: svn:executable + * Index: etc/systemimager.conf =================================================================== --- etc/systemimager.conf (revision 3410) +++ etc/systemimager.conf (working copy) @@ -53,3 +53,9 @@ # NET_BOOT_DEFAULT = net +################################################################################ +# +# The directory where files needed by bittorrent transport are stored. +# +AUTOINSTALL_TORRENT_DIR = /var/lib/systemimager/torrents + Index: etc/autoinstallscript.template =================================================================== --- etc/autoinstallscript.template (revision 3410) +++ etc/autoinstallscript.template (working copy) @@ -216,6 +216,33 @@ if [ ! -z $MONITOR_SERVER ]; then stop_report_task 100 fi +elif [ ! -z $BITTORRENT_STAGING ]; then + # Get the right .torrent file. + if [ -f "${TORRENTS_DIR}/${IMAGENAME}.tar.gz.torrent" ]; then + bittorrent_tarball="${IMAGENAME}.tar.gz" + compress='z' + elif [ -f "${TORRENTS_DIR}/${IMAGENAME}.tar.torrent" ]; then + bittorrent_tarball="${IMAGENAME}.tar" + compress='' + else + logmsg "error: cannot find a valid torrent file for the image ${IMAGENAME}" + shellout + fi + + # Download image from peers + logmsg "Start downloading image using torrent ${bittorrent_tarball}.torrent" + logmsg "" + logmsg "--> INFO: remember to run si_installbtimage on the image server!" + logmsg "" + bittorrent_get_file ${TORRENTS_DIR}/${bittorrent_tarball}.torrent ${BITTORRENT_STAGING} + if [ ! -z $MONITOR_SERVER ]; then + stop_report_task 100 + fi + # Extract image. + logmsg "Extracting image from ${bittorrent_tarball} ..." + (cd /a/ && tar -xv${compress}f ${BITTORRENT_STAGING}/${bittorrent_tarball}) || shellout + rm -f ${BITTORRRENT_STAGING}/${bittorrent_tarball} + unset bittorrent_tarball else # Use rsync if [ $NO_LISTING ]; then Index: etc/rsync_stubs/10header =================================================================== --- etc/rsync_stubs/10header (revision 3410) +++ etc/rsync_stubs/10header (working copy) @@ -36,3 +36,6 @@ [scripts] path = /var/lib/systemimager/scripts +[torrents] +path = /var/lib/systemimager/torrents + Index: Makefile =================================================================== --- Makefile (revision 3410) +++ Makefile (working copy) @@ -40,6 +40,7 @@ # SystemImager file location standards: # o images will be stored in: /var/lib/systemimager/images/ # o autoinstall scripts: /var/lib/systemimager/scripts/ +# o torrent files: /var/lib/systemimager/torrents/ # o override directories: /var/lib/systemimager/overrides/ # # o web gui pages: /usr/share/systemimager/web-gui/ @@ -77,6 +78,7 @@ # boot files for clients) # o overrides # o scripts +# o torrents # # # To include the ctcs test suite, and associated files, do a 'make WITH_CTCS=1 all' @@ -164,7 +166,7 @@ PXE_CONF_DEST = $(ETC)/systemimager/pxelinux.cfg BINARIES := si_mkautoinstallcd si_mkautoinstalldiskette si_mkbootmedia -SBINARIES := si_addclients si_cpimage si_getimage si_mkdhcpserver si_mkdhcpstatic si_mkautoinstallscript si_mkbootserver si_mvimage si_pushupdate si_rmimage si_mkrsyncd_conf si_mkclientnetboot si_netbootmond si_imagemanip si_mkbootpackage si_monitor si_monitortk +SBINARIES := si_addclients si_cpimage si_getimage si_mkdhcpserver si_mkdhcpstatic si_mkautoinstallscript si_mkbootserver si_mvimage si_pushupdate si_rmimage si_mkrsyncd_conf si_mkclientnetboot si_netbootmond si_imagemanip si_mkbootpackage si_monitor si_monitortk si_installbtimage CLIENT_SBINARIES := si_updateclient si_prepareclient COMMON_BINARIES = si_lsimage @@ -172,6 +174,7 @@ IMAGEDEST = $(DESTDIR)/var/lib/systemimager/images WARNING_FILES = $(IMAGESRC)/README $(IMAGESRC)/CUIDADO $(IMAGESRC)/ACHTUNG AUTOINSTALL_SCRIPT_DIR = $(DESTDIR)/var/lib/systemimager/scripts +AUTOINSTALL_TORRENT_DIR = $(DESTDIR)/var/lib/systemimager/torrents OVERRIDES_DIR = $(DESTDIR)/var/lib/systemimager/overrides OVERRIDES_README = $(TOPDIR)/var/lib/systemimager/overrides/README FLAMETHROWER_STATE_DIR = $(DESTDIR)/var/state/systemimager/flamethrower @@ -282,6 +285,8 @@ $(SI_INSTALL) -d -m 755 $(BOOT_BIN_DEST) $(SI_INSTALL) -d -m 755 $(AUTOINSTALL_SCRIPT_DIR) + $(SI_INSTALL) -d -m 755 $(AUTOINSTALL_TORRENT_DIR) + $(SI_INSTALL) -d -m 755 $(AUTOINSTALL_SCRIPT_DIR)/pre-install $(SI_INSTALL) -m 644 --backup --text \ $(TOPDIR)/var/lib/systemimager/scripts/pre-install/99all.harmless_example_script \ @@ -314,6 +319,9 @@ $(SI_INSTALL) -d -m 755 $(FLAMETHROWER_STATE_DIR) + # Install server-side BitTorrent. + cd $(INITRD_SRC_DIR)/$(BITTORRENT_DIR) && $(PYTHON) setup.py install --prefix $(PREFIX) + # install client-only files .PHONY: install_client install_client: install_client_man install_client_libs @@ -386,6 +394,7 @@ $(SI_INSTALL) -d $(ETC)/systemimager $(SI_INSTALL) -m 644 etc/systemimager.conf $(ETC)/systemimager/ $(SI_INSTALL) -m 644 etc/flamethrower.conf $(ETC)/systemimager/ + $(SI_INSTALL) -m 644 etc/bittorrent.conf $(ETC)/systemimager/ $(SI_INSTALL) -m 644 etc/autoinstallscript.template $(ETC)/systemimager/ $(SI_INSTALL) -m 644 etc/imagemanip.conf $(ETC)/systemimager/ $(SI_INSTALL) -m 644 etc/imagemanip.perm $(ETC)/systemimager/ @@ -402,6 +411,7 @@ $(SI_INSTALL) -b -m 755 etc/init.d/systemimager-server-rsyncd $(INITD) $(SI_INSTALL) -b -m 755 etc/init.d/systemimager-server-netbootmond $(INITD) $(SI_INSTALL) -b -m 755 etc/init.d/systemimager-server-flamethrowerd $(INITD) + $(SI_INSTALL) -b -m 755 etc/init.d/systemimager-server-bittorrent $(INITD) $(SI_INSTALL) -b -m 755 etc/init.d/systemimager-server-monitord $(INITD) ########## END initrd ########## @@ -591,6 +601,7 @@ test ! -d /lib64 || TGTLIBDIR=lib64 ; \ cd $(BOEL_BINARIES_DIR) \ && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(SRC_DIR)/$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib -v -d $$TGTLIBDIR bin/* sbin/* + # # Include other files required by openssh that apparently aren't # picked up by mklibs for some reason. -BEF-
