Hi all,
attached is some work I've done regarding bittorrent...
Since this seems to work and since it can be useful also for those that
are working for the bittorrent porting (in particular for Bernard) I've
decided to post this patch (based on the developer trunk).
I've used the standard bt client (written in python) both for clients
and image server, so I've included both the python interpreter and
bittorrent scripts in BOEL binaries.
Bernard is using libBt, it seems more light since it doesn't require
python, maybe it could be interesting do some tests to evaluate the
difference in performances between the two products.
To use my bittorrent patch you've to start the tracker and the first
seeder on the image server, simply using the following command:
# si_startbttrack --image_server <ip_image_server> --image <image_name>
This command creates also a single tar file for the image specified by
comdline; run with --update_image to update the tar file.
I've not written the si_stopbttrack command, even if it is very
simple... but for now you have to stop it only killing the tracker and
the seeder manually... ;-)
For the client side I don't use flamethrower, the bt client works alone
without further packages (except python, of course).
You must only define BITTORRENT_STAGING=<dir_path> as a kernel boot
parameter. In this way the bittorrent protocol is used to deploy the
image in the clients. First of all the image is downloaded in a staging
directory (the path is specified by the parameter), then it is untarred
in /a/ (the root directory of the installed fs) and the tar is removed
when the exctraction is completed.
If the client has not a lot of memory, like my test machine :-(, you can
deploy the tar file using the client disk space, for example I usually
use BITTORRENT_STAGE=/a/tmp... otherwise if you have enough RAM, to
increase performance, is better to deploy the tar file in tmpfs (for
example using /tmp - we can assume this as the default option).
If you can do some tests let me know your feedback... in particular for
Bernard that it's working hardly with bittorrent... ;-)
Best regards and happy new year!
-Andrea
diff -urN trunk/etc/autoinstallscript.template bittorrent/etc/autoinstallscript.template
--- trunk/etc/autoinstallscript.template 2006-01-02 09:40:53.671461104 +0100
+++ bittorrent/etc/autoinstallscript.template 2006-01-02 09:44:25.384275864 +0100
@@ -216,6 +216,29 @@
if [ ! -z $MONITOR_SERVER ]; then
stop_report_task 100
fi
+elif [ ! -z $BITTORRENT_STAGING ]; then
+ bittorrent_log=/tmp/bittorrent.log
+ bittorrent_polling_time=30
+ logmsg "Start downloading image using torrent ${IMAGENAME}.tar.torrent"
+ cd ${BITTORRENT_STAGING} && /usr/bin/python /usr/bin/bittorrent-console --no_start_trackerless_client --max_upload_rate
+ pid=$!
+ while :; do
+ status=`grep "percent done:" $bittorrent_log | sed -ne '$p' | sed 's/percent done: *//'`
+ [ -z "$status" ] && status=0.0
+ echo "percent done: $status %"
+ if [ "$status" = "100.0" ]; then
+ kill -9 $pid
+ unset bittorrent_log pid
+ break
+ fi
+ sleep $bittorrent_polling_time
+ done
+ unset bittorrent_polling_time
+ if [ ! -z $MONITOR_SERVER ]; then
+ stop_report_task 100
+ fi
+ logmsg "Extracting image ${IMAGENAME}.tar ..."
+ cd /a/ && tar -xvf ${BITTORRENT_STAGING}/${IMAGENAME}.tar && rm -f ${BITTORRENT_STAGING}/${IMAGENAME}.tar
else
# Use rsync
if [ $NO_LISTING ]; then
diff -urN trunk/initrd_source/skel/etc/init.d/functions bittorrent/initrd_source/skel/etc/init.d/functions
--- trunk/initrd_source/skel/etc/init.d/functions 2006-01-02 09:40:53.388504120 +0100
+++ bittorrent/initrd_source/skel/etc/init.d/functions 2006-01-02 09:44:40.177027024 +0100
@@ -193,6 +193,8 @@
echo "MONITOR_SERVER=$MONITOR_SERVER" >> /tmp/variables.txt
echo "MONITOR_PORT=$MONITOR_PORT" >> /tmp/variables.txt
echo "MONITOR_CONSOLE=$MONITOR_CONSOLE" >> /tmp/variables.txt
+
+ echo "BITTORRENT_STAGING=$BITTORRENT_STAGING" >> /tmp/variables.txt
}
#
################################################################################
diff -urN trunk/make.d/bittorrent.rul bittorrent/make.d/bittorrent.rul
--- trunk/make.d/bittorrent.rul 1970-01-01 01:00:00.000000000 +0100
+++ bittorrent/make.d/bittorrent.rul 2006-01-02 09:49:50.227892128 +0100
@@ -0,0 +1,31 @@
+#
+# 2005-12-28 Andrea Righi
+# - Bittorrent binaries
+#
+
+
+BITTORRENT_VERSION := 4.3.4
+BITTORRENT_DIR := BitTorrent-$(BITTORRENT_VERSION)
+BITTORRENT_TARBALL := BitTorrent-$(BITTORRENT_VERSION).tar.gz
+BITTORRENT_URL := http://www.bittorrent.com/dl/$(BITTORRENT_TARBALL)
+BITTORRENT_BINARY := $(SRC_DIR)/$(BITTORRENT_DIR)/bittorrent.py
+
+PHONY += bittorrent
+
+bittorrent: $(BITTORRENT_BINARY)
+
+$(BITTORRENT_BINARY): $(SRC_DIR)/$(BITTORRENT_TARBALL)
+ rm -rf $(SRC_DIR)/$(BITTORRENT_DIR)
+ cd $(SRC_DIR) && tar -xvzf $(BITTORRENT_TARBALL)
+
+$(SRC_DIR)/$(BITTORRENT_TARBALL):
+ [ -d $(SRC_DIR) ] || mkdir -p $(SRC_DIR)
+ $(GETSOURCE) $(BITTORRENT_URL) $(SRC_DIR)
+
+ALL_SOURCE += $(SRC_DIR)/$(BITTORRENT_TARBALL)
+
+PHONY += bittorrent_clean
+bittorrent_clean:
+ rm -rf $(SRC_DIR)/$(BITTORRENT_DIR)
+
+# /* vi: set noet ts=4: */
diff -urN trunk/make.d/python.rul bittorrent/make.d/python.rul
--- trunk/make.d/python.rul 1970-01-01 01:00:00.000000000 +0100
+++ bittorrent/make.d/python.rul 2006-01-02 09:48:55.530207440 +0100
@@ -0,0 +1,33 @@
+#
+# 2005-12-20 Andrea Righi
+# - Python interpreter
+#
+
+
+PYTHON_VERSION := 2.4.2
+PYTHON_DIR := Python-$(PYTHON_VERSION)
+PYTHON_TARBALL := Python-$(PYTHON_VERSION).tar.bz2
+PYTHON_URL := http://www.python.org/ftp/python/2.4.2/$(PYTHON_TARBALL)
+PYTHON_BINARY := $(SRC_DIR)/$(PYTHON_DIR)/python
+
+PHONY += python
+python: $(PYTHON_BINARY)
+$(PYTHON_BINARY): $(SRC_DIR)/$(PYTHON_TARBALL)
+ rm -rf $(SRC_DIR)/$(PYTHON_DIR)
+ cd $(SRC_DIR) && tar -xvjf $(PYTHON_TARBALL)
+ cd $(SRC_DIR)/$(PYTHON_DIR) && \
+ ./configure --prefix=/usr --disable-shared --disable-ipv6 --disable-profiling --without-tsc --without-pymalloc --disable-toolbox-glue CFLAGS="-Os"
+ export CFLAGS="-Os"
+ $(MAKE) -C $(SRC_DIR)/$(PYTHON_DIR) CFLAGS="-Os"
+
+$(SRC_DIR)/$(PYTHON_TARBALL):
+ [ -d $(SRC_DIR) ] || mkdir -p $(SRC_DIR)
+ $(GETSOURCE) $(PYTHON_URL) $(SRC_DIR)
+
+ALL_SOURCE += $(SRC_DIR)/$(PYTHON_TARBALL)
+
+PHONY += python_clean
+python_clean:
+ rm -rf $(SRC_DIR)/$(PYTHON_DIR)
+
+# /* vi: set noet ts=4: */
diff -urN trunk/Makefile bittorrent/Makefile
--- trunk/Makefile 2006-01-02 09:40:53.766446664 +0100
+++ bittorrent/Makefile 2006-01-02 09:45:23.339465336 +0100
@@ -164,7 +164,7 @@
PXE_CONF_DEST = $(ETC)/systemimager/pxelinux.cfg
BINARIES := si_mkautoinstallcd si_mkautoinstalldiskette si_mkbootmedia
-SBINARIES := si_addclients si_cpimage si_getimage si_mkdhcpserver si_mkdhcpstatic si_mkautoinstallscript si_mkbootserver si_mvimage si_pushupdate si_rmimage si_mkrsyncd_conf si_mkclientnetboot si_netbootmond si_imagemanip si_mkbootpackage si_monitor si_monitortk
+SBINARIES := si_addclients si_cpimage si_getimage si_mkdhcpserver si_mkdhcpstatic si_mkautoinstallscript si_mkbootserver si_mvimage si_pushupdate si_rmimage si_mkrsyncd_conf si_mkclientnetboot si_netbootmond si_imagemanip si_mkbootpackage si_monitor si_monitortk si_startbttrack
CLIENT_SBINARIES := si_updateclient si_prepareclient
COMMON_BINARIES = si_lsimage
@@ -314,6 +314,9 @@
$(SI_INSTALL) -d -m 755 $(FLAMETHROWER_STATE_DIR)
+ # Install server-side BitTorrent.
+ cd $(SRC_DIR)/$(BITTORRENT_DIR) && $(PYTHON) setup.py install --prefix /usr
+
# install client-only files
.PHONY: install_client
install_client: install_client_man install_client_libs
@@ -497,6 +500,8 @@
$(OPENSSH_BINARIES) \
$(OPENSSH_CONF_FILES) \
$(LVM_BINARY) \
+ $(PYTHON_BINARY) \
+ $(BITTORRENT_BINARY) \
$(SRC_DIR)/modules_build-stamp
#
# Put binaries in the boel_binaries_tarball...
@@ -535,6 +540,12 @@
#
cd $(BOEL_BINARIES_DIR)/sbin && $(foreach binary,$(shell cat $(SRC_DIR)/$(LVM_DIR)/tools/.commands),ln -s -f lvm $(binary) && ) /bin/true
+ # Python installation
+ cd $(SRC_DIR)/$(PYTHON_DIR) && export DESTDIR=$(BOEL_BINARIES_DIR) && make install
+
+ # BitTorrent installation (refquire python-devel and python-xml packages)
+ cd $(SRC_DIR)/$(BITTORRENT_DIR) && $(BOEL_BINARIES_DIR)/usr/bin/python setup.py install --prefix $(PREFIX) --root $(BOEL_BINARIES_DIR)
+
mkdir -m 755 -p $(BOEL_BINARIES_DIR)/lib
test ! -d /lib64 || mkdir -m 755 -p $(BOEL_BINARIES_DIR)/lib64
@@ -590,7 +601,7 @@
TGTLIBDIR=lib ; \
test ! -d /lib64 || TGTLIBDIR=lib64 ; \
cd $(BOEL_BINARIES_DIR) \
- && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(SRC_DIR)/$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib -v -d $$TGTLIBDIR bin/* sbin/*
+ && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(SRC_DIR)/$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/*
#
# Include other files required by openssh that apparently aren't
# picked up by mklibs for some reason. -BEF-
diff -urN trunk/sbin/si_startbttrack bittorrent/sbin/si_startbttrack
--- trunk/sbin/si_startbttrack 1970-01-01 01:00:00.000000000 +0100
+++ bittorrent/sbin/si_startbttrack 2006-01-02 09:45:37.262348736 +0100
@@ -0,0 +1,139 @@
+#!/usr/bin/perl -w
+#
+# "SystemImager"
+#
+# Copyright (C) 2005 Andrea Righi <[EMAIL PROTECTED]>
+
+use lib "USR_PREFIX/lib/systemimager/perl";
+use strict;
+use Getopt::Long;
+use vars qw($config $VERSION);
+use constant DEFAULT_TRACKER_PORT => 6969;
+
+my $VERSION = "SYSTEMIMAGER_VERSION_STRING";
+my $program_name = "si_startbttrack";
+my $version_info = << "EOF";
+$program_name (part of SystemImager) v$VERSION
+
+Copyright (C) 1999-2001 Andrea Righi <[EMAIL PROTECTED]>
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+EOF
+
+my $get_help = "\n Try \"--help\" for more options.";
+
+my $help_info = $version_info . <<"EOF";
+
+Usage: $program_name --image_server <Hostname|IP> --image <IMAGE>
+ [OPTION]...
+
+Options: (options can be presented in any order and may be abbreviated)
+ --help Display this output.
+
+ --version Display version and copyright information.
+
+ --image_server HOSTNAME | IP
+ Specify the hostname or ip of the image server.
+
+ --image IMAGENAME
+ Specify the name of the image to distribute.
+
+ --tracker_port PORT Specify an optional tracker port (default 6969).
+
+ --update_image Update image tar file.
+
+ --quiet Run image server seeder in background.
+
+Download, report bugs, and make suggestions at:
+http://systemimager.org/
+
+EOF
+
+my ($help, $version, $quiet, $update_image, $image_server, $image_name, $tracker_port);
+GetOptions(
+ "help" => \$help,
+ "version" => \$version,
+ "quiet" => \$quiet,
+ "update_image" => \$update_image,
+ "image_server=s" => \$image_server,
+ "tracker_port=s" => \$tracker_port,
+ "image=s" => \$image_name,
+) or die "$help_info";
+
+### BEGIN evaluate commad line options ###
+if ($help) {
+ print "$help_info";
+ exit(0);
+}
+
+if ($version) {
+ print "$version_info";
+ exit(0);
+}
+
+unless ($image_server) {
+ die "\n$program_name: --image_server HOSTNAME | IP must be specified.\n$get_help\n\n";
+}
+
+unless ($image_name) {
+ die "\n$program_name: --image IMAGENAME must be specified.\n$get_help\n\n";
+}
+
+unless ($tracker_port) {
+ $tracker_port = DEFAULT_TRACKER_PORT;
+}
+
+my $IMAGE_DIR = "/var/lib/systemimager/images";
+my $SCRIPTS_DIR = "/var/lib/systemimager/scripts";
+
+unless (-d "$IMAGE_DIR/$image_name") {
+ die "\n$program_name --image $image_name: not a valid image!.\n$get_help\n\n";
+}
+
+# Prepare tar file.
+print "Preparing tar file for $image_name...\n";
+if ($update_image) {
+ unlink("$IMAGE_DIR/$image_name.tar");
+}
+unless (-f "$IMAGE_DIR/$image_name.tar") {
+ system "cd $IMAGE_DIR/$image_name && tar -cf $IMAGE_DIR/$image_name.tar .";
+ if ($?) {
+ die "error: cannot create image tar file!\n";
+ }
+}
+print "done.\n";
+
+# Prepare torrent file.
+print "Preparing torrent file for $image_name...\n";
+if ($update_image) {
+ unlink("$SCRIPTS_DIR/$image_name.tar.torrent");
+}
+unless (-f "$SCRIPTS_DIR/$image_name.tar.torrent") {
+ system "maketorrent-console --target $SCRIPTS_DIR/$image_name.tar.torrent http://$image_server:$tracker_port/announce $IMAGE_DIR/$image_name.tar";
+ if ($?) {
+ die "error: cannot create torrent file!\n";
+ }
+}
+print "done.\n";
+
+# Start tracker.
+print "Starting tracker...\n";
+unlink('/tmp/dstate', '/var/log/systemimager/tracker.log');
+system "bittorrent-tracker --port $tracker_port --dfile /tmp/dstate --logfile /var/log/systemimager/tracker.log &";
+if ($?) {
+ die "error: cannot start tracker!\n";
+}
+print "done.\n";
+
+print "Starting first seeder...\n";
+if ($quiet) {
+ $quiet = '>/dev/null 2>&1 &';
+} else {
+ $quiet = '';
+}
+system "cd $IMAGE_DIR && bittorrent-console --no_start_trackerless_client --max_upload_rate 0 $SCRIPTS_DIR/$image_name.tar.torrent $quiet";
+print "done.\n";
+
+# Well done.
+exit(0);
+
