Hi all, the following patch includes the rtorrent client (http://libtorrent.rakshasa.no/) into the initrd.img and uses it instead of the standard BitTorrent client. This would allow to remove the tricky stuff to freeze the python scripts, since it's fully implemented in C++. As reported in the rtorrent website, this client is designed with a particular attention on high performance and this is just what we really need (the standard BT client doesn't seem to be able to exploit all the available bandwidth, and in our cases choking algorithms, fairness, etc. don't seem to be very useful stuff).
Tests with a single imaging client looks very promising! I can reach more than 2x of the download rate, respect to the standard client. ok, well... 1 client is not a meaningful test for a p2p protocol... :-) I hope I'll be able to test it in a bigger environment soon... If you can/want to test it RPMs will be available soon at http://download.systemimager.org/~arighi/systemimager (look for *_rtorrent* versions). Before merging it into the trunk I would like to rewrite it better, in particular, I'd like to add an option that allows to select rtorrent or standard BitTorrent clients (like RTORRENT=y|n), to switch from a client to another without recompiling and easily do some comparisons. -Andrea Build Requirements: sigc++-2.0 libtorrent >= 0.11.1 --- Index: make.d/boel_binaries.inc =================================================================== --- make.d/boel_binaries.inc (revision 4062) +++ make.d/boel_binaries.inc (working copy) @@ -137,7 +137,7 @@ TGTLIBDIR=lib ; \ test ! -d /lib64 || TGTLIBDIR=lib64 ; \ cd $(BOEL_BINARIES_DIR) \ - && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/* + && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/usr/local/lib:/opt/gnome/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/* ifdef IS_PPC64 # Copy all the libnss*. It seems that with ppc64 these are not Index: initrd_source/make.d/rtorrent.rul =================================================================== --- initrd_source/make.d/rtorrent.rul (revision 0) +++ initrd_source/make.d/rtorrent.rul (revision 0) @@ -0,0 +1,51 @@ +# +# $Id$ +# vi: set filetype=make: +# +# 2007-05-14 Andrea Righi +# - rtorrent binaries +# + +RTORRENT_VERSION := 0.7.1 +RTORRENT_DIR := $(INITRD_SRC_DIR)/rtorrent-$(RTORRENT_VERSION) +RTORRENT_TARBALL := rtorrent-$(RTORRENT_VERSION).tar.gz +RTORRENT_URL := http://libtorrent.rakshasa.no/downloads/$(RTORRENT_TARBALL) +#RTORRENT_URL := http://download.systemimager.org/pub/bittorrent/$(RTORRENT_TARBALL) +RTORRENT_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/rtorrent.*.patch 2>/dev/null | sort) + +ALL_SOURCE += $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL) + +PHONY += rtorrent +rtorrent: $(RTORRENT_DIR).build + +PHONY += rtorrent_install +rtorrent_install: $(RTORRENT_DIR).install +$(RTORRENT_DIR).install: $(RTORRENT_DIR).build \ + $(INITRD_BUILD_DIR).prep + install -m 755 $(RTORRENT_DIR)/src/rtorrent $(INITRD_BUILD_DIR)/bin + + +$(RTORRENT_DIR).build: $(RTORRENT_DIR).unpack + cd $(RTORRENT_DIR) && ./configure --without-ncursesw --disable-debug + cd $(RTORRENT_DIR) && make + touch $@ + +$(RTORRENT_DIR).unpack: $(INITRD_DIR)/make.d/rtorrent.rul \ + $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL) \ + $(RTORRENT_PATCHES) + rm -rf $(RTORRENT_DIR) + cd $(INITRD_SRC_DIR) && tar -xvzf $(RTORRENT_TARBALL) + cd $(RTORRENT_DIR) && cat $(RTORRENT_PATCHES) < /dev/null | patch -p1 + touch $@ + +$(INITRD_SRC_DIR)/$(RTORRENT_TARBALL): + mkdir -p $(INITRD_SRC_DIR) + $(GETSOURCE) $(RTORRENT_URL) $(INITRD_SRC_DIR) + +PHONY += rtorrent_clean +rtorrent_clean: + rm -rf $(RTORRENT_DIR) + rm -f $(RTORRENT_DIR).unpack + rm -f $(RTORRENT_DIR).build + rm -f $(RTORRENT_DIR).install + Index: initrd_source/make.d/dtach.rul =================================================================== --- initrd_source/make.d/dtach.rul (revision 0) +++ initrd_source/make.d/dtach.rul (revision 0) @@ -0,0 +1,48 @@ +# +# $Id$ +# vi: set filetype=make: +# + +DTACH_VERSION := 0.7 +DTACH_DIR := $(INITRD_SRC_DIR)/dtach-$(DTACH_VERSION) +DTACH_TARBALL := dtach-$(DTACH_VERSION).tar.gz +DTACH_URL := http://switch.dl.sourceforge.net/sourceforge/dtach/$(DTACH_TARBALL) +#DTACH_URL := http://download.systemimager.org/pub/bittorrent/$(DTACH_TARBALL) +DTACH_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/dtach.*.patch 2>/dev/null | sort) + +ALL_SOURCE += $(INITRD_SRC_DIR)/$(DTACH_TARBALL) + +PHONY += dtach +dtach: $(DTACH_DIR).build + +PHONY += dtach_install +dtach_install: $(DTACH_DIR).install +$(DTACH_DIR).install: $(DTACH_DIR).build \ + $(INITRD_BUILD_DIR).prep + install -m 755 $(DTACH_DIR)/dtach $(INITRD_BUILD_DIR)/bin + + +$(DTACH_DIR).build: $(DTACH_DIR).unpack + cd $(DTACH_DIR) && ./configure + cd $(DTACH_DIR) && make + touch $@ + +$(DTACH_DIR).unpack: $(INITRD_DIR)/make.d/dtach.rul \ + $(INITRD_SRC_DIR)/$(DTACH_TARBALL) \ + $(DTACH_PATCHES) + rm -rf $(DTACH_DIR) + cd $(INITRD_SRC_DIR) && tar -xvzf $(DTACH_TARBALL) + cd $(DTACH_DIR) && cat $(DTACH_PATCHES) < /dev/null | patch -p1 + touch $@ + +$(INITRD_SRC_DIR)/$(DTACH_TARBALL): + mkdir -p $(INITRD_SRC_DIR) + $(GETSOURCE) $(DTACH_URL) $(INITRD_SRC_DIR) + +PHONY += dtach_clean +dtach_clean: + rm -rf $(DTACH_DIR) + rm -f $(DTACH_DIR).unpack + rm -f $(DTACH_DIR).build + rm -f $(DTACH_DIR).install + Index: initrd_source/initrd.rul =================================================================== --- initrd_source/initrd.rul (revision 4062) +++ initrd_source/initrd.rul (working copy) @@ -61,6 +61,8 @@ # include $(INITRD_DIR)/make.d/cx_freeze.rul include $(INITRD_DIR)/make.d/bittorrent.rul +include $(INITRD_DIR)/make.d/dtach.rul +include $(INITRD_DIR)/make.d/rtorrent.rul include $(INITRD_DIR)/make.d/busybox.rul include $(INITRD_DIR)/make.d/coreutils.rul include $(INITRD_DIR)/make.d/dhclient.rul @@ -126,6 +128,8 @@ $(DHCLIENT_DIR).install \ $(RSYNC_DIR).install \ $(BITTORRENT_DIR).install \ + $(DTACH_DIR).install \ + $(RTORRENT_DIR).install \ $(CX_FREEZE_BINARY) \ $(UDPCAST_BINARY) @@ -139,14 +143,14 @@ ifeq ($(ARCH),x86_64) ( \ cd $(INITRD_BUILD_DIR) && \ - $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib -v -d lib64 ./*bin/* \ + $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib:/usr/local/lib:/opt/gnome/lib -v -d lib64 ./*bin/* \ ) # copy libnss files as they are not automatically included cp -a /lib64/libnss* $(INITRD_BUILD_DIR)/lib64 else ( \ cd $(INITRD_BUILD_DIR) && \ - $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64 -v -d lib ./*bin/* \ + $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64:/usr/local/lib:/opt/gnome/lib -v -d lib ./*bin/* \ ) # copy libnss files as they are not automatically included cp -a /lib/libnss* $(INITRD_BUILD_DIR)/lib Index: initrd_source/skel/etc/init.d/functions =================================================================== --- initrd_source/skel/etc/init.d/functions (revision 4062) +++ initrd_source/skel/etc/init.d/functions (working copy) @@ -547,11 +547,19 @@ # Minimum upload rate threshold (in KB/s), if lesser stop seeding bittorrent_upload_min=${BITTORRENT_UPLOAD_MIN:-50} + # dtach requires devpts: be sure it's properly mounted. + # XXX: honestly, I don't know why an additional remount is needed + # here... -AR- + mount -t devpts,remount devpts /dev/pts + # Start downloading. - /bin/bittorrent-console --no_upnp --no_start_trackerless_client --max_upload_rate 0 --display_interval 1 --rerequest_interval 1 --bind ${IPADDR} --save_in ${destination} ${torrent} > $bittorrent_log & + dtach -n /tmp/socket-`basename ${torrent}` \ + /bin/rtorrent -d ${destination} \ + -o tracker_dump=${bittorrent_log},download_rate=0,upload_rate=0 \ + ${torrent} </dev/console >/dev/console 2>&1 & pid=$! if [ ! -d /proc/$pid ]; then - logmsg "error: couldn't run bittorrent-console!" + logmsg "error: couldn't run rtorrent!" shellout fi unset pid @@ -562,27 +570,9 @@ done # Checking download... + logmsg "download started..." while :; do - while :; do - status=`grep 'percent done:' $bittorrent_log | sed -ne '$p' | sed 's/percent done: *//' | sed -ne '/^[0-9]*\.[0-9]*$/p'` - [ ! -z "$status" ] && break - done - logmsg "percent done: $status %" - if [ "$status" = "100.0" ]; then - # Sleep until upload rate reaches the minimum threshold - while [ "$bittorrent_seed_wait" = "y" ]; do - sleep $bittorrent_polling_time - while :; do - upload_rate=`grep 'upload rate:' $bittorrent_log | sed -ne '$p' | sed 's/upload rate: *\([0-9]*\)\.[0-9]* .*$/\1/' | sed -ne '/^\([0-9]*\)$/p'` - [ ! -z $upload_rate ] && break - done - logmsg "upload rate: $upload_rate KB/s" - [ $upload_rate -lt $bittorrent_upload_min ] && break - done - logmsg "Download completed" - unset bittorrent_log upload_rate counter - break - fi + grep "event=completed" ${bittorrent_log} && break sleep $bittorrent_polling_time done @@ -602,10 +592,10 @@ bittorrent_stop() { # Try to kill all the BitTorrent processes counter=1 - while ps | grep bittorrent-console | grep -qv grep; do + while ps | grep rtorrent | grep -qv grep; do [ $counter -ge 10 ] && logmsg "error: couldn't kill BitTorrent client!" && shellout logmsg "killing BitTorrent client..." - killall -9 bittorrent-console + killall -9 dtach rtorrent sleep 1 counter=$(($counter + 1)) done ------------------------------------------------------------------------- This SF.net email is sponsored by DB2 Express Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now. http://sourceforge.net/powerbar/db2/ _______________________________________________ sisuite-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/sisuite-devel
