Sure, the updated patch is attached. - dale
On Wed, Jul 6, 2011 at 4:09 PM, Lucas Meneghel Rodrigues <[email protected]>wrote: > On Wed, Jul 6, 2011 at 3:59 PM, Dale Curtis <[email protected]> wrote: > > pbzip2 provides a linear increase in performance per core. On most of our > > machines that means a 75% speedup in a --all packaging step, reducing the > > packaging time from ~5 minutes down to a ~1m15s. > > > > Also fixes a small bug where if no exclude_dir was specified None would > > be added to the command line. > > > > Also turns off verbosity. Is anyone using this? > > > > Review link: http://gerrit.chromium.org/gerrit/#change,3567 > > > > Signed-off-by: Dale Curtis <[email protected]> > > --- > > client/common_lib/base_packages.py | 23 +++++++++++++++++++++-- > > 1 files changed, 21 insertions(+), 2 deletions(-) > > > > diff --git a/client/common_lib/base_packages.py > b/client/common_lib/base_packages.py > > index 3c2b0bb..15bcd61 100644 > > --- a/client/common_lib/base_packages.py > > +++ b/client/common_lib/base_packages.py > > @@ -14,6 +14,19 @@ from autotest_lib.client.common_lib import error, > utils, global_config > > CHECKSUM_FILE = "packages.checksum" > > > > > > +def has_pbzip2(): > > + '''Check if parallel bzip2 is available on this system.''' > > + try: > > + utils.system('which pbzip2') > > ^ After seeing one of the comments on gerrit, it occurred to me that > we could use os_dep.command('pbzip2'). os_dep.command is a simple, > python only and distro agnostic implementation of what you want to > accomplish. > > So, even though this is already applied to your tree, would you > consider changing the call to which to a os_dep.command() call? > > > + except error.CmdError, e: > > + return False > > + return True > > + > > + > > +# is parallel bzip2 available for use? > > +_PBZIP2_AVAILABLE = has_pbzip2() > > + > > + > > def parse_ssh_path(repo): > > ''' > > Parse ssh://xx@xx/path/to/ and return a tuple with host_line and > > @@ -759,10 +772,16 @@ class BasePackageManager(object): > > ''' > > tarball_path = os.path.join(dest_dir, pkg_name) > > temp_path = tarball_path + '.tmp' > > - cmd = "tar -cvjf %s -C %s %s " % (temp_path, src_dir, > exclude_string) > > + cmd_list = ['tar', '-cf', temp_path, '-C', src_dir] > > + if _PBZIP2_AVAILABLE: > > + cmd_list.append('--use-compress-prog=pbzip2') > > + else: > > + cmd_list.append('-j') > > + if exclude_string is not None: > > + cmd_list.append(exclude_string) > > > > try: > > - utils.system(cmd) > > + utils.system(' '.join(cmd_list)) > > except: > > os.unlink(temp_path) > > raise > > -- > > 1.7.3.1 > > > > _______________________________________________ > > Autotest mailing list > > [email protected] > > http://test.kernel.org/cgi-bin/mailman/listinfo/autotest > > > > > > -- > Lucas >
From 725a3e2d25a436549aa9052ea67827698ae0e17d Mon Sep 17 00:00:00 2001 From: Dale Curtis <[email protected]> Date: Wed, 6 Jul 2011 11:50:23 -0700 Subject: [PATCH] Add parallel bzip2 support to package manager. pbzip2 provides a linear increase in performance per core. On most of our machines that means a 75% speedup in a --all packaging step, reducing the packaging time from ~5 minutes down to a ~1m15s. Also fixes a small bug where if no exclude_dir was specified None would be added to the command line. Also turns off verbosity. Is anyone using this? Review link: http://gerrit.chromium.org/gerrit/#change,3567 Signed-off-by: Dale Curtis <[email protected]> --- client/common_lib/base_packages.py | 24 ++++++++++++++++++++++-- 1 files changed, 22 insertions(+), 2 deletions(-) diff --git a/client/common_lib/base_packages.py b/client/common_lib/base_packages.py index 3c2b0bb..2a60b07 100644 --- a/client/common_lib/base_packages.py +++ b/client/common_lib/base_packages.py @@ -7,6 +7,7 @@ should inherit this class. import re, os, sys, traceback, subprocess, shutil, time, traceback, urlparse import fcntl, logging +from autotest_lib.client.bin import os_dep from autotest_lib.client.common_lib import error, utils, global_config @@ -14,6 +15,19 @@ from autotest_lib.client.common_lib import error, utils, global_config CHECKSUM_FILE = "packages.checksum" +def has_pbzip2(): + '''Check if parallel bzip2 is available on this system.''' + try: + os_dep.command('pbzip2') + except ValueError: + return False + return True + + +# is parallel bzip2 available for use? +_PBZIP2_AVAILABLE = has_pbzip2() + + def parse_ssh_path(repo): ''' Parse ssh://xx@xx/path/to/ and return a tuple with host_line and @@ -759,10 +773,16 @@ class BasePackageManager(object): ''' tarball_path = os.path.join(dest_dir, pkg_name) temp_path = tarball_path + '.tmp' - cmd = "tar -cvjf %s -C %s %s " % (temp_path, src_dir, exclude_string) + cmd_list = ['tar', '-cf', temp_path, '-C', src_dir] + if _PBZIP2_AVAILABLE: + cmd_list.append('--use-compress-prog=pbzip2') + else: + cmd_list.append('-j') + if exclude_string is not None: + cmd_list.append(exclude_string) try: - utils.system(cmd) + utils.system(' '.join(cmd_list)) except: os.unlink(temp_path) raise -- 1.7.3.1
_______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
