pbzip2 provides a linear increase in performance per core. On most of our machines that means a 75% speedup in a --all packaging step, reducing the packaging time from ~5 minutes down to a ~1m15s.
Also fixes a small bug where if no exclude_dir was specified None would be added to the command line. Also turns off verbosity. Is anyone using this? Review link: http://gerrit.chromium.org/gerrit/#change,3567 Signed-off-by: Dale Curtis <[email protected]> --- client/common_lib/base_packages.py | 23 +++++++++++++++++++++-- 1 files changed, 21 insertions(+), 2 deletions(-) diff --git a/client/common_lib/base_packages.py b/client/common_lib/base_packages.py index 3c2b0bb..15bcd61 100644 --- a/client/common_lib/base_packages.py +++ b/client/common_lib/base_packages.py @@ -14,6 +14,19 @@ from autotest_lib.client.common_lib import error, utils, global_config CHECKSUM_FILE = "packages.checksum" +def has_pbzip2(): + '''Check if parallel bzip2 is available on this system.''' + try: + utils.system('which pbzip2') + except error.CmdError, e: + return False + return True + + +# is parallel bzip2 available for use? +_PBZIP2_AVAILABLE = has_pbzip2() + + def parse_ssh_path(repo): ''' Parse ssh://xx@xx/path/to/ and return a tuple with host_line and @@ -759,10 +772,16 @@ class BasePackageManager(object): ''' tarball_path = os.path.join(dest_dir, pkg_name) temp_path = tarball_path + '.tmp' - cmd = "tar -cvjf %s -C %s %s " % (temp_path, src_dir, exclude_string) + cmd_list = ['tar', '-cf', temp_path, '-C', src_dir] + if _PBZIP2_AVAILABLE: + cmd_list.append('--use-compress-prog=pbzip2') + else: + cmd_list.append('-j') + if exclude_string is not None: + cmd_list.append(exclude_string) try: - utils.system(cmd) + utils.system(' '.join(cmd_list)) except: os.unlink(temp_path) raise -- 1.7.3.1 _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
