On Fri, 2012-07-06 at 12:05 -0400, Don Zickus wrote:
> This adds git as a new fetcher. Git is great for fetching tarballs directly
> from
> its source tree using git-archive. This method allows us to skip packaging
> tests
> into tar.bz2 and instead grab them directly from a branch or commit or tag.
>
> Underneath the command looks like:
>
> git archive --remote=git://<git-site>/users/dzickus/autotest-tests -o don.tar
> master testsuite
>
> This would download everything under the directory testsuite into don.tar and
> autotest would
> install it locally for use.
>
> All the local control file would need to do is add a git repo using
>
> job.add_repository(git://<git-location>:[<branch>])
>
> where <branch> is assumed to be master if not specified.
>
> Tested by creating a git tree with all the autotest tests in there and using
> a local
> control file to fetch them.
Ok, looks good, minor comments follow:
> Signed-off-by: Don Zickus <dzic...@redhat.com>
> ---
> client/shared/base_packages.py | 80
> ++++++++++++++++++++++++++++++++++++++++
> 1 files changed, 80 insertions(+), 0 deletions(-)
>
> diff --git a/client/shared/base_packages.py b/client/shared/base_packages.py
> index 12e4df1..478bc77 100644
> --- a/client/shared/base_packages.py
> +++ b/client/shared/base_packages.py
> @@ -233,6 +233,84 @@ class HttpFetcher(RepositoryFetcher):
>
> package_url))
>
>
> +class GitFetcher(RepositoryFetcher):
> + #need remote_url, output file, <branch>:<file name>
> + git_archive_cmd_pattern = 'git archive --remote=%s -o %s %s'
> +
> +
> + def __init__(self, package_manager, repository_url):
> + """
> + @param repository_url: The base URL of the http repository
> + """
> +
> + #do we have branch info in the repoistory_url?
> + branch = "master"
> + match = repository_url.split(":")
> + if len(match) > 2:
> + #we have a branch
> + branch = match[2]
> + repository_url = re.sub(":" + branch, "", repository_url)
> +
> + logging.debug('GitFetcher initialized with repo=%s and branch=%s' %
> (repository_url, branch))
> + self.run_command = package_manager._run_command
> + self.url = repository_url
> + self.branch = branch
> + self.pkgmgr = package_manager
> +
> + def install_pkg_setup(self, name, fetch_dir, install):
> + if not install:
> + return (name, fetch_dir)
> +
> + pkg_name = "%s.tar" % re.sub("/","_", name)
> + fetch_path = os.path.join(fetch_dir, pkg_name)
> + return pkg_name, fetch_path
> +
> + def fetch_pkg_file(self, filename, dest_path):
> + """git is an SCM, you can download the test directly. No need to
> + fetch a bz2'd tarball file. However 'filename' is
> <type>-<name>.tar.bz2
> + break this up and only fetch <name>
> + """
> + logging.info('Fetching %s from %s to %s', filename, self.url,
> + dest_path)
> +
> + # try to retrieve the package via http
> + package_path = self.branch + " " + filename
> + try:
> + cmd = self.git_archive_cmd_pattern % (self.url, dest_path,
> package_path)
> + result = self.run_command(cmd)
> +
> + file_exists = self.run_command(
> + 'ls %s' % dest_path,
> + _run_command_dargs={'ignore_status': True}).exit_status == 0
^ This is more precisely expressed as 'test -f %s' % dest_path
> + if not file_exists:
> + logging.error('git archive failed: %s', result)
> + raise error.CmdError(cmd, result)
> +
> + logging.debug('Successfully fetched %s from %s', package_path,
> + self.url)
> + except error.CmdError:
> + # remove whatever junk was retrieved when the get failed
> + #self.run_command('rm -f %s' % git_dest_path)
^ Let's remember to remove the commented out lines
> +
> + raise error.PackageFetchError('%s not found in %s' % (filename,
> +
> package_path))
> +
> +
> + def install_pkg_post(self, filename, fetch_dir, install_dir,
> preserve_install_dir=False):
> + # untar the package into install_dir and
> + # update the checksum in that directory
> + #if not preserve_install_dir:
> + # Make sure we clean up the install_dir
> + # self.pkgmgr._run_command('rm -rf %s' % install_dir)
^ Same here
> + #unlike the normal install path, git populates a path to its test
> + #in the tarball it produces, remove that path from the install_dir
> + install_path = re.sub(filename, "", install_dir)
> + pkg_name = "%s.tar" % re.sub("/","_", filename)
> + fetch_path = os.path.join(fetch_dir, pkg_name)
> + self.pkgmgr._run_command('tar -xf %s -C %s' % (fetch_path,
> install_path))
> +
> +
> class LocalFilesystemFetcher(RepositoryFetcher):
> def __init__(self, package_manager, local_dir):
> self.run_command = package_manager._run_command
> @@ -331,6 +409,8 @@ class BasePackageManager(object):
> def get_fetcher(self, url):
> if url.startswith('http://'):
> return HttpFetcher(self, url)
> + elif url.startswith('git://'):
> + return GitFetcher(self, url)
> else:
> return LocalFilesystemFetcher(self, url)
>
_______________________________________________
Autotest mailing list
Autotest@test.kernel.org
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
