On Sun, Feb 28, 2016 at 11:49 AM, Angel Ezquerra <[email protected]> wrote: > # HG changeset patch > # User Angel Ezquerra <[email protected]> > # Date 1455366858 -3600 > # Sat Feb 13 13:34:18 2016 +0100 > # Branch stable > # Node ID 083d69152e6949ff25e2a311b19105667468fb7d > # Parent 9621195d7b42a95ee84669ff888e551bf31e4148 > repo-scan: add option to scan nested repositories > > "nested repositories" are those that are found within other repositories. An > example of these nested repositories are mercurial repositories. This change > adds an option to the admin settings page to look for nested repositories when > performing a repository scan. > > diff --git a/kallithea/controllers/admin/settings.py > b/kallithea/controllers/admin/settings.py > --- a/kallithea/controllers/admin/settings.py > +++ b/kallithea/controllers/admin/settings.py > @@ -196,8 +196,9 @@ > c.active = 'mapping' > if request.POST: > rm_obsolete = request.POST.get('destroy', False) > + find_nested = request.POST.get('find_nested', False) > install_git_hooks = request.POST.get('hooks', False) > - overwrite_git_hooks = request.POST.get('hooks_overwrite', False); > + overwrite_git_hooks = request.POST.get('hooks_overwrite', False) > invalidate_cache = request.POST.get('invalidate', False) > log.debug('rescanning repo location with destroy obsolete=%s, ' > 'install git hooks=%s and ' > @@ -208,7 +209,7 @@ > for repo in Repository.get_all(): > ScmModel().mark_for_invalidation(repo.repo_name, > delete=True) > > - filesystem_repos = ScmModel().repo_scan() > + filesystem_repos = ScmModel().repo_scan(find_nested=find_nested) > added, removed = repo2db_mapper(filesystem_repos, rm_obsolete, > > install_git_hooks=install_git_hooks, > user=c.authuser.username, > diff --git a/kallithea/lib/utils.py b/kallithea/lib/utils.py > --- a/kallithea/lib/utils.py > +++ b/kallithea/lib/utils.py > @@ -207,12 +207,15 @@ > sa.commit() > > > -def get_filesystem_repos(path, recursive=False, skip_removed_repos=True): > +def get_filesystem_repos(path, recursive=False, skip_removed_repos=True, > + find_nested=False): > """ > Scans given path for repos and return (name,(type,path)) tuple > > :param path: path to scan for repositories > :param recursive: recursive search and return names with subdirs in front > + :param find_nested: look for 'nested repositories' (e.g. subrepos). > + find_nested is ignored if recursive is false. > """ > > # remove ending slash for better results > @@ -239,16 +242,27 @@ > continue > > try: > + # try to access the cur_path as an SCM (i.e. a repository) > scm_info = get_scm(cur_path) > + # cur_path is the root of a repository > yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info > + if not find_nested: > + # do not recurse into the current repository > + continue > + # potentially recurse into the current repository (if > + # recursive is enabled), to look for nested repositories > except VCSError: > - if not recursive: > - continue > - #check if this dir containts other repos for recursive scan > - rec_path = os.path.join(p, dirpath) > - if not os.path.islink(rec_path) and os.path.isdir(rec_path): > - for inner_scm in _get_repos(rec_path): > - yield inner_scm > + # cur_path is not a repository -> do we need to recurse? > + pass > + if not recursive: > + continue > + # recurse into the current directory (or repository), looking for > + # repositories (or nested repositories) deeper down the directory > + # hierarchy > + rec_path = os.path.join(p, dirpath) > + if not os.path.islink(rec_path) and os.path.isdir(rec_path): > + for inner_scm in _get_repos(rec_path): > + yield inner_scm > > return _get_repos(path) > > diff --git a/kallithea/model/scm.py b/kallithea/model/scm.py > --- a/kallithea/model/scm.py > +++ b/kallithea/model/scm.py > @@ -260,12 +260,13 @@ > > return q.ui_value > > - def repo_scan(self, repos_path=None): > + def repo_scan(self, repos_path=None, find_nested=False): > """ > Listing of repositories in given path. This path should not be a > repository itself. Return a dictionary of repository objects > > :param repos_path: path to directory containing repositories > + :param find_nested: look for 'nested repositories' (e.g. subrepos). > """ > > if repos_path is None: > @@ -276,7 +277,8 @@ > baseui = make_ui('db') > repos = {} > > - for name, path in get_filesystem_repos(repos_path, recursive=True): > + for name, path in get_filesystem_repos(repos_path, recursive=True, > + find_nested=find_nested): > # name need to be decomposed and put back together using the / > # since this is internal storage separator for kallithea > name = Repository.normalize_repo_name(name) > diff --git a/kallithea/templates/admin/settings/settings_mapping.html > b/kallithea/templates/admin/settings/settings_mapping.html > --- a/kallithea/templates/admin/settings/settings_mapping.html > +++ b/kallithea/templates/admin/settings/settings_mapping.html > @@ -19,6 +19,12 @@ > <span class="help-block">${_('Check this to reload data > and clear cache keys for all repositories.')}</span> > > <div class="checkbox"> > + ${h.checkbox('find_nested',False)} > + <label for="find_nested">${_('Look for nested > repositories (e.g. Mercurial subrepos)')}</label> > + </div> > + <span class="help-block">${_('Check this option to look > for repositories within existing repositories (changing the default behavior > which is to not recurse into repositories during a repository scan). This > option can be useful, for example, when you use Mercurial subrepositories > (which are normally found within the parent repository working directory, and > thus will not be found unless you enable this option).')}</span> > + > + <div class="checkbox"> > ${h.checkbox('hooks',True)} > <label for="hooks"> ${_('Install Git hooks')} > </label> > </div>
V2 of this patch tries to address most of Mads' comments. I did not edit the vcs_support.rst file yet though. I could change that once I make all subrepo related changes that I have in mind, since that would change the way people could use subrepos with Kallithea. Cheers, Angel _______________________________________________ kallithea-general mailing list [email protected] http://lists.sfconservancy.org/mailman/listinfo/kallithea-general
