New issue 337: index creation fails on git repo with submodules in subdirectory https://bitbucket.org/conservancy/kallithea/issues/337/index-creation-fails-on-git-repo-with
Thomas De Schampheleire: (reported on IRC by ayleph) Steps to reproduce: 1. Clone git repo http://source.salmonlabs.net/dotfiles into kallithea 0.4.0rc1 2. `kallithea-cli index-create -c development.ini --index-only dotfiles` Error: ``` Traceback (most recent call last): File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/bin/kallithea-cli", line 11, in <module> load_entry_point('Kallithea', 'console_scripts', 'kallithea-cli')() File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 764, in __call__ return self.main(*args, **kwargs) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 717, in main rv = self.invoke(ctx) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 1137, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 956, in invoke return ctx.invoke(self.callback, **ctx.params) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 555, in invoke return callback(*args, **kwargs) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_base.py", line 52, in runtime_wrapper return annotated(*args, **kwargs) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_index.py", line 59, in index_create .run(full_index=full_index) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 461, in run self.update_indexes() File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 453, in update_indexes self.update_file_index() File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 394, in update_file_index for path in self.get_paths(repo): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 136, in get_paths for _topnode, _dirs, files in cs.walk('/'): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk for tup in self.walk(dirnode.path): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk for tup in self.walk(dirnode.path): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 650, in walk yield (topnode, topnode.dirs, topnode.files) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__ value = self._func(obj) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 511, in dirs return sorted((node for node in self.nodes if node.is_dir())) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__ value = self._func(obj) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 499, in nodes nodes = self.changeset.get_nodes(self.path) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/git/changeset.py", line 418, in get_nodes cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(tree['.gitmodules'][1]).data)) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/dulwich/objects.py", line 936, in __getitem__ return self._entries[name] KeyError: '.gitmodules' ``` Investigation of the problem yields to the following info: - The root of the repo does contain a .gitmodules file, with following contents: ``` [submodule "dircolors-solarized"] path = dircolors-solarized url = https://github.com/seebi/dircolors-solarized.git [submodule "vim/bundle/vim-colors-solarized"] path = vim/bundle/vim-colors-solarized url = https://github.com/altercation/vim-colors-solarized [submodule "vim/bundle/VOoM"] path = vim/bundle/VOoM url = https://github.com/vim-voom/VOoM.git [submodule "mintty-colors-solarized"] path = mintty-colors-solarized url = https://github.com/karlin/mintty-colors-solarized ``` The problem happens with the VOoM submodule. The code is traversing the files in the repo, determines the 'git tree' for it, then trying to read the .gitmodules file from that git tree. The 'git tree' corresponds to a directory tree, and in the case of VOoM the 'tree' is rooted at 'vim/bundle', not at the root of the repo. As a result, in that subtree there is no '.gitmodules' file. Following change will start from the root tree: ``` diff --git a/kallithea/lib/vcs/backends/git/changeset.py b/kallithea/lib/vcs/backends/git/changeset.py --- a/kallithea/lib/vcs/backends/git/changeset.py +++ b/kallithea/lib/vcs/backends/git/changeset.py @@ -411,7 +412,8 @@ class GitChangeset(BaseChangeset): als = self.repository.alias for name, stat, id in tree.iteritems(): if objects.S_ISGITLINK(stat): - cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(tree['.gitmodules'][1]).data)) + root_tree = self.repository._repo[self._tree_id] + cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(root_tree['.gitmodules'][1]).data)) url = cf.get(('submodule', name), 'url') dirnodes.append(SubModuleNode(name, url=url, changeset=id, alias=als)) ``` and then the error becomes: ``` Traceback (most recent call last): File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/bin/kallithea-cli", line 11, in <module> load_entry_point('Kallithea', 'console_scripts', 'kallithea-cli')() File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 764, in __call__ return self.main(*args, **kwargs) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 717, in main rv = self.invoke(ctx) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 1137, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 956, in invoke return ctx.invoke(self.callback, **ctx.params) File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 555, in invoke return callback(*args, **kwargs) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_base.py", line 52, in runtime_wrapper return annotated(*args, **kwargs) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_index.py", line 59, in index_create .run(full_index=full_index) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 461, in run self.update_indexes() File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 453, in update_indexes self.update_file_index() File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 394, in update_file_index for path in self.get_paths(repo): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 136, in get_paths for _topnode, _dirs, files in cs.walk('/'): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk for tup in self.walk(dirnode.path): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk for tup in self.walk(dirnode.path): File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 650, in walk yield (topnode, topnode.dirs, topnode.files) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__ value = self._func(obj) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 511, in dirs return sorted((node for node in self.nodes if node.is_dir())) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__ value = self._func(obj) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 499, in nodes nodes = self.changeset.get_nodes(self.path) File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/git/changeset.py", line 417, in get_nodes url = cf.get(('submodule', name), 'url') File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/dulwich/config.py", line 240, in get return self._values[(section[0],)][name] File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/dulwich/config.py", line 87, in __getitem__ return super(CaseInsensitiveDict, self).__getitem__(key) KeyError: ('submodule',) ``` We now found the .gitmodules file, but we try to find the submodule called 'VOoM' (based on the directory name) which does not exist. More changes are thus necessary to correctly map a file path on the right entry in .gitmodules... _______________________________________________ kallithea-general mailing list [email protected] https://lists.sfconservancy.org/mailman/listinfo/kallithea-general
