D4895: narrow: when widening, don't include manifests the client already has
yuja added a comment. >> narrowwirepeer.py:15 > > hg, > > match as matchmod, > > narrowspec, > > removed this in flight to make pyflakes happy. So did I. :) REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4895 To: martinvonz, durin42, #hg-reviewers, pulkit Cc: yuja, pulkit, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: D4895: narrow: when widening, don't include manifests the client already has
> > narrowwirepeer.py:15 > > hg, > > match as matchmod, > > narrowspec, > > removed this in flight to make pyflakes happy. So did I. :) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4895: narrow: when widening, don't include manifests the client already has
pulkit added inline comments. INLINE COMMENTS > narrowwirepeer.py:15 > hg, > match as matchmod, > narrowspec, removed this in flight to make pyflakes happy. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4895 To: martinvonz, durin42, #hg-reviewers, pulkit Cc: pulkit, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4895: narrow: when widening, don't include manifests the client already has
This revision was automatically updated to reflect the committed changes.
Closed by commit rHG89cba88e95ed: narrow: when widening, dont include
manifests the client already has (authored by martinvonz, committed by ).
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4895?vs=12217=12237
REVISION DETAIL
https://phab.mercurial-scm.org/D4895
AFFECTED FILES
hgext/narrow/narrowbundle2.py
hgext/narrow/narrowwirepeer.py
mercurial/bundle2.py
mercurial/changegroup.py
mercurial/exchange.py
CHANGE DETAILS
diff --git a/mercurial/exchange.py b/mercurial/exchange.py
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -2153,14 +2153,12 @@
if kwargs.get(r'narrow', False):
include = sorted(filter(bool, kwargs.get(r'includepats', [])))
exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
-filematcher = narrowspec.match(repo.root, include=include,
- exclude=exclude)
+matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
else:
-filematcher = None
+matcher = None
cgstream = changegroup.makestream(repo, outgoing, version, source,
- bundlecaps=bundlecaps,
- filematcher=filematcher)
+ bundlecaps=bundlecaps, matcher=matcher)
part = bundler.newpart('changegroup', data=cgstream)
if cgversions:
diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -727,14 +727,17 @@
progress.complete()
class cgpacker(object):
-def __init__(self, repo, filematcher, version,
+def __init__(self, repo, oldmatcher, matcher, version,
builddeltaheader, manifestsend,
forcedeltaparentprev=False,
bundlecaps=None, ellipses=False,
shallow=False, ellipsisroots=None, fullnodes=None):
"""Given a source repo, construct a bundler.
-filematcher is a matcher that matches on files to include in the
+oldmatcher is a matcher that matches on files the client already has.
+These will not be included in the changegroup.
+
+matcher is a matcher that matches on files to include in the
changegroup. Used to facilitate sparse changegroups.
forcedeltaparentprev indicates whether delta parents must be against
@@ -761,8 +764,10 @@
ellipsis because for very large histories we expect this to be
significantly smaller.
"""
-assert filematcher
-self._filematcher = filematcher
+assert oldmatcher
+assert matcher
+self._oldmatcher = oldmatcher
+self._matcher = matcher
self.version = version
self._forcedeltaparentprev = forcedeltaparentprev
@@ -1027,7 +1032,7 @@
tree, nodes = tmfnodes.popitem()
store = mfl.getstorage(tree)
-if not self._filematcher.visitdir(store.tree[:-1] or '.'):
+if not self._matcher.visitdir(store.tree[:-1] or '.'):
# No nodes to send because this directory is out of
# the client's view of the repository (probably
# because of narrow clones).
@@ -1051,7 +1056,16 @@
fullclnodes=self._fullclnodes,
precomputedellipsis=self._precomputedellipsis)
-yield tree, deltas
+if not self._oldmatcher.visitdir(store.tree[:-1] or '.'):
+yield tree, deltas
+else:
+# 'deltas' is a generator and we need to consume it even if
+# we are not going to send it because a side-effect is that
+# it updates tmdnodes (via lookupfn)
+for d in deltas:
+pass
+if not tree:
+yield tree, []
def _prunemanifests(self, store, nodes, commonrevs):
# This is split out as a separate method to allow filtering
@@ -1066,7 +1080,8 @@
# The 'source' parameter is useful for extensions
def generatefiles(self, changedfiles, commonrevs, source,
mfdicts, fastpathlinkrev, fnodes, clrevs):
-changedfiles = list(filter(self._filematcher, changedfiles))
+changedfiles = [f for f in changedfiles
+if self._matcher(f) and not self._oldmatcher(f)]
if not fastpathlinkrev:
def normallinknodes(unused, fname):
@@ -1151,12 +1166,13 @@
progress.complete()
-def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
- shallow=False, ellipsisroots=None, fullnodes=None):
+def _makecg1packer(repo, oldmatcher, matcher, bundlecaps,
+ ellipses=False, shallow=False, ellipsisroots=None,
+ fullnodes=None):
builddeltaheader
D4895: narrow: when widening, don't include manifests the client already has
pulkit accepted this revision. pulkit added a comment. Many thanks for investing time and efforts into this. \o/ INLINE COMMENTS > changegroup.py:1065 > +# it updates tmdnodes (via lookupfn) > +for d in deltas: > +pass maybe, we can prevent walking the deltas in case of trees because if the root does not match, the rest of tree won't match. I will try to test this and send a patch if that works. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4895 To: martinvonz, durin42, #hg-reviewers, pulkit Cc: pulkit, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4895: narrow: when widening, don't include manifests the client already has
martinvonz added a comment. In https://phab.mercurial-scm.org/D4895#76913, @martinvonz wrote: > In https://phab.mercurial-scm.org/D4895#76912, @pulkit wrote: > > > I have tested this and it works well with our internal repo too. Thanks! > > > > We already have some tests and none fails, do you want to add more tests or I can queue this? > > > I should at least update the commit message. I'll try to do that ~now. Done. I also added an extra test. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4895 To: martinvonz, durin42, #hg-reviewers Cc: pulkit, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4895: narrow: when widening, don't include manifests the client already has
martinvonz updated this revision to Diff 12217.
martinvonz edited the summary of this revision.
martinvonz retitled this revision from "narrow: don't include manifests the
client already has" to "narrow: when widening, don't include manifests the
client already has".
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4895?vs=12184=12217
REVISION DETAIL
https://phab.mercurial-scm.org/D4895
AFFECTED FILES
hgext/narrow/narrowbundle2.py
hgext/narrow/narrowwirepeer.py
mercurial/bundle2.py
mercurial/changegroup.py
mercurial/exchange.py
CHANGE DETAILS
diff --git a/mercurial/exchange.py b/mercurial/exchange.py
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -2145,14 +2145,12 @@
if kwargs.get(r'narrow', False):
include = sorted(filter(bool, kwargs.get(r'includepats', [])))
exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
-filematcher = narrowspec.match(repo.root, include=include,
- exclude=exclude)
+matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
else:
-filematcher = None
+matcher = None
cgstream = changegroup.makestream(repo, outgoing, version, source,
- bundlecaps=bundlecaps,
- filematcher=filematcher)
+ bundlecaps=bundlecaps, matcher=matcher)
part = bundler.newpart('changegroup', data=cgstream)
if cgversions:
diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -727,14 +727,17 @@
progress.complete()
class cgpacker(object):
-def __init__(self, repo, filematcher, version,
+def __init__(self, repo, oldmatcher, matcher, version,
builddeltaheader, manifestsend,
forcedeltaparentprev=False,
bundlecaps=None, ellipses=False,
shallow=False, ellipsisroots=None, fullnodes=None):
"""Given a source repo, construct a bundler.
-filematcher is a matcher that matches on files to include in the
+oldmatcher is a matcher that matches on files the client already has.
+These will not be included in the changegroup.
+
+matcher is a matcher that matches on files to include in the
changegroup. Used to facilitate sparse changegroups.
forcedeltaparentprev indicates whether delta parents must be against
@@ -761,8 +764,10 @@
ellipsis because for very large histories we expect this to be
significantly smaller.
"""
-assert filematcher
-self._filematcher = filematcher
+assert oldmatcher
+assert matcher
+self._oldmatcher = oldmatcher
+self._matcher = matcher
self.version = version
self._forcedeltaparentprev = forcedeltaparentprev
@@ -1027,7 +1032,7 @@
tree, nodes = tmfnodes.popitem()
store = mfl.getstorage(tree)
-if not self._filematcher.visitdir(store.tree[:-1] or '.'):
+if not self._matcher.visitdir(store.tree[:-1] or '.'):
# No nodes to send because this directory is out of
# the client's view of the repository (probably
# because of narrow clones).
@@ -1051,7 +1056,16 @@
fullclnodes=self._fullclnodes,
precomputedellipsis=self._precomputedellipsis)
-yield tree, deltas
+if not self._oldmatcher.visitdir(store.tree[:-1] or '.'):
+yield tree, deltas
+else:
+# 'deltas' is a generator and we need to consume it even if
+# we are not going to send it because a side-effect is that
+# it updates tmdnodes (via lookupfn)
+for d in deltas:
+pass
+if not tree:
+yield tree, []
def _prunemanifests(self, store, nodes, commonrevs):
# This is split out as a separate method to allow filtering
@@ -1066,7 +1080,8 @@
# The 'source' parameter is useful for extensions
def generatefiles(self, changedfiles, commonrevs, source,
mfdicts, fastpathlinkrev, fnodes, clrevs):
-changedfiles = list(filter(self._filematcher, changedfiles))
+changedfiles = [f for f in changedfiles
+if self._matcher(f) and not self._oldmatcher(f)]
if not fastpathlinkrev:
def normallinknodes(unused, fname):
@@ -1151,12 +1166,13 @@
progress.complete()
-def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
- shallow=False, ellipsisroots=None, fullnodes=None):
+def _makecg1packer(repo, oldmatcher, matcher, bundlecaps,
+ ellipses=False, shallow=False, ellipsisroots=None,
+
