D1952: bundlespec: move computing the bundle contentops in parsebundlespec
This revision was automatically updated to reflect the committed changes. Closed by commit rHG6c7a6b04b274: bundlespec: move computing the bundle contentops in parsebundlespec (authored by lothiraldan, committed by ). REPOSITORY rHG Mercurial CHANGES SINCE LAST UPDATE https://phab.mercurial-scm.org/D1952?vs=7374&id=7383 REVISION DETAIL https://phab.mercurial-scm.org/D1952 AFFECTED FILES hgext/lfs/__init__.py mercurial/bundle2.py mercurial/commands.py mercurial/exchange.py tests/flagprocessorext.py CHANGE DETAILS diff --git a/tests/flagprocessorext.py b/tests/flagprocessorext.py --- a/tests/flagprocessorext.py +++ b/tests/flagprocessorext.py @@ -105,8 +105,8 @@ revlog.REVIDX_FLAGS_ORDER.extend(flags) # Teach exchange to use changegroup 3 -for k in exchange._bundlespeccgversions.keys(): -exchange._bundlespeccgversions[k] = b'03' +for k in exchange._bundlespeccontentopts.keys(): +exchange._bundlespeccontentopts[k]["cg.version"] = "03" # Add wrappers for addrevision, responsible to set flags depending on the # revision data contents. diff --git a/mercurial/exchange.py b/mercurial/exchange.py --- a/mercurial/exchange.py +++ b/mercurial/exchange.py @@ -52,14 +52,39 @@ 'bundle2': '02', #legacy } +# Maps bundle version with content opts to choose which part to bundle +_bundlespeccontentopts = { +'v1': { +'changegroup': True, +'cg.version': '01', +'obsolescence': False, +'phases': False, +'tagsfnodescache': False, +'revbranchcache': False +}, +'v2': { +'changegroup': True, +'cg.version': '02', +'obsolescence': False, +'phases': False, +'tagsfnodescache': True, +'revbranchcache': True +}, +'packed1' : { +'cg.version': 's1' +} +} +_bundlespeccontentopts['bundle2'] = _bundlespeccontentopts['v2'] + # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. _bundlespecv1compengines = {'gzip', 'bzip2', 'none'} @attr.s class bundlespec(object): compression = attr.ib() version = attr.ib() params = attr.ib() +contentopts = attr.ib() def parsebundlespec(repo, spec, strict=True, externalnames=False): """Parse a bundle string specification into parts. @@ -178,12 +203,15 @@ _('missing support for repository features: %s') % ', '.join(sorted(missingreqs))) +# Compute contentopts based on the version +contentopts = _bundlespeccontentopts.get(version, {}).copy() + if not externalnames: engine = util.compengines.forbundlename(compression) compression = engine.bundletype()[1] version = _bundlespeccgversions[version] -return bundlespec(compression, version, params) +return bundlespec(compression, version, params, contentopts) def readbundle(ui, fh, fname, vfs=None): header = changegroup.readexactly(fh, 4) diff --git a/mercurial/commands.py b/mercurial/commands.py --- a/mercurial/commands.py +++ b/mercurial/commands.py @@ -1204,7 +1204,7 @@ raise error.Abort(pycompat.bytestr(e), hint=_("see 'hg help bundlespec' for supported " "values for --type")) -cgversion = bundlespec.version +cgversion = bundlespec.contentopts["cg.version"] # Packed bundles are a pseudo bundle format for now. if cgversion == 's1': @@ -1267,14 +1267,15 @@ if complevel is not None: compopts['level'] = complevel - -contentopts = {'cg.version': cgversion, 'changegroup': True} +# Allow overriding the bundling of obsmarker in phases through +# configuration while we don't have a bundle version that include them if repo.ui.configbool('experimental', 'evolution.bundle-obsmarker'): -contentopts['obsolescence'] = True +bundlespec.contentopts['obsolescence'] = True if repo.ui.configbool('experimental', 'bundle-phases'): -contentopts['phases'] = True +bundlespec.contentopts['phases'] = True + bundle2.writenewbundle(ui, repo, 'bundle', fname, bversion, outgoing, - contentopts, compression=bcompression, + bundlespec.contentopts, compression=bcompression, compopts=compopts) @command('cat', diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py --- a/mercurial/bundle2.py +++ b/mercurial/bundle2.py @@ -1596,8 +1596,11 @@ outgoing.missingheads): part.addparam('targetphase', '%d' % phases.secret, mandatory=False) -addparttagsfnodescache(repo, bundler, outgoing) -addpartrevbranchcache(repo, bundler, outgoing) +if opts.get('tagsfnodescache', True): +addparttagsfnodescache(repo, bundler, outgoing) + +if opts.get('revbranchcache', True): +addpartrevbranchcac
D1952: bundlespec: move computing the bundle contentops in parsebundlespec
indygreg accepted this revision. indygreg added a comment. This revision is now accepted and ready to land. Thanks for following up. This looks great! REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D1952 To: lothiraldan, #hg-reviewers, indygreg Cc: indygreg, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D1952: bundlespec: move computing the bundle contentops in parsebundlespec
lothiraldan updated this revision to Diff 7374. REPOSITORY rHG Mercurial CHANGES SINCE LAST UPDATE https://phab.mercurial-scm.org/D1952?vs=5151&id=7374 REVISION DETAIL https://phab.mercurial-scm.org/D1952 AFFECTED FILES hgext/lfs/__init__.py mercurial/bundle2.py mercurial/commands.py mercurial/exchange.py tests/flagprocessorext.py CHANGE DETAILS diff --git a/tests/flagprocessorext.py b/tests/flagprocessorext.py --- a/tests/flagprocessorext.py +++ b/tests/flagprocessorext.py @@ -105,8 +105,8 @@ revlog.REVIDX_FLAGS_ORDER.extend(flags) # Teach exchange to use changegroup 3 -for k in exchange._bundlespeccgversions.keys(): -exchange._bundlespeccgversions[k] = b'03' +for k in exchange._bundlespeccontentopts.keys(): +exchange._bundlespeccontentopts[k]["cg.version"] = "03" # Add wrappers for addrevision, responsible to set flags depending on the # revision data contents. diff --git a/mercurial/exchange.py b/mercurial/exchange.py --- a/mercurial/exchange.py +++ b/mercurial/exchange.py @@ -52,14 +52,39 @@ 'bundle2': '02', #legacy } +# Maps bundle version with content opts to choose which part to bundle +_bundlespeccontentopts = { +'v1': { +'changegroup': True, +'cg.version': '01', +'obsolescence': False, +'phases': False, +'tagsfnodescache': False, +'revbranchcache': False +}, +'v2': { +'changegroup': True, +'cg.version': '02', +'obsolescence': False, +'phases': False, +'tagsfnodescache': True, +'revbranchcache': True +}, +'packed1' : { +'cg.version': 's1' +} +} +_bundlespeccontentopts['bundle2'] = _bundlespeccontentopts['v2'] + # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. _bundlespecv1compengines = {'gzip', 'bzip2', 'none'} @attr.s class bundlespec(object): compression = attr.ib() version = attr.ib() params = attr.ib() +contentopts = attr.ib() def parsebundlespec(repo, spec, strict=True, externalnames=False): """Parse a bundle string specification into parts. @@ -178,12 +203,15 @@ _('missing support for repository features: %s') % ', '.join(sorted(missingreqs))) +# Compute contentopts based on the version +contentopts = _bundlespeccontentopts.get(version, {}).copy() + if not externalnames: engine = util.compengines.forbundlename(compression) compression = engine.bundletype()[1] version = _bundlespeccgversions[version] -return bundlespec(compression, version, params) +return bundlespec(compression, version, params, contentopts) def readbundle(ui, fh, fname, vfs=None): header = changegroup.readexactly(fh, 4) diff --git a/mercurial/commands.py b/mercurial/commands.py --- a/mercurial/commands.py +++ b/mercurial/commands.py @@ -1203,7 +1203,7 @@ raise error.Abort(pycompat.bytestr(e), hint=_("see 'hg help bundlespec' for supported " "values for --type")) -cgversion = bundlespec.version +cgversion = bundlespec.contentopts["cg.version"] # Packed bundles are a pseudo bundle format for now. if cgversion == 's1': @@ -1266,14 +1266,15 @@ if complevel is not None: compopts['level'] = complevel - -contentopts = {'cg.version': cgversion, 'changegroup': True} +# Allow overriding the bundling of obsmarker in phases through +# configuration while we don't have a bundle version that include them if repo.ui.configbool('experimental', 'evolution.bundle-obsmarker'): -contentopts['obsolescence'] = True +bundlespec.contentopts['obsolescence'] = True if repo.ui.configbool('experimental', 'bundle-phases'): -contentopts['phases'] = True +bundlespec.contentopts['phases'] = True + bundle2.writenewbundle(ui, repo, 'bundle', fname, bversion, outgoing, - contentopts, compression=bcompression, + bundlespec.contentopts, compression=bcompression, compopts=compopts) @command('cat', diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py --- a/mercurial/bundle2.py +++ b/mercurial/bundle2.py @@ -1596,8 +1596,11 @@ outgoing.missingheads): part.addparam('targetphase', '%d' % phases.secret, mandatory=False) -addparttagsfnodescache(repo, bundler, outgoing) -addpartrevbranchcache(repo, bundler, outgoing) +if opts.get('tagsfnodescache', True): +addparttagsfnodescache(repo, bundler, outgoing) + +if opts.get('revbranchcache', True): +addpartrevbranchcache(repo, bundler, outgoing) if opts.get('obsolescence', False): obsmarkers = repo.obsstore.relevantmarkers(outgoing.missing) diff --git a/hgext/lfs/__init_
D1952: bundlespec: move computing the bundle contentops in parsebundlespec
lothiraldan added a comment. In https://phab.mercurial-scm.org/D1952#38770, @indygreg wrote: > I like where this is going. > > It's worth noting that at some point this will reinvent the //capabilities// mechanism of bundle2. Over the wire protocol today, the client submits its bundle2 capabilities and the server emits parts based on the peer's advertised bundle2 feature support. If you squint, this looks a lot like //content options//. Have you given any consideration to merging the two concepts and having e.g. a bundlespec map to a pre-defined set of bundle2 capabilities? Both concepts seem intimately intricated but I was afraid of the complexity of the series that actually merge them and all the potential backward-incompatible implications. Do you have a specific design in mind we could iterate on? I will update this series according to your feedbacks. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D1952 To: lothiraldan, #hg-reviewers, indygreg Cc: indygreg, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D1952: bundlespec: move computing the bundle contentops in parsebundlespec
indygreg added a comment. I took a bit of a deeper look and found a few more things. Again, I like the direction of this patch. I think our goal for bundle2 generation should be to derive a set of options for its parts up front as quick as possible and pass that data structure into the lower-level bundle2 generation mechanism. INLINE COMMENTS > exchange.py:49-51 > +# Maps bundle version with content opts to choose which part to bundle > +_bundlespeccontentops = { > +'v1': { I *really* like this and this is the direction we need to go. i.e. a bundlespec base version should imply a set of options for generation of the bundle. > exchange.py:89 > > Returns a 3-tuple of (compression, version, parameters). Compression will > be ``None`` if not in strict mode and a compression isn't defined. This comment needs updated. > exchange.py:193 > + > +return compression, version, params, contentops > At the point we're returning yet another element, it might be worth converting the return into an `attr`-derived class. We're starting to use those a bit to define data structures that are too large for tuples but don't yet merit a normal class. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D1952 To: lothiraldan, #hg-reviewers, indygreg Cc: indygreg, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D1952: bundlespec: move computing the bundle contentops in parsebundlespec
indygreg requested changes to this revision. indygreg added a comment. This revision now requires changes to proceed. I like where this is going. It's worth noting that at some point this will reinvent the //capabilities// mechanism of bundle2. Over the wire protocol today, the client submits its bundle2 capabilities and the server emits parts based on the peer's advertised bundle2 feature support. If you squint, this looks a lot like //content options//. Have you given any consideration to merging the two concepts and having e.g. a bundlespec map to a pre-defined set of bundle2 capabilities? INLINE COMMENTS > exchange.py:191 > +# Set the cg.version > +contentops["cg.version"] = version > + `version` here does not refer to the changegroup version but rather the bundlespec version. IMO the changegroup version should be implied by the bundlespec version. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D1952 To: lothiraldan, #hg-reviewers, indygreg Cc: indygreg, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D1952: bundlespec: move computing the bundle contentops in parsebundlespec
lothiraldan updated this revision to Diff 5151. REPOSITORY rHG Mercurial CHANGES SINCE LAST UPDATE https://phab.mercurial-scm.org/D1952?vs=5024&id=5151 REVISION DETAIL https://phab.mercurial-scm.org/D1952 AFFECTED FILES mercurial/commands.py mercurial/exchange.py CHANGE DETAILS diff --git a/mercurial/exchange.py b/mercurial/exchange.py --- a/mercurial/exchange.py +++ b/mercurial/exchange.py @@ -46,6 +46,20 @@ 'bundle2': '02', #legacy } +# Maps bundle version with content opts to choose which part to bundle +_bundlespeccontentops = { +'v1': { +'changegroup': True, +'obsolescence': False, +'phases': False +}, +'v2': { +'changegroup': True, +'obsolescence': False, +'phases': False +} +} + # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. _bundlespecv1compengines = {'gzip', 'bzip2', 'none'} @@ -165,11 +179,18 @@ _('missing support for repository features: %s') % ', '.join(sorted(missingreqs))) +# Compute contentops based on the version +contentops = _bundlespeccontentops.get(version, {}).copy() + if not externalnames: engine = util.compengines.forbundlename(compression) compression = engine.bundletype()[1] version = _bundlespeccgversions[version] -return compression, version, params + +# Set the cg.version +contentops["cg.version"] = version + +return compression, version, params, contentops def readbundle(ui, fh, fname, vfs=None): header = changegroup.readexactly(fh, 4) @@ -2107,7 +2128,7 @@ # component of the BUNDLESPEC. if key == 'BUNDLESPEC': try: -comp, version, params = parsebundlespec(repo, value, +comp, version, params, _ = parsebundlespec(repo, value, externalnames=True) attrs['COMPRESSION'] = comp attrs['VERSION'] = version @@ -2135,7 +2156,7 @@ spec = entry.get('BUNDLESPEC') if spec: try: -comp, version, params = parsebundlespec(repo, spec, strict=True) +comp, version, params, _ = parsebundlespec(repo, spec, strict=True) # If a stream clone was requested, filter out non-streamclone # entries. diff --git a/mercurial/commands.py b/mercurial/commands.py --- a/mercurial/commands.py +++ b/mercurial/commands.py @@ -1189,7 +1189,7 @@ bundletype = opts.get('type', 'bzip2').lower() try: -bcompression, cgversion, params = exchange.parsebundlespec( +bcompression, cgversion, params, contentopts = exchange.parsebundlespec( repo, bundletype, strict=False) except error.UnsupportedBundleSpecification as e: raise error.Abort(str(e), @@ -1256,12 +1256,13 @@ if complevel is not None: compopts['level'] = complevel - -contentopts = {'cg.version': cgversion, 'changegroup': True} +# Allow overriding the bundling of obsmarker in phases through +# configuration while we don't have a bundle version that include them if repo.ui.configbool('experimental', 'evolution.bundle-obsmarker'): contentopts['obsolescence'] = True if repo.ui.configbool('experimental', 'bundle-phases'): contentopts['phases'] = True + bundle2.writenewbundle(ui, repo, 'bundle', fname, bversion, outgoing, contentopts, compression=bcompression, compopts=compopts) To: lothiraldan, #hg-reviewers Cc: mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D1952: bundlespec: move computing the bundle contentops in parsebundlespec
lothiraldan created this revision. Herald added a subscriber: mercurial-devel. Herald added a reviewer: hg-reviewers. REVISION SUMMARY We will introduce a new bundlespec for stream bundle which will influence the contentops. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D1952 AFFECTED FILES mercurial/commands.py mercurial/exchange.py CHANGE DETAILS diff --git a/mercurial/exchange.py b/mercurial/exchange.py --- a/mercurial/exchange.py +++ b/mercurial/exchange.py @@ -46,6 +46,20 @@ 'bundle2': '02', #legacy } +# Maps bundle version with content opts to choose which part to bundle +_bundlespeccontentops = { +'v1': { +'changegroup': True, +'obsolescence': False, +'phases': False +}, +'v2': { +'changegroup': True, +'obsolescence': False, +'phases': False +} +} + # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. _bundlespecv1compengines = {'gzip', 'bzip2', 'none'} @@ -165,11 +179,18 @@ _('missing support for repository features: %s') % ', '.join(sorted(missingreqs))) +# Compute contentops based on the version +contentops = _bundlespeccontentops.get(version, {}).copy() + if not externalnames: engine = util.compengines.forbundlename(compression) compression = engine.bundletype()[1] version = _bundlespeccgversions[version] -return compression, version, params + +# Set the cg.version +contentops["cg.version"] = version + +return compression, version, params, contentops def readbundle(ui, fh, fname, vfs=None): header = changegroup.readexactly(fh, 4) @@ -2107,7 +2128,7 @@ # component of the BUNDLESPEC. if key == 'BUNDLESPEC': try: -comp, version, params = parsebundlespec(repo, value, +comp, version, params, _ = parsebundlespec(repo, value, externalnames=True) attrs['COMPRESSION'] = comp attrs['VERSION'] = version @@ -2135,7 +2156,7 @@ spec = entry.get('BUNDLESPEC') if spec: try: -comp, version, params = parsebundlespec(repo, spec, strict=True) +comp, version, params, _ = parsebundlespec(repo, spec, strict=True) # If a stream clone was requested, filter out non-streamclone # entries. diff --git a/mercurial/commands.py b/mercurial/commands.py --- a/mercurial/commands.py +++ b/mercurial/commands.py @@ -1189,7 +1189,7 @@ bundletype = opts.get('type', 'bzip2').lower() try: -bcompression, cgversion, params = exchange.parsebundlespec( +bcompression, cgversion, params, contentopts = exchange.parsebundlespec( repo, bundletype, strict=False) except error.UnsupportedBundleSpecification as e: raise error.Abort(str(e), @@ -1256,12 +1256,13 @@ if complevel is not None: compopts['level'] = complevel - -contentopts = {'cg.version': cgversion, 'changegroup': True} +# Allow overriding the bundling of obsmarker in phases through +# configuration while we don't have a bundle version that include them if repo.ui.configbool('experimental', 'evolution.bundle-obsmarker'): contentopts['obsolescence'] = True if repo.ui.configbool('experimental', 'bundle-phases'): contentopts['phases'] = True + bundle2.writenewbundle(ui, repo, 'bundle', fname, bversion, outgoing, contentopts, compression=bcompression, compopts=compopts) To: lothiraldan, #hg-reviewers Cc: mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel