Re: [PATCH 1 of 3] contrib: add a codemod script to write coreconfigitem

[Augie Fackler]

On Wed, Jul 12, 2017 at 06:22:46PM -0700, Jun Wu wrote:
> # HG changeset patch
> # User Jun Wu 
> # Date 1499891115 25200
> #  Wed Jul 12 13:25:15 2017 -0700
> # Node ID 695702ea1caedaeeed9a6d63473c0e338adf35f4
> # Parent  26e4ba058215e536d3827befbea99ff6203d35f8
> # Available At https://bitbucket.org/quark-zju/hg-draft
> #  hg pull https://bitbucket.org/quark-zju/hg-draft -r 
> 695702ea1cae
> contrib: add a codemod script to write coreconfigitem
>
> The coreconfigitem migration seems possible to be automatized. I have tried
> RedBaron [1] which seems easy to use and suitable for this usecase. The
> script is kept in contrib in case we want to re-run it in the future to
> cover new config options. [web] section is ignored now since its usage of
> config is a bit weird and we will have issues like check-config complaining
> undocumented web.* after codemod.
>
> Note that the script only works for core hg code and does not work for
> extension code now (it writes to mercurial/configitems.py). This gives other
> people a chance to learn this area and improve the codemod script.
>
> [1]: http://redbaron.pycqa.org/
>
> diff --git a/contrib/codemod/codemod_configitems.py 
> b/contrib/codemod/codemod_configitems.py
> new file mode 100755
> --- /dev/null
> +++ b/contrib/codemod/codemod_configitems.py

This is going to seem highly nitpicky, but can we just call this
"contrib/codemod/configitems.py" instead of stuttering the codemod
part?

> @@ -0,0 +1,182 @@
> +#!/usr/bin/env python
> +# codemod_configitems.py - codemod tool to fill configitems
> +#
> +# Copyright 2017 Facebook, Inc.
> +#
> +# This software may be used and distributed according to the terms of the
> +# GNU General Public License version 2 or any later version.
> +from __future__ import absolute_import, print_function
> +
> +import os
> +import sys
> +
> +import redbaron
> +
> +def readpath(path):
> +with open(path) as f:
> +return f.read()
> +
> +def writepath(path, content):
> +with open(path, 'w') as f:
> +f.write(content)
> +
> +_configmethods = {'config', 'configbool', 'configint', 'configbytes',
> +  'configlist', 'configdate'}
> +
> +def extractstring(rnode):
> +"""get the string from a RedBaron string or call_argument node"""
> +while rnode.type != 'string':
> +rnode = rnode.value
> +return rnode.value[1:-1]  # unquote, "'str'" -> "str"
> +
> +def uiconfigitems(red):
> +"""match *.ui.config* pattern, yield (node, method, args, section, 
> name)"""
> +for node in red.find_all('atomtrailers'):
> +entry = None
> +try:
> +obj = node[-3].value
> +method = node[-2].value
> +args = node[-1]
> +section = args[0].value
> +name = args[1].value
> +if (obj in ('ui', 'self') and method in _configmethods
> +and section.type == 'string' and name.type == 'string'):
> +entry = (node, method, args, extractstring(section),
> + extractstring(name))
> +except Exception:
> +pass
> +else:
> +if entry:
> +yield entry
> +
> +def coreconfigitems(red):
> +"""match coreconfigitem(...) pattern, yield (node, args, section, 
> name)"""
> +for node in red.find_all('atomtrailers'):
> +entry = None
> +try:
> +args = node[1]
> +section = args[0].value
> +name = args[1].value
> +if (node[0].value == 'coreconfigitem' and section.type == 
> 'string'
> +and name.type == 'string'):
> +entry = (node, args, extractstring(section),
> + extractstring(name))
> +except Exception:
> +pass
> +else:
> +if entry:
> +yield entry
> +
> +def registercoreconfig(cfgred, section, name, defaultrepr):
> +"""insert coreconfigitem to cfgred AST
> +
> +section and name are plain string, defaultrepr is a string
> +"""
> +# find a place to insert the "coreconfigitem" item
> +entries = list(coreconfigitems(cfgred))
> +for node, args, nodesection, nodename in reversed(entries):
> +if (nodesection, nodename) < (section, name):
> +# insert after this entry
> +node.insert_after(
> +'coreconfigitem(%r, %r,\n'
> +'default=%s,\n'
> +')' % (section, name, defaultrepr))
> +return
> +
> +def main(argv):
> +if not argv:
> +print('Usage: codemod_configitems.py FILES\n'
> +  'For example, FILES could be "{hgext,mercurial}/*/**.py"')
> +dirname = os.path.dirname
> +reporoot = dirname(dirname(dirname(os.path.abspath(__file__
> +
> +# register configitems to this destination
> +cfgpath = os.path.join(reporoot, 'mercurial', 'configitems.py')
> +cfgred = 

[PATCH 1 of 3] contrib: add a codemod script to write coreconfigitem

[Jun Wu]

# HG changeset patch
# User Jun Wu 
# Date 1499891115 25200
#  Wed Jul 12 13:25:15 2017 -0700
# Node ID 695702ea1caedaeeed9a6d63473c0e338adf35f4
# Parent  26e4ba058215e536d3827befbea99ff6203d35f8
# Available At https://bitbucket.org/quark-zju/hg-draft
#  hg pull https://bitbucket.org/quark-zju/hg-draft -r 695702ea1cae
contrib: add a codemod script to write coreconfigitem

The coreconfigitem migration seems possible to be automatized. I have tried
RedBaron [1] which seems easy to use and suitable for this usecase. The
script is kept in contrib in case we want to re-run it in the future to
cover new config options. [web] section is ignored now since its usage of
config is a bit weird and we will have issues like check-config complaining
undocumented web.* after codemod.

Note that the script only works for core hg code and does not work for
extension code now (it writes to mercurial/configitems.py). This gives other
people a chance to learn this area and improve the codemod script.

[1]: http://redbaron.pycqa.org/

diff --git a/contrib/codemod/codemod_configitems.py 
b/contrib/codemod/codemod_configitems.py
new file mode 100755
--- /dev/null
+++ b/contrib/codemod/codemod_configitems.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+# codemod_configitems.py - codemod tool to fill configitems
+#
+# Copyright 2017 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+from __future__ import absolute_import, print_function
+
+import os
+import sys
+
+import redbaron
+
+def readpath(path):
+with open(path) as f:
+return f.read()
+
+def writepath(path, content):
+with open(path, 'w') as f:
+f.write(content)
+
+_configmethods = {'config', 'configbool', 'configint', 'configbytes',
+  'configlist', 'configdate'}
+
+def extractstring(rnode):
+"""get the string from a RedBaron string or call_argument node"""
+while rnode.type != 'string':
+rnode = rnode.value
+return rnode.value[1:-1]  # unquote, "'str'" -> "str"
+
+def uiconfigitems(red):
+"""match *.ui.config* pattern, yield (node, method, args, section, name)"""
+for node in red.find_all('atomtrailers'):
+entry = None
+try:
+obj = node[-3].value
+method = node[-2].value
+args = node[-1]
+section = args[0].value
+name = args[1].value
+if (obj in ('ui', 'self') and method in _configmethods
+and section.type == 'string' and name.type == 'string'):
+entry = (node, method, args, extractstring(section),
+ extractstring(name))
+except Exception:
+pass
+else:
+if entry:
+yield entry
+
+def coreconfigitems(red):
+"""match coreconfigitem(...) pattern, yield (node, args, section, name)"""
+for node in red.find_all('atomtrailers'):
+entry = None
+try:
+args = node[1]
+section = args[0].value
+name = args[1].value
+if (node[0].value == 'coreconfigitem' and section.type == 'string'
+and name.type == 'string'):
+entry = (node, args, extractstring(section),
+ extractstring(name))
+except Exception:
+pass
+else:
+if entry:
+yield entry
+
+def registercoreconfig(cfgred, section, name, defaultrepr):
+"""insert coreconfigitem to cfgred AST
+
+section and name are plain string, defaultrepr is a string
+"""
+# find a place to insert the "coreconfigitem" item
+entries = list(coreconfigitems(cfgred))
+for node, args, nodesection, nodename in reversed(entries):
+if (nodesection, nodename) < (section, name):
+# insert after this entry
+node.insert_after(
+'coreconfigitem(%r, %r,\n'
+'default=%s,\n'
+')' % (section, name, defaultrepr))
+return
+
+def main(argv):
+if not argv:
+print('Usage: codemod_configitems.py FILES\n'
+  'For example, FILES could be "{hgext,mercurial}/*/**.py"')
+dirname = os.path.dirname
+reporoot = dirname(dirname(dirname(os.path.abspath(__file__
+
+# register configitems to this destination
+cfgpath = os.path.join(reporoot, 'mercurial', 'configitems.py')
+cfgred = redbaron.RedBaron(readpath(cfgpath))
+
+# state about what to do
+registered = set((s, n) for n, a, s, n in coreconfigitems(cfgred))
+toregister = {} # {(section, name): defaultrepr}
+coreconfigs = set() # {(section, name)}, whether it's used in core
+
+# first loop: scan all files before taking any action
+for i, path in enumerate(argv):
+print('(%d/%d) scanning %s' % (i + 1, len(argv), path))
+iscore = ('mercurial' in path) and