On 10/12/2016 10:35 AM, Philippe Pepiot wrote:
# HG changeset patch
# User Philippe Pepiot <philippe.pep...@logilab.fr>
# Date 1475136994 -7200
#      Thu Sep 29 10:16:34 2016 +0200
# Node ID f7847ea1b58780e3508d57376c4fdd63d6aedfcd
# Parent  b85fa6bf298be07804a74d8fdec0d19fdbc6d740
# EXP-Topic hgperf
perf: add asv benchmarks

For the record, this is unlikely that we reaches the point were we can review this to the point of acceptance before the freeze on Tuesday.

Unless another reviewer pick this, You'll have to resend this when 4.0 is released on November 1st.


Airspeed velocity (ASV) is a python framework for benchmarking Python packages
over their lifetime. The results are displayed in an interactive web frontend.

Add ASV benchmarks for mercurial that use contrib/perf.py extension that could
be run against multiple reference repositories.

The benchmark suite now includes revsets from contrib/base-revsets.txt with
variants, perftags, perfstatus, perfmanifest and perfheads.

Installation requires ASV (not yet released
https://github.com/spacetelescope/asv master branch), python-hglib and
virtualenv.

This is part of PerformanceTrackingSuitePlan
https://www.mercurial-scm.org/wiki/PerformanceTrackingSuitePlan

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -49,6 +49,7 @@ mercurial.egg-info
 tags
 cscope.*
 .idea/*
+.asv/*
 i18n/hg.pot
 locale/*/LC_MESSAGES/hg.mo
 hgext/__index__.py
diff --git a/contrib/asv.conf.json b/contrib/asv.conf.json
new file mode 100644
--- /dev/null
+++ b/contrib/asv.conf.json
@@ -0,0 +1,127 @@
+{
+    // The version of the config file format.  Do not change, unless
+    // you know what you are doing.
+    "version": 1,
+
+    // The name of the project being benchmarked
+    "project": "mercurial",
+
+    // The project's homepage
+    "project_url": "http://mercurial-scm.org/";,
+
+    // The URL or local path of the source code repository for the
+    // project being benchmarked
+    "repo": "..",
+
+    // List of branches to benchmark. If not provided, defaults to "master"
+    // (for git) or "default" (for mercurial).
+    // "branches": ["master"], // for git
+    // "branches": ["default"],    // for mercurial
+    "branches": ["default", "stable"],
+
+    // The DVCS being used.  If not set, it will be automatically
+    // determined from "repo" by looking at the protocol in the URL
+    // (if remote), or by looking for special directories, such as
+    // ".git" (if local).
+    // "dvcs": "git",
+
+    // The tool to use to create environments.  May be "conda",
+    // "virtualenv" or other value depending on the plugins in use.
+    // If missing or the empty string, the tool will be automatically
+    // determined by looking for tools on the PATH environment
+    // variable.
+    "environment_type": "virtualenv",
+
+    // the base URL to show a commit for the project.
+    "show_commit_url": "https://www.selenic.com/hg/rev/";,
+
+    // The Pythons you'd like to test against.  If not provided, defaults
+    // to the current version of Python used to run `asv`.
+    // "pythons": ["2.7", "3.3"],
+
+    // The matrix of dependencies to test.  Each key is the name of a
+    // package (in PyPI) and the values are version numbers.  An empty
+    // list or empty string indicates to just test against the default
+    // (latest) version. null indicates that the package is to not be
+    // installed. If the package to be tested is only available from
+    // PyPi, and the 'environment_type' is conda, then you can preface
+    // the package name by 'pip+', and the package will be installed via
+    // pip (with all the conda available packages installed first,
+    // followed by the pip installed packages).
+    //
+    // "matrix": {
+    //     "numpy": ["1.6", "1.7"],
+    //     "six": ["", null],        // test with and without six installed
+    //     "pip+emcee": [""],   // emcee is only available for install with 
pip.
+    // },
+
+    // Combinations of libraries/python versions can be excluded/included
+    // from the set to test. Each entry is a dictionary containing additional
+    // key-value pairs to include/exclude.
+    //
+    // An exclude entry excludes entries where all values match. The
+    // values are regexps that should match the whole string.
+    //
+    // An include entry adds an environment. Only the packages listed
+    // are installed. The 'python' key is required. The exclude rules
+    // do not apply to includes.
+    //
+    // In addition to package names, the following keys are available:
+    //
+    // - python
+    //     Python version, as in the *pythons* variable above.
+    // - environment_type
+    //     Environment type, as above.
+    // - sys_platform
+    //     Platform, as in sys.platform. Possible values for the common
+    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
+    //
+    // "exclude": [
+    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
+    //     {"environment_type": "conda", "six": null}, // don't run without 
six on conda
+    // ],
+    //
+    // "include": [
+    //     // additional env for python2.7
+    //     {"python": "2.7", "numpy": "1.8"},
+    //     // additional env if run on windows+conda
+    //     {"platform": "win32", "environment_type": "conda", "python": "2.7", 
"libpython": ""},
+    // ],
+
+    // The directory (relative to the current directory) that benchmarks are
+    // stored in.  If not provided, defaults to "benchmarks"
+    "benchmark_dir": "benchmarks",
+
+    // The directory (relative to the current directory) to cache the Python
+    // environments in.  If not provided, defaults to "env"
+    "env_dir": "../.asv/env",
+
+    // The directory (relative to the current directory) that raw benchmark
+    // results are stored in.  If not provided, defaults to "results".
+    "results_dir": "../.asv/results",
+
+    // The directory (relative to the current directory) that the html tree
+    // should be written to.  If not provided, defaults to "html".
+    "html_dir": "../.asv/html",
+
+    // The number of characters to retain in the commit hashes.
+    // "hash_length": 8,
+
+    // `asv` will cache wheels of the recent builds in each
+    // environment, making them faster to install next time.  This is
+    // number of builds to keep, per environment.
+    // "wheel_cache_size": 0
+
+    // The commits after which the regression search in `asv publish`
+    // should start looking for regressions. Dictionary whose keys are
+    // regexps matching to benchmark names, and values corresponding to
+    // the commit (exclusive) after which to start looking for
+    // regressions.  The default is to start from the first commit
+    // with results. If the commit is `null`, regression detection is
+    // skipped for the matching benchmark.
+    //
+    // "regressions_first_commits": {
+    //    "some_benchmark": "352cdf",  // Consider regressions only after this 
commit
+    //    "another_benchmark": null,   // Skip regression detection altogether
+    // }
+}
diff --git a/contrib/benchmarks/__init__.py b/contrib/benchmarks/__init__.py
new file mode 100644
--- /dev/null
+++ b/contrib/benchmarks/__init__.py
@@ -0,0 +1,106 @@
+"""ASV (http://asv.readthedocs.io) benchmark suite
+
+Benchmark are parameterized against reference repositories found in the
+directory pointed by the REPOS_DIR environment variable.
+
+Invocation example:
+
+    $ export REPOS_DIR=~/hgperf/repos
+    # run suite on given revision
+    $ asv --config contrib/asv.conf.json run REV
+    # run suite on new changesets found in stable and default branch
+    $ asv --config contrib/asv.conf.json run NEW
+    # display a comparative result table of benchmark results between two given
+    # revisions
+    $ asv --config contrib/asv.conf.json compare REV1 REV2
+    # compute regression detection and generate ASV static website
+    $ asv --config contrib/asv.conf.json publish
+    # serve the static website
+    $ asv --config contrib/asv.conf.json preview
+"""
+
+from __future__ import absolute_import
+
+import contextlib
+import functools
+import io
+import os
+import re
+
+from mercurial import (
+    extensions,
+    hg,
+    ui as uimod,
+)
+
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                          os.path.pardir, os.path.pardir))
+reposdir = os.environ['REPOS_DIR']
+reposnames = [name for name in os.listdir(reposdir)
+              if os.path.isdir(os.path.join(reposdir, name, ".hg"))]
+outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
+                      r'\d+.\d+ \(best of \d+\)'))
+
+@contextlib.contextmanager
+def redirectuioutput(ui):
+    old = ui.fout, ui.ferr
+    new = io.BytesIO()
+    try:
+        ui.fout = ui.ferr = new
+        yield new
+    finally:
+        ui.fout, ui.ferr = old
+
+def runperfcommand(reponame, command, *args, **kwargs):
+    os.environ["HGRCPATH"] = ""
+    ui = uimod.ui()
+    repo = hg.repository(ui, os.path.join(reposdir, reponame))
+    perfext = extensions.load(ui, 'perfext',
+                              os.path.join(basedir, 'contrib', 'perf.py'))
+    cmd = getattr(perfext, command)
+    with redirectuioutput(ui) as buf:
+        cmd(ui, repo, *args, **kwargs)
+    buf.seek(0)
+    output = buf.read()
+    match = outputre.search(output)
+    if not match:
+        raise ValueError("Invalid output {0}".format(output))
+    return float(match.group(1))
+
+def perfbench(repos=reposnames, name=None, params=None):
+    """decorator to declare ASV benchmark based on contrib/perf.py extension
+
+    An ASV benchmark is a python function with the given attributes:
+
+    __name__: should start with track_, time_ or mem_ to be collected by ASV
+    params and param_name: parameter matrix to display multiple graphs on the
+    same page.
+    pretty_name: If defined it's displayed in web-ui instead of __name__
+    (useful for revsets)
+    the module name is prepended to the benchmark name and displayed as
+    "category" in webui.
+
+    Benchmarks are automatically parameterized with repositories found in the
+    REPOS_DIR environment variable.
+
+    `params` is the param matrix in the form of a list of tuple
+    (param_name, [value0, value1])
+
+    For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
+    (a, c), (a, d), (b, c) and (b, d).
+    """
+    params = list(params or [])
+    params.insert(0, ("repo", repos))
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapped(repo, *args):
+            def perf(command, *a, **kw):
+                return runperfcommand(repo, command, *a, **kw)
+            return func(perf, *args)
+
+        wrapped.params = [p[1] for p in params]
+        wrapped.param_names = [p[0] for p in params]
+        wrapped.pretty_name = name
+        return wrapped
+    return decorator
diff --git a/contrib/benchmarks/perf.py b/contrib/benchmarks/perf.py
new file mode 100644
--- /dev/null
+++ b/contrib/benchmarks/perf.py
@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+
+from . import perfbench
+
+@perfbench()
+def track_tags(perf):
+    return perf("perftags")
+
+@perfbench()
+def track_status(perf):
+    return perf("perfstatus", unknown=False)
+
+@perfbench(params=[('rev', ['1000', '10000', 'tip'])])
+def track_manifest(perf, rev):
+    return perf("perfmanifest", rev)
+
+@perfbench()
+def track_heads(perf):
+    return perf("perfheads")
diff --git a/contrib/benchmarks/revset.py b/contrib/benchmarks/revset.py
new file mode 100644
--- /dev/null
+++ b/contrib/benchmarks/revset.py
@@ -0,0 +1,45 @@
+"""ASV revset benchmarks generated from contrib/base-revsets.txt
+
+Each revset benchmark is parameterized with variants (first, last, sort, ...)
+"""
+from __future__ import absolute_import
+
+import os
+import string
+import sys
+
+from . import basedir, perfbench
+
+def createrevsetbenchmark(baseset, variants=None):
+    if variants is None:
+        # Default variants
+        variants = ["plain", "first", "last", "sort", "sort+first",
+                    "sort+last"]
+    fname = "track_" + "_".join("".join([
+        c if c in string.digits + string.letters else " "
+        for c in baseset
+    ]).split())
+
+    def wrap(fname, baseset):
+        @perfbench(name=baseset, params=[("variant", variants)])
+        def f(perf, variant):
+            revset = baseset
+            if variant != "plain":
+                for var in variant.split("+"):
+                    revset = "%s(%s)" % (var, revset)
+            return perf("perfrevset", revset)
+        f.__name__ = fname
+        return f
+    return wrap(fname, baseset)
+
+def initializerevsetbenchmarks():
+    mod = sys.modules[__name__]
+    with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'),
+              'rb') as fh:
+        for line in fh:
+            baseset = line.strip()
+            if baseset and not baseset.startswith('#'):
+                func = createrevsetbenchmark(baseset)
+                setattr(mod, func.__name__, func)
+
+initializerevsetbenchmarks()
_______________________________________________
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel


--
Pierre-Yves David
_______________________________________________
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Reply via email to