Gergő Tisza has uploaded a new change for review. https://gerrit.wikimedia.org/r/197240
Change subject: [WIP] Make vbench more generic ...................................................................... [WIP] Make vbench more generic Separate VE-specific and osmium-specific parts from generic JS performance benchmark code. Could not get this to work for reasons that might or might not be related to the patch itself (Chrome just refuses to start), hence the [WIP]. Bug: T92701 Change-Id: I34c13a32300899c7b5cdf98aeb82384bae2ca239 --- M files/ve/vb D files/ve/vbench M manifests/role/ve.pp 3 files changed, 7 insertions(+), 527 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/40/197240/1 diff --git a/files/ve/vb b/files/ve/vb index 60546cb..6dc6741 100755 --- a/files/ve/vb +++ b/files/ve/vb @@ -12,30 +12,10 @@ repo_status "/srv/mediawiki-local/extensions/VisualEditor" repo_status "/srv/mediawiki-local" -cd /srv/profiles -/usr/lib/chromium-browser/chromium-browser \ - --incognito \ - --remote-debugging-port=9222 \ - --display=:99 \ - --disable-background-networking \ - --disable-client-side-phishing-detection \ - --disable-component-update \ - --disable-default-apps \ - --disable-extensions \ - --disable-hang-monitor \ - --disable-infobars \ - --disable-plugins-discovery \ - --disable-prompt-on-repost \ - --disable-suggestions-service \ - --disable-sync \ - --disable-translate \ - --disable-v8-idle-tasks \ - --disable-web-resources \ - --no-default-browser-check \ - --no-first-run \ - --safebrowsing-disable-auto-update \ - --safebrowsing-disable-download-protection >/dev/null 2>&1 & +export JSBENCH_CPU=15 + +sudo service jsbench-browser start sleep 1 -taskset -p -c 15 "$(pgrep --newest chromium)" >/dev/null -vbench --repeat=30 --write-profile-data "http://osmium/wiki/Barack_Obama"; -pkill -u "$(whoami)" chrom +jsbench --repeat=30 --write-profile-data --jsvar stage init "http://osmium/wiki/Barack_Obama"; + +sudo service jsbench-browser start diff --git a/files/ve/vbench b/files/ve/vbench deleted file mode 100755 index fb022f2..0000000 --- a/files/ve/vbench +++ /dev/null @@ -1,451 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - vbench - VisualEditor benchmarking tool - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Usage: vbench [OPTIONS] URL - -h, --help show this help message and exit - --host HOST Chromium host (default: localhost) - --port PORT Chromium port (default: 9222) - --repeat N times to repeat (default: 5) - --disable-cache disable network cache - --write-profile-data save profiling data as .cpuprofile files. - --verbose log verbosely - --display DISPLAY set device display metrics to WIDTHxHEIGHT or - WIDTHxHEIGHT*SCALE_FACTOR - - Copyright 2015 Ori Livneh <[email protected]> - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -""" -from __future__ import division, print_function - -import sys -reload(sys) -sys.setdefaultencoding('utf-8') - -import argparse -import functools -import itertools -import json -import logging -import re -import string -import time -import urllib2 -import weakref - -try: - from twisted.internet import defer, reactor - from twisted.python.log import PythonLoggingObserver - - from autobahn.twisted.websocket import (WebSocketClientProtocol, - WebSocketClientFactory) -except ImportError: - print('vbench requires autobahn (https://pypi.python.org/pypi/autobahn/) ' - 'and twisted (https://pypi.python.org/pypi/Twisted/).') - sys.exit(1) - - -# FIXME: Make this configurable. -JS = ''' - window.onload = function () { - var stage = "%s"; - ve.trackSubscribe( 'trace.' + stage + '.', function ( topic ) { - switch ( topic.split( '.' ).pop() ) { - case 'enter': - console.profile( stage ); - break; - case 'exit': - console.profileEnd( stage ); - break; - } - } ); - - // Don't show the welcome dialog. - localStorage.clear() - localStorage.setItem( 've-beta-welcome-dialog', 1 ); - - // Wait 200ms for any load handlers to run, then start VE. - setTimeout( function () { - mw.libs.ve.onEditTabClick( { preventDefault: $.noop } ); - }, 200 ); - }; -''' - - -class Highlighter(object): - RED, GREEN, YELLOW, BLUE, WHITE = ('\x1b[1;3%sm' % n for n in '12347') - RESET = '\x1b[0m' - SUBS = {'^': RED, '@': GREEN, '_': YELLOW, '~': BLUE, '*': WHITE} - RE = re.compile(r'(?P<meta>[%s])\((?P<text>.+?)\)(?P=meta)' - % ''.join(SUBS)) - - def __init__(self, use_color=True): - self.use_color = use_color - - def repl(self, match): - start = self.SUBS[match.group('meta')] if self.use_color else '' - end = self.RESET if self.use_color else '' - return start + match.group('text') + end - - def highlight(self, s): - return self.RE.sub(self.repl, s) - - -class HighlightingFormatter(logging.Formatter, Highlighter): - """A formatter for the Python standard library's logging module - that uses `Highlighter` to colorize text.""" - - def __init__(self, use_color=False): - self.use_color = use_color - super(HighlightingFormatter, self).__init__( - self.highlight('[_(%(asctime)s)_] %(message)s'), '%H:%M:%S') - - def format(self, record): - if isinstance(record.msg, basestring): - record.msg = self.highlight(record.msg) - return super(HighlightingFormatter, self).format(record) - - -def mean(data): - """Compute arithmetic mean ("average") of data.""" - data = list(data) - n = len(data) - if not n: - raise ValueError('cannot compute mean of empty set') - return sum(data) / n - - -def median(data): - """Compute median (middle value) of data.""" - data = list(data) - n = len(data) - if not n: - raise ValueError('cannot compute median of empty set') - data.sort() - i = n // 2 - return data[i] if n % 2 == 1 else (data[i - 1] + data[i]) / 2 - - -def std(data): - """Compute the population standard deviation.""" - data = list(data) - n = len(data) - if n == 0: - raise ValueError('cannot compute stdev of empty set') - m = mean(data) - ss = sum((x - m) ** 2 for x in data) - return (ss / n) ** 0.5 - - -def parse_display_opts(opt_string): - """Parse display options in format WIDTHxHEIGHT or WIDTHxHEIGHT*FACTOR.""" - opts = {'deviceScaleFactor': 1, 'emulateViewport': False, - 'fitWindow': False, 'mobile': False} - match = re.match(r'(?P<width>\d+)x(?P<height>\d+)(\*(?P<deviceScaleFactor' - r'>\d+))?', opt_string) - if match is None: - raise ValueError('Invalid display options string.') - opts.update(((k, int(v)) for k, v in match.groupdict().items())) - return opts - - -def upper_first(s): - """Returns a copy of a string with the first letter capitalized.""" - return s[0:1].upper() + s[1:] - - -def summarize(data): - data = list(data) - template = ('min: *({: >7.2f})* max: *({: >7.2f})* ' - 'avg: *({: >7.2f})* med: *({: >7.2f})* ' - 'std: *({: >7.2f})*') - return template.format(min(data), max(data), mean(data), - median(data), std(data)) - - -class ChromeCPUProfile(object): - """Represents the result of a Chrome CPU profiler run.""" - - def __init__(self, profile, title=None): - self.profile = profile - self.title = title - self.total_hit_count = self.get_hit_count(profile['head']) - self.wall_time = (profile['endTime'] - profile['startTime']) * 1000 - self.sampling_interval = self.wall_time / self.total_hit_count - self.cpu_time = 0 - self.gc_time = 0 - self.idle_time = 0 - - for node in profile['head']['children']: - if node['functionName'] == '(garbage collector)': - self.gc_time = self.get_time(node) - elif node['functionName'] == '(idle)': - self.idle_time = self.get_time(node) - else: - self.cpu_time += self.get_time(node) - - def get_hit_count(self, node): - children = node.get('children', ()) - return node['hitCount'] + sum(self.get_hit_count(x) for x in children) - - def get_time(self, node): - return self.get_hit_count(node) * self.sampling_interval - - def write(self, file_name=None): - if file_name is None: - prefix = self.title or 'profile' - file_name = '%s.%d.cpuprofile' % (prefix, time.time()) - with open(file_name, 'w') as f: - json.dump(self.profile, f, sort_keys=True, indent=2) - - -class ChromeRemoteDebuggingDomain(object): - """Represents a Chrome Remote Debugging API domain.""" - - def __init__(self, domain, proto): - self.domain = upper_first(domain) - self.proto = weakref.proxy(proto) - - def __getattr__(self, name): - command = self.domain + '.' + name - return functools.partial(self.proto.sendCommand, command) - - -class ChromeRemoteDebuggingProtocol(WebSocketClientProtocol): - """Protocol for communicating with Chrome via the Remote Debugging API.""" - - domain_names = ( - 'heapProfiler', - 'inspector', - 'network', - 'page', - 'profiler', - ) - - def sendCommand(self, method, **params): - id = next(self.message_ids) - deferred = self.deferreds[id] = defer.Deferred() - command = {'id': id, 'method': method, 'params': params} - self.sendMessage(json.dumps(command)) - return deferred - - def onConnect(self, response): - self.message_ids = itertools.count() - self.deferreds = {} - self.profiles = [] - reactor.addSystemEventTrigger('after', 'shutdown', self.showSummary) - - def onConsoleMessageAdded(self, message): - level = message['level'] - text = message.get('text', '') - meta = dict(log='~', error='^', warning='_', debug='@').get(level, '') - log_level = getattr(logging, level.upper(), logging.DEBUG) - log.log(log_level, '[%s(console.%s)%s] %s' % (meta, level, meta, text)) - - def onInspectorTargetCrashed(self): - log.critical('^(Aw, snap!)^ Target has crashed. Trying to recover...') - return self.onProfilerReady() - - @defer.inlineCallbacks - def onOpen(self): - log.info('Loading *(%s)*...', self.factory.target_url) - - for domain_name in self.domain_names: - domain = ChromeRemoteDebuggingDomain(domain_name, self) - self.__dict__[domain_name] = domain - yield domain.enable() - - scriptSource = JS % self.factory.stage - yield self.page.addScriptToEvaluateOnLoad(scriptSource=scriptSource) - yield self.profiler.setSamplingInterval(interval=100) - - if self.factory.disable_cache: - yield self.network.setCacheDisabled(cacheDisabled=True) - - if self.factory.display: - yield self.page.setDeviceMetricsOverride(**self.factory.display) - - yield self.onProfilerReady() - - @defer.inlineCallbacks - def onProfilerReady(self): - yield self.network.clearBrowserCache() - yield self.heapProfiler.collectGarbage() - yield self.page.navigate(url=self.factory.target_url) - - def onProfilerConsoleProfileFinished(self, id, location, profile, - title=None): - profile = ChromeCPUProfile(profile, title) - - if self.factory.warmups > 0: - t = 'XX/{:02d}: CPU: {: <7.2f} Wall: {: >7.2f} GC: {: >7.2f}' - log.info(t.format(self.factory.repetitions, profile.cpu_time, - profile.wall_time, profile.gc_time)) - self.factory.warmups -= 1 - return self.onProfilerReady() - - self.profiles.append(profile) - - if self.factory.write_profile_data: - profile.write() - - if self.factory.repetitions > 1: - t = '{:02d}/{:02d}: CPU: {: <7.2f} Wall: {: >7.2f} GC: {: >7.2f}' - log.info(t.format(len(self.profiles), self.factory.repetitions, - profile.cpu_time, profile.wall_time, - profile.gc_time)) - if len(self.profiles) < self.factory.repetitions: - return self.onProfilerReady() - else: - reactor.stop() - - def getHandler(self, method): - object, event = method.split('.') - handler = 'on' + object + upper_first(event) - return getattr(self, handler, None) - - def onMessage(self, payload, isBinary): - message = json.loads(payload.decode('utf-8')) - - id = message.get('id') - error = message.get('error') - method = message.get('method') - params = message.get('params') - result = message.get('result') - deferred = self.deferreds.get(id) - - if deferred: - if error: - log.error(error) - return deferred.errback(error) - else: - return deferred.callback(result) - - if method: - handler = self.getHandler(method) - if handler: - return handler(**params) - - def showSummary(self): - if not len(self.profiles): - return - log.info('CPU: ' + summarize(p.cpu_time for p in self.profiles)) - log.info('Wall: ' + summarize(p.wall_time for p in self.profiles)) - - -class ChromeRemoteDebuggingFactory(WebSocketClientFactory): - - def __init__(self, target_url, **kwargs): - self.__dict__.update(kwargs) - self.target_url = target_url.replace('https://', 'http://', 1) - WebSocketClientFactory.__init__(self, self.getConnectableTabUrl()) - self.protocol = ChromeRemoteDebuggingProtocol - - def getConnectableTabUrl(self): - for tab in self.getTabs(): - if 'webSocketDebuggerUrl' in tab: - return tab['webSocketDebuggerUrl'] - - def getTabs(self): - req = urllib2.urlopen('http://%s:%s/json' % (self.host, self.port)) - return json.load(req) - - -ap = argparse.ArgumentParser(fromfile_prefix_chars='@') -ap.add_argument( - '--host', - default='localhost', - help='Chromium host (default: localhost)', -) -ap.add_argument( - '--port', - default=9222, - help='Chromium port (default: 9222)', - type=int, -) -ap.add_argument( - '--repeat', - default=5, - dest='repetitions', - help='times to repeat (default: 5)', - metavar='N', - type=int, -) -ap.add_argument( - '--warmups', - default=2, - help='number of warm-up runs (default: 3)', - metavar='N', - type=int, -) -ap.add_argument( - '--write-profile-data', - action='store_true', - default=False, - help='save profiling data as .cpuprofile files', -) -ap.add_argument( - '--verbose', - action='store_const', - const=logging.DEBUG, - default=logging.INFO, - dest='log_level', - help='log verbosely', -) -ap.add_argument( - '--display', - type=parse_display_opts, - help='set device display metrics to WIDTHxHEIGHT or ' - 'WIDTHxHEIGHT*SCALE_FACTOR', -) -ap.add_argument( - '--disable-cache', - action='store_true', - default=False, - help='disable network cache', -) -ap.add_argument( - '--stage', - default='init', - help='Name of Specific stage to profile (default: "init")', -) -ap.add_argument( - 'target_url', - metavar='URL', - help='URL to load', -) - -args = ap.parse_args() - -log = logging.getLogger('vbench') -log.setLevel(args.log_level) - -is_tty = sys.stdout.isatty() -stdout_handler = logging.StreamHandler(stream=sys.stdout) -stdout_handler.setFormatter(HighlightingFormatter(use_color=is_tty)) -log.addHandler(stdout_handler) -if not is_tty: - stderr_handler = logging.StreamHandler(stream=sys.stderr) - stderr_handler.setFormatter(HighlightingFormatter(use_color=True)) - log.addHandler(stderr_handler) - -observer = PythonLoggingObserver('vbench') -observer.start() - -factory = ChromeRemoteDebuggingFactory(**vars(args)) -reactor.connectTCP(args.host, args.port, factory) -reactor.run() diff --git a/manifests/role/ve.pp b/manifests/role/ve.pp index 13a1bcc..f12a0e2 100644 --- a/manifests/role/ve.pp +++ b/manifests/role/ve.pp @@ -4,60 +4,11 @@ # Chromium instance that supports remote debugging. # class role::ve { + include ::role::jsbench include ::mediawiki include ::mediawiki::web include ::mediawiki::web::sites include ::admin - - # 1366x768 is the most common display resolution, according - # to http://gs.statcounter.com/. - - class { 'xvfb': - resolution => '1366x768x24', - } - - # Instruct Chromium to route all requests to localhost, and to - # disable various features that add noise to profiling or that - # rely on user input. - - # class { 'chromium': - # extra_args => [ - # '--disable-background-networking', - # '--disable-client-side-phishing-detection', - # '--disable-component-update', - # '--disable-default-apps', - # '--disable-extensions', - # '--disable-hang-monitor', - # '--disable-infobars', - # '--disable-plugins-discovery', - # '--disable-prompt-on-repost', - # '--disable-suggestions-service', - # '--disable-sync', - # '--disable-translate', - # '--disable-v8-idle-tasks', - # '--disable-web-resources', - # '--no-default-browser-check', - # '--no-first-run', - # '--host-rules="MAP * localhost, EXCLUDE upload.wikimedia.org"', - # '--safebrowsing-disable-auto-update', - # '--safebrowsing-disable-download-protection', - # ], - # } - - - # vbench is a CLI tool for benchmarking VisualEditor. - # It uses `autobahn` and `twisted` for WebSocket support, which - # it needs so it can speak Chrome's remote debugging protocol. - # It uses `numpy` to calculate summary statistics. - - require_package('python-autobahn', 'python-twisted', 'python-numpy') - - file { '/usr/local/bin/vbench': - source => 'puppet:///files/ve/vbench', - owner => 'root', - group => 'root', - mode => '0555', - } file { '/usr/local/bin/vb': source => 'puppet:///files/ve/vb', -- To view, visit https://gerrit.wikimedia.org/r/197240 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I34c13a32300899c7b5cdf98aeb82384bae2ca239 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Gergő Tisza <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
