Diff
Modified: trunk/Tools/ChangeLog (90769 => 90770)
--- trunk/Tools/ChangeLog 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/ChangeLog 2011-07-11 18:51:13 UTC (rev 90770)
@@ -1,5 +1,27 @@
2011-07-11 Adam Barth <aba...@webkit.org>
+ Add a webkit-patch command for rebaselining an individual test
+ https://bugs.webkit.org/show_bug.cgi?id=64246
+
+ Reviewed by Eric Seidel.
+
+ This patch introduces a command that's able to rebaseline a single
+ test. Currently, the command works only with the build.chromium.org
+ buildbots, but extending it to work with the build.webkit.org bots
+ shouldn't be that hard.
+
+ A complete rebaseling tool should also include an "optimize baselines"
+ command (which moves/deletes baselines in order to reduce the number of
+ expected results files), but that will come in a future patch.
+
+ Really BuilderToPort should be merged into builders.py, but I'm going
+ to save that for a future patch as well. (We need to stop shaving yaks
+ at some point.)
+
+ * Scripts/webkitpy/tool/commands/rebaseline.py:
+
+2011-07-11 Adam Barth <aba...@webkit.org>
+
garden-o-matic should be able to roll out patches
https://bugs.webkit.org/show_bug.cgi?id=64185
Modified: trunk/Tools/Scripts/webkitpy/common/host.py (90769 => 90770)
--- trunk/Tools/Scripts/webkitpy/common/host.py 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/Scripts/webkitpy/common/host.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -31,7 +31,7 @@
from webkitpy.common.checkout import Checkout
from webkitpy.common.checkout.scm import default_scm
from webkitpy.common.config.ports import WebKitPort
-from webkitpy.common.net import bugzilla, buildbot, statusserver
+from webkitpy.common.net import bugzilla, buildbot, statusserver, web
from webkitpy.common.net.irc import ircproxy
from webkitpy.common.system import executive, filesystem, platforminfo, user, workspace
from webkitpy.layout_tests import port
@@ -42,6 +42,7 @@
self.bugs = bugzilla.Bugzilla()
self.buildbot = buildbot.BuildBot()
self.executive = executive.Executive()
+ self.web = web.Web()
self._irc = None
self.filesystem = filesystem.FileSystem()
self.workspace = workspace.Workspace(self.filesystem, self.executive)
Modified: trunk/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py (90769 => 90770)
--- trunk/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -69,6 +69,11 @@
def results_url(self):
return "%s/results/%s" % (self._buildbot.buildbot_url, self.url_encoded_name())
+ # In addition to per-build results, the build.chromium.org builders also
+ # keep a directory that accumulates test results over many runs.
+ def accumulated_results_url(self):
+ return None
+
def url_encoded_name(self):
return urllib.quote(self._name)
@@ -280,8 +285,11 @@
class BuildBot(object):
- def __init__(self, url=""
- self.buildbot_url = url
+ _builder_factory = Builder
+ _default_url = config_urls.buildbot_url
+
+ def __init__(self, url=""
+ self.buildbot_url = url if url else self._default_url
self._builder_by_name = {}
# If any core builder is red we should not be landing patches. Other
@@ -440,7 +448,7 @@
def builder_with_name(self, name):
builder = self._builder_by_name.get(name)
if not builder:
- builder = Builder(name, self)
+ builder = self._builder_factory(name, self)
self._builder_by_name[name] = builder
return builder
Added: trunk/Tools/Scripts/webkitpy/common/net/buildbot/chromiumbuildbot.py (0 => 90770)
--- trunk/Tools/Scripts/webkitpy/common/net/buildbot/chromiumbuildbot.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/common/net/buildbot/chromiumbuildbot.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -0,0 +1,47 @@
+# Copyright (c) 2011, Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import webkitpy.common.config.urls as config_urls
+from webkitpy.common.net.buildbot.buildbot import Builder, BuildBot
+# FIXME: builders should probably be in webkitpy.common.config.
+from webkitpy.layout_tests.port.builders import builder_path_from_name
+
+
+class ChromiumBuilder(Builder):
+ # The build.chromium.org builders store their results in a different
+ # location than the build.webkit.org builders.
+ def results_url(self):
+ return "http://build.chromium.org/f/chromium/layout_test_results/%s" % builder_path_from_name(self._name)
+
+ def accumulated_results_url(self):
+ return self.results_url() + "/results/layout-test-results"
+
+
+class ChromiumBuildBot(BuildBot):
+ _builder_factory = ChromiumBuilder
+ _default_url = config_urls.chromium_buildbot_url
Added: trunk/Tools/Scripts/webkitpy/common/net/web.py (0 => 90770)
--- trunk/Tools/Scripts/webkitpy/common/net/web.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/common/net/web.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -0,0 +1,36 @@
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import urllib2
+
+from webkitpy.common.net.networktransaction import NetworkTransaction
+
+
+class Web(object):
+ def get_binary(self, url):
+ return NetworkTransaction().run(lambda: urllib2.urlopen(url).read())
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/builders.py (90769 => 90770)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/builders.py 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/builders.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -30,31 +30,32 @@
import re
-def _path_from_name(name):
- return re.sub(r'[\s().]', '_', name)
+def builder_path_from_name(builder_name):
+ return re.sub(r'[\s().]', '_', builder_name)
+
# Compiled manually from http://build.chromium.org/p/chromium/json/builders/help?as_text=1
# Values of None mean there are no bots running at build.webkit.org or
# build.chromium.org for that port.
# FIXME Make the values in this map into lists.
CHROMIUM_PORT_TO_BUILDER_NAME = {
- 'chromium-gpu-linux': _path_from_name('Webkit Linux - GPU'),
+ 'chromium-gpu-linux': builder_path_from_name('Webkit Linux - GPU'),
- 'chromium-gpu-mac-snowleopard': _path_from_name('Webkit Mac10.6 - GPU'),
- 'chromium-gpu-mac-leopard': _path_from_name('Webkit Mac10.5 - GPU'),
+ 'chromium-gpu-mac-snowleopard': builder_path_from_name('Webkit Mac10.6 - GPU'),
+ 'chromium-gpu-mac-leopard': builder_path_from_name('Webkit Mac10.5 - GPU'),
- 'chromium-gpu-win-xp': _path_from_name('Webkit Win - GPU'),
- 'chromium-gpu-win-vista': _path_from_name('Webkit Vista - GPU'),
- 'chromium-gpu-win-win7': _path_from_name('Webkit Win7 - GPU'),
+ 'chromium-gpu-win-xp': builder_path_from_name('Webkit Win - GPU'),
+ 'chromium-gpu-win-vista': builder_path_from_name('Webkit Vista - GPU'),
+ 'chromium-gpu-win-win7': builder_path_from_name('Webkit Win7 - GPU'),
- 'chromium-linux-x86_64': _path_from_name('Linux Tests x64'),
- 'chromium-linux-x86': _path_from_name('Linux Tests (dbg)(1)'),
+ 'chromium-linux-x86_64': builder_path_from_name('Linux Tests x64'),
+ 'chromium-linux-x86': builder_path_from_name('Linux Tests (dbg)(1)'),
- 'chromium-mac-leopard': _path_from_name('Mac10.5 Tests (1)'),
- 'chromium-mac-snowleopard': _path_from_name('Mac 10.6 Tests (dbg)(1)'),
+ 'chromium-mac-leopard': builder_path_from_name('Mac10.5 Tests (1)'),
+ 'chromium-mac-snowleopard': builder_path_from_name('Mac 10.6 Tests (dbg)(1)'),
- 'chromium-win-xp': _path_from_name('XP Tests (dbg)(5)'),
- 'chromium-win-vista': _path_from_name('Vista Tests (dbg)(1)'),
+ 'chromium-win-xp': builder_path_from_name('XP Tests (dbg)(5)'),
+ 'chromium-win-vista': builder_path_from_name('Vista Tests (dbg)(1)'),
'chromium-win-win7': None,
'google-chrome-linux32': None,
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/builders_unittest.py (90769 => 90770)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/builders_unittest.py 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/builders_unittest.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -38,7 +38,7 @@
'(.) ': '____',
}
for name, expected in tests.items():
- self.assertEquals(expected, builders._path_from_name(name))
+ self.assertEquals(expected, builders.builder_path_from_name(name))
if __name__ == '__main__':
unittest.main()
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py (90769 => 90770)
--- trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -31,7 +31,9 @@
import shutil
import urllib
+import webkitpy.common.config.urls as config_urls
from webkitpy.common.net.buildbot import BuildBot
+from webkitpy.common.net.buildbot.chromiumbuildbot import ChromiumBuildBot
from webkitpy.common.net.layouttestresults import LayoutTestResults
from webkitpy.common.system.user import User
from webkitpy.layout_tests.models import test_failures
@@ -40,10 +42,10 @@
from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
-# FIXME: I'm not sure where this logic should go in the end.
-# For now it's here, until we have a second need for it.
+# FIXME: This logic should be moved to builders.py.
class BuilderToPort(object):
_builder_name_to_port_name = {
+ # These builders are on build.webkit.org.
r"SnowLeopard": "mac-snowleopard",
r"Leopard": "mac-leopard",
r"Windows": "win",
@@ -52,6 +54,22 @@
r"Chromium Mac": "chromium-mac",
r"Chromium Linux": "chromium-linux",
r"Chromium Win": "chromium-win",
+
+ # These builders are on build.chromium.org.
+ r"Webkit Win": "chromium-win-xp",
+ r"Webkit Vista": "chromium-win-vista",
+ r"Webkit Win7": "chromium-win-win7",
+ r"Webkit Win (dbg)(1)": "chromium-win-win7", # FIXME: Is this correct?
+ r"Webkit Win (dbg)(2)": "chromium-win-win7", # FIXME: Is this correct?
+ r"Webkit Linux": "chromium-linux-x86_64",
+ r"Webkit Linux 32": "chromium-linux-x86",
+ r"Webkit Linux (dbg)(1)": "chromium-linux-x86_64",
+ r"Webkit Linux (dbg)(2)": "chromium-linux-x86_64",
+ r"Webkit Mac10\.5": "chromium-mac-leopard",
+ r"Webkit Mac10\.5 (dbg)(1)": "chromium-mac-leopard",
+ r"Webkit Mac10\.5 (dbg)(2)": "chromium-mac-leopard",
+ r"Webkit Mac10\.6": "chromium-mac-snowleopard",
+ r"Webkit Mac10\.6 (dbg)": "chromium-mac-snowleopard",
}
def _port_name_for_builder_name(self, builder_name):
@@ -67,6 +85,49 @@
return port
+class RebaselineTest(AbstractDeclarativeCommand):
+ name = "rebaseline-test"
+ help_text = "Rebaseline a single test from a buildbot. (Currently works only with build.chromium.org buildbots.)"
+ argument_names = "BUILDER_NAME TEST_NAME SUFFIX"
+
+ def _results_url(self, builder_name):
+ # FIXME: Generalize this command to work with non-build.chromium.org builders.
+ # FIXME: We should really get the buildbot from the tool!
+ builder = ChromiumBuildBot().builder_with_name(builder_name)
+ return builder.accumulated_results_url()
+
+ def _baseline_directory(self, builder_name):
+ port = BuilderToPort().port_for_builder(builder_name)
+ return port.baseline_path()
+
+ def _save_baseline(self, data, target_baseline):
+ self._tool.filesystem.write_binary_file(target_baseline, data)
+ if not self._tool.scm().exists(target_baseline):
+ self._tool.scm().add(target_baseline)
+
+ def _test_root(self, test_name):
+ return os.path.splitext(test_name)[0]
+
+ def _file_name_for_actual_result(self, test_name, suffix):
+ return "%s-actual.%s" % (self._test_root(test_name), suffix)
+
+ def _file_name_for_expected_result(self, test_name, suffix):
+ return "%s-expected.%s" % (self._test_root(test_name), suffix)
+
+ def _rebaseline_test(self, builder_name, test_name, suffix):
+ results_url = self._results_url(builder_name)
+ baseline_directory = self._baseline_directory(builder_name)
+
+ source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix))
+ target_baseline = os.path.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix))
+
+ print "Retrieving %s ..." % source_baseline
+ self._save_baseline(self._tool.web.get_binary(source_baseline), target_baseline)
+
+ def execute(self, options, args, tool):
+ self._rebaseline_test(args[0], args[1], args[2])
+
+
class Rebaseline(AbstractDeclarativeCommand):
name = "rebaseline"
help_text = "Replaces local expected.txt files with new results from build bots"
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py (90769 => 90770)
--- trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -30,11 +30,11 @@
from webkitpy.common.system.outputcapture import OutputCapture
from webkitpy.thirdparty.mock import Mock
-from webkitpy.tool.commands.rebaseline import BuilderToPort, Rebaseline
+from webkitpy.tool.commands.rebaseline import *
from webkitpy.tool.mocktool import MockTool
-class RebaselineTest(unittest.TestCase):
+class TestRebaseline(unittest.TestCase):
# This just makes sure the code runs without exceptions.
def test_tests_to_update(self):
command = Rebaseline()
@@ -43,6 +43,15 @@
OutputCapture().assert_outputs(self, command._tests_to_update, [build])
+class TestRebaselineTest(unittest.TestCase):
+ def test_tests_to_update(self):
+ command = RebaselineTest()
+ command.bind_to_tool(MockTool())
+ build = Mock()
+ expected_stdout = "Retrieving http://build.chromium.org/f/chromium/layout_test_results/Webkit_Linux/results/layout-test-results/userscripts/another-test-actual.txt ...\n"
+ OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Linux", "userscripts/another-test.html", "txt"], expected_stdout=expected_stdout)
+
+
class BuilderToPortTest(unittest.TestCase):
def test_port_for_builder(self):
converter = BuilderToPort()
Modified: trunk/Tools/Scripts/webkitpy/tool/mocktool.py (90769 => 90770)
--- trunk/Tools/Scripts/webkitpy/tool/mocktool.py 2011-07-11 18:48:34 UTC (rev 90769)
+++ trunk/Tools/Scripts/webkitpy/tool/mocktool.py 2011-07-11 18:51:13 UTC (rev 90770)
@@ -747,6 +747,11 @@
return object() # Something that is not None
+class MockWeb(object):
+ def get_binary(self, url):
+ return "MOCK Web result"
+
+
class MockTool(object):
def __init__(self, log_executive=False):
@@ -754,6 +759,7 @@
self.bugs = MockBugzilla()
self.buildbot = MockBuildBot()
self.executive = MockExecutive(should_log=log_executive)
+ self.web = MockWeb()
self.filesystem = MockFileSystem()
self.workspace = MockWorkspace()
self._irc = None
