Diff
Modified: trunk/Tools/ChangeLog (160458 => 160459)
--- trunk/Tools/ChangeLog 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/ChangeLog 2013-12-11 22:47:07 UTC (rev 160459)
@@ -1,3 +1,63 @@
+2013-12-11 Tim Horton <[email protected]>
+
+ Make it possible to compare layout test results between various configurations of WebKit
+ https://bugs.webkit.org/show_bug.cgi?id=125212
+ <rdar://problem/15199108>
+
+ Reviewed by Darin Adler.
+
+ * Scripts/compare-webkit-configurations: Added.
+ Add a script that allows comparison between the "normal" WebKit configuration
+ used for testing and a different configuration (e.g. with accelerated drawing
+ turned on, with the remote layer tree enabled, or WebKit1 vs WebKit2), like so:
+
+ compare-webkit-configurations -2 --comparison=accelerated-drawing compositing/background-color
+
+ This script runs the tests once in the default configuration, ignoring
+ all test expectations, writing the results out to a temporary directory.
+
+ It then runs the tests again in the modified configuration, against the
+ just-written temporary results.
+
+ Ref tests are treated as pixel tests, with their pixel output dumped to disk.
+
+ * DumpRenderTree/mac/DumpRenderTree.mm:
+ (resetWebPreferencesToConsistentValues):
+ (initializeGlobalsFromCommandLineOptions):
+ Enable accelerated drawing if requested.
+
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+ (parse_args):
+ Add --accelerated-drawing and --remote-layer-tree feature switches, which
+ are forwarded on to the test drivers.
+ Move those two and --complex-text into their own "feature switches" section.
+ Drive-by change "Mac OS X" to "OS X".
+ Add --treat-ref-tests-as-pixel-tests option, which treats ref tests
+ as traditional pixel tests (with PNGs dumped on disk), and
+ ignores e.g. -expected.html files.
+
+ * Scripts/webkitpy/port/base.py:
+ (Port.reference_files):
+ Pretend that there are no ref tests.
+ Since is_reference_html_file is intact, our mechanism to ignore
+ the expected files will continue working in this case.
+
+ * Scripts/webkitpy/port/driver.py:
+ (Driver.cmd_line):
+ Forward --accelerated-drawing and --remote-layer-tree to DRT/WKTR.
+
+ * WebKitTestRunner/TestController.cpp:
+ (WTR::TestController::TestController):
+ (WTR::TestController::initialize):
+ (WTR::TestController::resetPreferencesToConsistentValues):
+ * WebKitTestRunner/TestController.h:
+ (WTR::TestController::shouldUseRemoteLayerTree):
+ * WebKitTestRunner/TestInvocation.cpp:
+ (WTR::updateThreadedScrollingForCurrentTest):
+ * WebKitTestRunner/mac/PlatformWebViewMac.mm:
+ (WTR::PlatformWebView::PlatformWebView):
+ Parse and implement the three feature switches.
+
2013-12-11 Mario Sanchez Prada <[email protected]>
[ATK] Expose accessibility objects WAI-ARIA landmark roles
Modified: trunk/Tools/DumpRenderTree/mac/DumpRenderTree.mm (160458 => 160459)
--- trunk/Tools/DumpRenderTree/mac/DumpRenderTree.mm 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/DumpRenderTree/mac/DumpRenderTree.mm 2013-12-11 22:47:07 UTC (rev 160459)
@@ -157,6 +157,7 @@
static int dumpTree = YES;
static int useTimeoutWatchdog = YES;
static int forceComplexText;
+static int useAcceleratedDrawing;
static int gcBetweenTests;
static BOOL printSeparators;
static RetainPtr<CFStringRef> persistentUserStyleSheetLocation;
@@ -654,7 +655,7 @@
[preferences setAcceleratedCompositingEnabled:YES];
#if USE(CA)
[preferences setCanvasUsesAcceleratedDrawing:YES];
- [preferences setAcceleratedDrawingEnabled:NO];
+ [preferences setAcceleratedDrawingEnabled:useAcceleratedDrawing];
#endif
[preferences setWebGLEnabled:NO];
[preferences setCSSRegionsEnabled:YES];
@@ -787,6 +788,7 @@
{"tree", no_argument, &dumpTree, YES},
{"threaded", no_argument, &threaded, YES},
{"complex-text", no_argument, &forceComplexText, YES},
+ {"accelerated-drawing", no_argument, &useAcceleratedDrawing, YES},
{"gc-between-tests", no_argument, &gcBetweenTests, YES},
{"no-timeout", no_argument, &useTimeoutWatchdog, NO},
{NULL, 0, NULL, 0}
Added: trunk/Tools/Scripts/compare-webkit-configurations (0 => 160459)
--- trunk/Tools/Scripts/compare-webkit-configurations (rev 0)
+++ trunk/Tools/Scripts/compare-webkit-configurations 2013-12-11 22:47:07 UTC (rev 160459)
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2013 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import subprocess
+import sys
+import tempfile
+
+def main():
+ parser = argparse.ArgumentParser(description="Compare layout test results between two configurations of WebKit.")
+
+ parser.add_argument("--comparison", action="" choices=["remote-layer-tree", "accelerated-drawing", "webkit2"], required=True, help="configuration to test")
+ parser.add_argument("-2", "--webkit2", action="" default=False, help="use WebKit2 always (some configurations may force this)")
+ parser.add_argument('tests', metavar='TESTS', type=str, help='list of LayoutTest subdirectories to run', default=None, nargs="*")
+
+ args = parser.parse_args()
+
+ force_webkit2 = args.webkit2
+ if args.comparison == "remote-layer-tree":
+ print "Forcing use of WebKit2, as the remote layer tree depends on WebKit2."
+ force_webkit2 = True
+
+ if args.comparison == "webkit2" and force_webkit2:
+ print "It doesn't make sense to test WebKit1 vs. WebKit2 *and* force WebKit2 on."
+ sys.exit(1)
+
+ configuration_flag = flag_for_comparison(args.comparison)
+
+ results_directory = tempfile.mkdtemp(prefix="webkit-comparison-results-", suffix="-" + args.comparison)
+
+ # Run first in the default configuration, generating new baselines, then run
+ # again against those results in the configuration being tested.
+ run_webkit_tests(tests=args.tests, results_directory=results_directory, use_webkit2=force_webkit2, additional_arguments=["--no-show-results", "--new-baseline"])
+ run_webkit_tests(tests=args.tests, results_directory=results_directory, additional_arguments=[configuration_flag], use_webkit2=force_webkit2)
+
+def flag_for_comparison(comparison_name):
+ if comparison_name == "remote-layer-tree":
+ return "--remote-layer-tree"
+ if comparison_name == "accelerated-drawing":
+ return "--accelerated-drawing"
+ if comparison_name == "webkit2":
+ return "-2"
+
+ print "Unknown comparison:", comparison_name
+ sys.exit(1)
+
+def run_webkit_tests(tests, results_directory, use_webkit2, additional_arguments=[]):
+ if use_webkit2:
+ use_webkit2_arguments = ["-2"]
+ else:
+ use_webkit2_arguments = []
+
+ subprocess.call(["run-webkit-tests", "-p", "--force", "--treat-ref-tests-as-pixel-tests", "--additional-platform-directory=" + results_directory, "--run-singly", "--no-retry-failures"] + use_webkit2_arguments + additional_arguments + tests)
+
+if __name__ == '__main__':
+ main()
Property changes on: trunk/Tools/Scripts/compare-webkit-configurations
___________________________________________________________________
Added: svn:executable
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py (160458 => 160459)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2013-12-11 22:47:07 UTC (rev 160459)
@@ -104,15 +104,22 @@
default=False, help="Prefix used when spawning the Web process (Debug mode only)"),
]))
+ option_group_definitions.append(("Feature Switches", [
+ optparse.make_option("--complex-text", action="" default=False,
+ help="Use the complex text code path for all text (OS X and Windows only)"),
+ optparse.make_option("--accelerated-drawing", action="" default=False,
+ help="Use accelerated drawing (OS X only)"),
+ optparse.make_option("--remote-layer-tree", action="" default=False,
+ help="Use the remote layer tree drawing model (OS X WebKit2 only)"),
+ ]))
+
option_group_definitions.append(("WebKit Options", [
optparse.make_option("--gc-between-tests", action="" default=False,
help="Force garbage collection between each test"),
- optparse.make_option("--complex-text", action="" default=False,
- help="Use the complex text code path for all text (Mac OS X and Windows only)"),
optparse.make_option("-l", "--leaks", action="" default=False,
- help="Enable leaks checking (Mac OS X and Gtk+ only)"),
+ help="Enable leaks checking (OS X and Gtk+ only)"),
optparse.make_option("-g", "--guard-malloc", action="" default=False,
- help="Enable Guard Malloc (Mac OS X only)"),
+ help="Enable Guard Malloc (OS X only)"),
optparse.make_option("--threaded", action="" default=False,
help="Run a concurrent _javascript_ thread with each test"),
optparse.make_option("--webkit-test-runner", "-2", action=""
@@ -128,7 +135,7 @@
optparse.make_option("--no-pixel", "--no-pixel-tests", action=""
dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
optparse.make_option("--no-sample-on-timeout", action=""
- dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"),
+ dest="sample_on_timeout", help="Don't run sample on timeout (OS X only)"),
optparse.make_option("--no-ref-tests", action=""
dest="no_ref_tests", help="Skip all ref tests"),
optparse.make_option("--tolerance",
@@ -149,6 +156,8 @@
optparse.make_option("--no-new-test-results", action=""
dest="new_test_results", default=True,
help="Don't create new baselines when no expected results exist"),
+ optparse.make_option("--treat-ref-tests-as-pixel-tests", action="" default=False,
+ help="Run ref tests, but treat them as if they were traditional pixel tests"),
#FIXME: we should support a comma separated list with --pixel-test-directory as well.
optparse.make_option("--pixel-test-directory", action="" default=[], dest="pixel_test_directories",
Modified: trunk/Tools/Scripts/webkitpy/port/base.py (160458 => 160459)
--- trunk/Tools/Scripts/webkitpy/port/base.py 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/Scripts/webkitpy/port/base.py 2013-12-11 22:47:07 UTC (rev 160459)
@@ -542,6 +542,9 @@
def reference_files(self, test_name):
"""Return a list of expectation (== or !=) and filename pairs"""
+ if self.get_option('treat_ref_tests_as_pixel_tests'):
+ return []
+
reftest_list = self._get_reftest_list(test_name)
if not reftest_list:
reftest_list = []
Modified: trunk/Tools/Scripts/webkitpy/port/driver.py (160458 => 160459)
--- trunk/Tools/Scripts/webkitpy/port/driver.py 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/Scripts/webkitpy/port/driver.py 2013-12-11 22:47:07 UTC (rev 160459)
@@ -336,6 +336,10 @@
cmd.append('--gc-between-tests')
if self._port.get_option('complex_text'):
cmd.append('--complex-text')
+ if self._port.get_option('accelerated_drawing'):
+ cmd.append('--accelerated-drawing')
+ if self._port.get_option('remote_layer_tree'):
+ cmd.append('--remote-layer-tree')
if self._port.get_option('threaded'):
cmd.append('--threaded')
if self._no_timeout:
Modified: trunk/Tools/WebKitTestRunner/TestController.cpp (160458 => 160459)
--- trunk/Tools/WebKitTestRunner/TestController.cpp 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/WebKitTestRunner/TestController.cpp 2013-12-11 22:47:07 UTC (rev 160459)
@@ -111,6 +111,9 @@
, m_policyDelegatePermissive(false)
, m_handlesAuthenticationChallenges(false)
, m_shouldBlockAllPlugins(false)
+ , m_forceComplexText(false)
+ , m_shouldUseAcceleratedDrawing(false)
+ , m_shouldUseRemoteLayerTree(false)
{
initialize(argc, argv);
controller = this;
@@ -309,6 +312,18 @@
printSupportedFeatures = true;
break;
}
+ if (argument == "--complex-text") {
+ m_forceComplexText = true;
+ continue;
+ }
+ if (argument == "--accelerated-drawing") {
+ m_shouldUseAcceleratedDrawing = true;
+ continue;
+ }
+ if (argument == "--remote-layer-tree") {
+ m_shouldUseRemoteLayerTree = true;
+ continue;
+ }
// Skip any other arguments that begin with '--'.
@@ -380,10 +395,21 @@
if (testPluginDirectory())
WKContextSetAdditionalPluginsDirectory(m_context.get(), testPluginDirectory());
+ if (m_forceComplexText)
+ WKContextSetAlwaysUsesComplexTextCodePath(m_context.get(), true);
+
// Some preferences (notably mock scroll bars setting) currently cannot be re-applied to an existing view, so we need to set them now.
resetPreferencesToConsistentValues();
- createWebViewWithOptions(0);
+ WKRetainPtr<WKMutableDictionaryRef> viewOptions;
+ if (m_shouldUseRemoteLayerTree) {
+ viewOptions = adoptWK(WKMutableDictionaryCreate());
+ WKRetainPtr<WKStringRef> useRemoteLayerTreeKey = adoptWK(WKStringCreateWithUTF8CString("RemoteLayerTree"));
+ WKRetainPtr<WKBooleanRef> useRemoteLayerTreeValue = adoptWK(WKBooleanCreate(m_shouldUseRemoteLayerTree));
+ WKDictionaryAddItem(viewOptions.get(), useRemoteLayerTreeKey.get(), useRemoteLayerTreeValue.get());
+ }
+
+ createWebViewWithOptions(viewOptions.get());
}
void TestController::createWebViewWithOptions(WKDictionaryRef options)
@@ -557,6 +583,8 @@
#if ENABLE(WEB_AUDIO)
WKPreferencesSetMediaSourceEnabled(preferences, true);
#endif
+
+ WKPreferencesSetAcceleratedDrawingEnabled(preferences, m_shouldUseAcceleratedDrawing);
}
bool TestController::resetStateToConsistentValues()
Modified: trunk/Tools/WebKitTestRunner/TestController.h (160458 => 160459)
--- trunk/Tools/WebKitTestRunner/TestController.h 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/WebKitTestRunner/TestController.h 2013-12-11 22:47:07 UTC (rev 160459)
@@ -66,6 +66,7 @@
EventSenderProxy* eventSenderProxy() { return m_eventSenderProxy.get(); }
void ensureViewSupportsOptions(WKDictionaryRef options);
+ bool shouldUseRemoteLayerTree() const { return m_shouldUseRemoteLayerTree; }
// Runs the run loop until `done` is true or the timeout elapses.
enum TimeoutDuration { ShortTimeout, LongTimeout, NoTimeout, CustomTimeout };
@@ -218,6 +219,10 @@
bool m_shouldBlockAllPlugins;
+ bool m_forceComplexText;
+ bool m_shouldUseAcceleratedDrawing;
+ bool m_shouldUseRemoteLayerTree;
+
OwnPtr<EventSenderProxy> m_eventSenderProxy;
WorkQueueManager m_workQueueManager;
Modified: trunk/Tools/WebKitTestRunner/TestInvocation.cpp (160458 => 160459)
--- trunk/Tools/WebKitTestRunner/TestInvocation.cpp 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/WebKitTestRunner/TestInvocation.cpp 2013-12-11 22:47:07 UTC (rev 160459)
@@ -151,6 +151,10 @@
WKRetainPtr<WKBooleanRef> useThreadedScrollingValue = adoptWK(WKBooleanCreate(shouldUseThreadedScrolling(pathOrURL)));
WKDictionaryAddItem(viewOptions.get(), useThreadedScrollingKey.get(), useThreadedScrollingValue.get());
+ WKRetainPtr<WKStringRef> useRemoteLayerTreeKey = adoptWK(WKStringCreateWithUTF8CString("RemoteLayerTree"));
+ WKRetainPtr<WKBooleanRef> useRemoteLayerTreeValue = adoptWK(WKBooleanCreate(TestController::shared().shouldUseRemoteLayerTree()));
+ WKDictionaryAddItem(viewOptions.get(), useRemoteLayerTreeKey.get(), useRemoteLayerTreeValue.get());
+
TestController::shared().ensureViewSupportsOptions(viewOptions.get());
#else
UNUSED_PARAM(pathOrURL);
Modified: trunk/Tools/WebKitTestRunner/mac/PlatformWebViewMac.mm (160458 => 160459)
--- trunk/Tools/WebKitTestRunner/mac/PlatformWebViewMac.mm 2013-12-11 22:46:47 UTC (rev 160458)
+++ trunk/Tools/WebKitTestRunner/mac/PlatformWebViewMac.mm 2013-12-11 22:47:07 UTC (rev 160459)
@@ -119,6 +119,7 @@
, m_options(options)
{
WKRetainPtr<WKStringRef> useThreadedScrollingKey(AdoptWK, WKStringCreateWithUTF8CString("ThreadedScrolling"));
+ WKRetainPtr<WKStringRef> useRemoteLayerTreeKey(AdoptWK, WKStringCreateWithUTF8CString("RemoteLayerTree"));
WKTypeRef useThreadedScrollingValue = options ? WKDictionaryGetItemForKey(options, useThreadedScrollingKey.get()) : NULL;
bool useThreadedScrolling = useThreadedScrollingValue && WKBooleanGetValue(static_cast<WKBooleanRef>(useThreadedScrollingValue));
@@ -126,6 +127,11 @@
WKPreferencesRef preferences = WKPageGroupGetPreferences(pageGroupRef);
WKPreferencesSetThreadedScrollingEnabled(preferences, useThreadedScrolling);
+ // FIXME: Not sure this is the best place for this; maybe we should have API to set this so we can do it from TestController?
+ WKTypeRef useRemoteLayerTreeValue = options ? WKDictionaryGetItemForKey(options, useRemoteLayerTreeKey.get()) : NULL;
+ if (useRemoteLayerTreeValue && WKBooleanGetValue(static_cast<WKBooleanRef>(useRemoteLayerTreeValue)))
+ [[NSUserDefaults standardUserDefaults] setValue:@YES forKey:@"WebKit2UseRemoteLayerTreeDrawingArea"];
+
NSRect rect = NSMakeRect(0, 0, TestController::viewWidth, TestController::viewHeight);
m_view = [[TestRunnerWKView alloc] initWithFrame:rect contextRef:contextRef pageGroupRef:pageGroupRef relatedToPage:relatedPage useThreadedScrolling:useThreadedScrolling];
[m_view setWindowOcclusionDetectionEnabled:NO];
