Title: [225548] trunk/Tools
Revision
225548
Author
[email protected]
Date
2017-12-05 14:34:38 -0800 (Tue, 05 Dec 2017)

Log Message

[build.webkit.org] Rename mastercfg_unittest to steps_unittest
https://bugs.webkit.org/show_bug.cgi?id=180432

Rubber-stamped by Alexey Proskuryakov.

* BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py: Removed.
* BuildSlaveSupport/build.webkit.org-config/steps_unittest.py: Copied from BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py.

Modified Paths

Added Paths

Removed Paths

Diff

Deleted: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py (225547 => 225548)


--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py	2017-12-05 22:28:20 UTC (rev 225547)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py	2017-12-05 22:34:38 UTC (rev 225548)
@@ -1,571 +0,0 @@
-#! /usr/bin/python
-
-import sys
-import os
-import StringIO
-import unittest
-import make_passwords_json
-import json
-
-from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
-from buildbot.steps import source
-from steps import (CheckOutSource, Run32bitJSCTests, RunAndUploadPerfTests, RunBenchmarkTests,
-                   RunJavaScriptCoreTests, RunLLINTCLoopTests, RunTest262Tests, RunUnitTests, RunWebKitTests)
-
-# Show DepricationWarnings come from buildbot - it isn't default with Python 2.7 or newer.
-# See https://bugs.webkit.org/show_bug.cgi?id=90161 for details.
-import warnings
-warnings.simplefilter('default')
-
-
-class BuildBotConfigLoader(object):
-    def _add_webkitpy_to_sys_path(self):
-        # When files are passed to the python interpreter on the command line (e.g. python test.py) __file__ is a relative path.
-        absolute_file_path = os.path.abspath(__file__)
-        webkit_org_config_dir = os.path.dirname(absolute_file_path)
-        build_slave_support_dir = os.path.dirname(webkit_org_config_dir)
-        webkit_tools_dir = os.path.dirname(build_slave_support_dir)
-        scripts_dir = os.path.join(webkit_tools_dir, 'Scripts')
-        sys.path.append(scripts_dir)
-
-    def _mock_open(self, filename):
-        if filename == 'passwords.json':
-            return StringIO.StringIO(json.dumps(make_passwords_json.create_mock_slave_passwords_dict()))
-        return __builtins__.open(filename)
-
-    def _add_dependant_modules_to_sys_modules(self):
-        from webkitpy.thirdparty.autoinstalled import buildbot
-        sys.modules['buildbot'] = buildbot
-
-    def load_config(self, master_cfg_path):
-        # Before we can use webkitpy.thirdparty, we need to fix our path to include webkitpy.
-        # FIXME: If we're ever run by test-webkitpy we won't need this step.
-        self._add_webkitpy_to_sys_path()
-        # master.cfg expects the buildbot module to be in sys.path.
-        self._add_dependant_modules_to_sys_modules()
-
-        # master.cfg expects a passwords.json file which is not checked in.  Fake it by mocking open().
-        globals()['open'] = self._mock_open
-        # Because the master_cfg_path may have '.' in its name, we can't just use import, we have to use execfile.
-        # We pass globals() as both the globals and locals to mimic exectuting in the global scope, so
-        # that globals defined in master.cfg will be global to this file too.
-        execfile(master_cfg_path, globals(), globals())
-        globals()['open'] = __builtins__.open  # Stop mocking open().
-
-
-class RunWebKitTestsTest(unittest.TestCase):
-    def test_nrwt_leaks_parsing(self):
-        run_webkit_tests = RunWebKitTests()  # pylint is confused by the way we import the module ... pylint: disable-msg=E0602
-        log_text = """
-12:44:24.295 77706 13981 total leaks found for a total of 197,936 bytes.
-12:44:24.295 77706 1 unique leaks found.
-"""
-        expected_incorrect_lines = [
-            '13981 total leaks found for a total of 197,936 bytes.',
-            '1 unique leaks found.',
-        ]
-        run_webkit_tests._parseRunWebKitTestsOutput(log_text)
-        self.assertEqual(run_webkit_tests.incorrectLayoutLines, expected_incorrect_lines)
-
-    def test_nrwt_missing_results(self):
-        run_webkit_tests = RunWebKitTests()  # pylint is confused by the way we import the module ... pylint: disable-msg=E0602
-        log_text = """
-Expected to fail, but passed: (2)
-  animations/additive-transform-animations.html
-  animations/cross-fade-webkit-mask-box-image.html
-
-Unexpected flakiness: text-only failures (2)
-  fast/events/touch/touch-inside-iframe.html [ Failure Pass ]
-  http/tests/inspector-enabled/console-clear-arguments-on-frame-navigation.html [ Failure Pass ]
-
-Unexpected flakiness: timeouts (1)
-  svg/text/foreignObject-repaint.xml [ Timeout Pass ]
-
-Regressions: Unexpected missing results (1)
-  svg/custom/zero-path-square-cap-rendering2.svg [ Missing ]
-
-Regressions: Unexpected text-only failures (1)
-  svg/custom/zero-path-square-cap-rendering2.svg [ Failure ]
-"""
-        run_webkit_tests._parseRunWebKitTestsOutput(log_text)
-        self.assertEqual(set(run_webkit_tests.incorrectLayoutLines),
-            set(['2 new passes', '3 flakes', '1 missing results', '1 failures']))
-
-
-class StubStdio(object):
-    def __init__(self, stdio):
-        self._stdio = stdio
-
-    def getText(self):
-        return self._stdio
-
-
-class StubRemoteCommand(object):
-    def __init__(self, rc, stdio):
-        self.rc = rc
-        self.logs = {'stdio': StubStdio(stdio)}
-
-
-class RunJavaScriptCoreTestsTest(unittest.TestCase):
-    def assertResults(self, expected_result, expected_text, rc, stdio):
-        cmd = StubRemoteCommand(rc, stdio)
-        step = RunJavaScriptCoreTests()
-        step.commandComplete(cmd)
-        actual_results = step.evaluateCommand(cmd)
-        actual_text = step.getText2(cmd, actual_results)
-
-        self.assertEqual(expected_result, actual_results)
-        self.assertEqual(actual_text, expected_text)
-
-    def test_no_regressions_old_output(self):
-        self.assertResults(SUCCESS, ["jscore-test"], 0, """Results for Mozilla tests:
-    0 regressions found.
-    0 tests fixed.
-    OK.""")
-
-    def test_no_failure_new_output(self):
-        self.assertResults(SUCCESS, ["jscore-test"], 0, """Results for JSC stress tests:
-    0 failures found.
-    OK.""")
-
-    def test_mozilla_failure_old_output(self):
-        self.assertResults(FAILURE, ["1 JSC test failed"], 1, """Results for Mozilla tests:
-    1 regression found.
-    0 tests fixed.""")
-
-    def test_mozilla_failures_old_output(self):
-        self.assertResults(FAILURE, ["2 JSC tests failed"], 1, """Results for Mozilla tests:
-    2 regressions found.
-    0 tests fixed.""")
-
-    def test_jsc_stress_failure_new_output(self):
-        self.assertResults(FAILURE, ["1 JSC test failed"], 1,  """Results for JSC stress tests:
-    1 failure found.""")
-
-    def test_jsc_stress_failures_new_output(self):
-        self.assertResults(FAILURE, ["5 JSC tests failed"], 1,  """Results for JSC stress tests:
-    5 failures found.""")
-
-
-class RunTest262TestsTest(unittest.TestCase):
-    def assertResults(self, expected_result, expected_text, rc, stdio):
-        cmd = StubRemoteCommand(rc, stdio)
-        step = RunTest262Tests()
-        step.commandComplete(cmd)
-        actual_results = step.evaluateCommand(cmd)
-        actual_text = step.getText2(cmd, actual_results)
-
-        self.assertEqual(expected_result, actual_results)
-        self.assertEqual(actual_text, expected_text)
-
-    def test_no_regressions_output(self):
-        self.assertResults(SUCCESS, ["test262-test"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc
-
-Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
-Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
-""")
-
-    def test_failure_output(self):
-        self.assertResults(FAILURE, ["1 Test262 test failed"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc
-
-Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
-Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
-test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict: ERROR: Unexpected exit code: 0
-FAIL: test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
-""")
-
-    def test_failures_output(self):
-        self.assertResults(FAILURE, ["2 Test262 tests failed"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc
-
-Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
-test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default: ERROR: Unexpected exit code: 0
-FAIL: test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
-Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
-test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict: ERROR: Unexpected exit code: 0
-FAIL: test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
-""")
-
-
-class RunLLINTCLoopTestsTest(unittest.TestCase):
-    def assertResults(self, expected_result, expected_text, rc, stdio):
-        cmd = StubRemoteCommand(rc, stdio)
-        step = RunLLINTCLoopTests()
-        step.commandComplete(cmd)
-        actual_results = step.evaluateCommand(cmd)
-        actual_text = step.getText2(cmd, actual_results)
-
-        self.assertEqual(expected_result, actual_results)
-        self.assertEqual(actual_text, expected_text)
-
-    def test_failures(self):
-        self.assertResults(FAILURE, ['5 regressions found.'], 1,  '    5 regressions found.')
-
-    def test_failure(self):
-        self.assertResults(FAILURE, ['1 regression found.'], 1,  '    1 regression found.')
-
-    def test_no_failure(self):
-        self.assertResults(SUCCESS, ['webkit-jsc-cloop-test'], 0,  '    0 regressions found.')
-
-
-class Run32bitJSCTestsTest(unittest.TestCase):
-    def assertResults(self, expected_result, expected_text, rc, stdio):
-        cmd = StubRemoteCommand(rc, stdio)
-        step = Run32bitJSCTests()
-        step.commandComplete(cmd)
-        actual_results = step.evaluateCommand(cmd)
-        actual_text = step.getText2(cmd, actual_results)
-
-        self.assertEqual(expected_result, actual_results)
-        self.assertEqual(actual_text, expected_text)
-
-    def test_failures(self):
-        self.assertResults(FAILURE, ['5 regressions found.'], 1,  '    5 failures found.')
-
-    def test_failure(self):
-        self.assertResults(FAILURE, ['1 regression found.'], 1,  '    1 failure found.')
-
-    def test_no_failure(self):
-        self.assertResults(SUCCESS, ['webkit-32bit-jsc-test'], 0,  '    0 failures found.')
-
-
-class RunUnitTestsTest(unittest.TestCase):
-    def assertFailures(self, expected_failure_count, stdio):
-        if expected_failure_count:
-            rc = 1
-            expected_results = FAILURE
-            plural_suffix = "" if expected_failure_count == 1 else "s"
-            expected_text = '%d unit test%s failed or timed out' % (expected_failure_count, plural_suffix)
-        else:
-            rc = 0
-            expected_results = SUCCESS
-            expected_text = 'run-api-tests'
-
-        cmd = StubRemoteCommand(rc, stdio)
-        step = RunUnitTests()
-        step.commandComplete(cmd)
-        actual_results = step.evaluateCommand(cmd)
-        actual_failure_count = step.failedTestCount
-        actual_text = step.getText(cmd, actual_results)[0]
-
-        self.assertEqual(expected_results, actual_results)
-        self.assertEqual(expected_failure_count, actual_failure_count)
-        self.assertEqual(expected_text, actual_text)
-
-    def test_no_failures_or_timeouts(self):
-        self.assertFailures(0, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[  PASSED  ] 1 test.
-""")
-
-    def test_one_failure(self):
-        self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[  PASSED  ] 1 test.
-Tests that failed:
-  WebKit2.WebKit2.CanHandleRequest
-""")
-
-    def test_multiple_failures(self):
-        self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[  PASSED  ] 1 test.
-Tests that failed:
-  WebKit2.WebKit2.CanHandleRequest
-  WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
-  WebKit2.WebKit2.HitTestResultNodeHandle
-  WebKit2.WebKit2.InjectedBundleBasic
-""")
-
-    def test_one_timeout(self):
-        self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[  PASSED  ] 1 test.
-Tests that timed out:
-  WebKit2.WebKit2.CanHandleRequest
-""")
-
-    def test_multiple_timeouts(self):
-        self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[  PASSED  ] 1 test.
-Tests that timed out:
-  WebKit2.WebKit2.CanHandleRequest
-  WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
-  WebKit2.WebKit2.HitTestResultNodeHandle
-  WebKit2.WebKit2.InjectedBundleBasic
-""")
-
-    def test_multiple_failures_and_timeouts(self):
-        self.assertFailures(8, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[  PASSED  ] 1 test.
-Tests that failed:
-  WebKit2.WebKit2.CanHandleRequest
-  WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
-  WebKit2.WebKit2.HitTestResultNodeHandle
-Tests that timed out:
-  WebKit2.WebKit2.InjectedBundleBasic
-  WebKit2.WebKit2.LoadCanceledNoServerRedirectCallback
-  WebKit2.WebKit2.MouseMoveAfterCrash
-  WebKit2.WebKit2.ResponsivenessTimerDoesntFireEarly
-  WebKit2.WebKit2.WebArchive
-""")
-
-
-class SVNMirrorTest(unittest.TestCase):
-    def setUp(self):
-        self.config = json.load(open('config.json'))
-
-    def get_SVNMirrorFromConfig(self, builderName):
-        SVNMirror = None
-        for builder in self.config['builders']:
-            if builder['name'] == builderName:
-                SVNMirror = builder.pop('SVNMirror', 'https://svn.webkit.org/repository/webkit/')
-        return SVNMirror
-
-    def test_CheckOutSource(self):
-        # SVN mirror feature isn't unittestable now with source.oldsource.SVN(==source.SVN) , only with source.svn.SVN(==SVN)
-        # https://bugs.webkit.org/show_bug.cgi?id=85887
-        if issubclass(CheckOutSource, source.SVN):
-            return
-
-        # Compare CheckOutSource.baseURL with SVNMirror (or with the default URL) in config.json for all builders
-        for builder in c['builders']:
-            for buildStepFactory, kwargs in builder['factory'].steps:
-                if str(buildStepFactory).split('.')[-1] == 'CheckOutSource':
-                        CheckOutSourceInstance = buildStepFactory(**kwargs)
-                        self.assertEqual(CheckOutSourceInstance.baseURL, self.get_SVNMirrorFromConfig(builder['name']))
-
-
-class BuildStepsConstructorTest(unittest.TestCase):
-    # "Passing a BuildStep subclass to factory.addStep is deprecated. Please pass a BuildStep instance instead.  Support will be dropped in v0.8.7."
-    # It checks if all builder's all buildsteps can be insantiated after migration.
-    # https://bugs.webkit.org/show_bug.cgi?id=89001
-    # http://buildbot.net/buildbot/docs/0.8.6p1/manual/customization.html#writing-buildstep-constructors
-
-    @staticmethod
-    def generateTests():
-        for builderNumber, builder in enumerate(c['builders']):
-            for stepNumber, step in enumerate(builder['factory'].steps):
-                builderName = builder['name'].encode('ascii', 'ignore')
-                setattr(BuildStepsConstructorTest, 'test_builder%02d_step%02d' % (builderNumber, stepNumber), BuildStepsConstructorTest.createTest(builderName, step))
-
-    @staticmethod
-    def createTest(builderName, step):
-        def doTest(self):
-            try:
-                buildStepFactory, kwargs = step
-                buildStepFactory(**kwargs)
-            except TypeError as e:
-                buildStepName = str(buildStepFactory).split('.')[-1]
-                self.fail("Error during instantiation %s buildstep for %s builder: %s\n" % (buildStepName, builderName, e))
-        return doTest
-
-
-expected_build_steps = {
-    'Apple Win 7 Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple Win 7 Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple Win Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple Win Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-
-    'Apple El Capitan 32-bit JSC (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-32bit-jsc-test'],
-    'Apple El Capitan CMake Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-    'Apple El Capitan Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple El Capitan Debug JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
-    'Apple El Capitan Debug Test262 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],
-    'Apple El Capitan Debug WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple El Capitan Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple El Capitan LLINT CLoop (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'],
-    'Apple El Capitan Release (32-bit Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-    'Apple El Capitan Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple El Capitan Release WK2 (Perf)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'perf-test'],
-    'Apple El Capitan Release JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
-    'Apple El Capitan Release Test262 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],
-    'Apple El Capitan Release WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple El Capitan Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-
-    'Apple High Sierra 32-bit JSC (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-32bit-jsc-test'],
-    'Apple High Sierra (Leaks)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple High Sierra Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple High Sierra Debug JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
-    'Apple High Sierra Debug WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple High Sierra Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple High Sierra LLINT CLoop (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'],
-    'Apple High Sierra Release (32-bit Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-    'Apple High Sierra Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple High Sierra Release JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
-    'Apple High Sierra Release WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple High Sierra Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-
-    'Apple Sierra Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple Sierra Debug WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple Sierra Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple Sierra Release (32-bit Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-    'Apple Sierra Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple Sierra Release WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-    'Apple Sierra Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-
-    'Apple iOS 11 Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-    'Apple iOS 11 Simulator Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple iOS 11 Simulator Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-
-    'Apple iOS 11 Simulator Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'Apple iOS 11 Simulator Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
-
-    'JSCOnly Linux ARMv7 Thumb2 Release': ['configure build', 'wait-for-svn-server', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
-    'JSCOnly Linux ARMv7 Traditional Release': ['configure build', 'wait-for-svn-server', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
-    'JSCOnly Linux AArch64 Release': ['configure build', 'wait-for-svn-server', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
-
-    'GTK Linux 32-bit Release': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'jscore-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'API tests'],
-    'GTK Linux 64-bit Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'GTK Linux 64-bit Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
-    'GTK Linux 64-bit Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'GTK Linux 64-bit Release (Perf)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'perf-test', 'benchmark-test'],
-    'GTK Linux 64-bit Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
-    'GTK Linux 64-bit Release Wayland (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
-    'GTK Linux 64-bit Release Ubuntu LTS (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-    'GTK Linux 64-bit Release Debian Stable (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-    'GTK Linux ARM Release': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'jscore-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'API tests'],
-
-    'WinCairo 64-Bit Release': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
-
-    'WPE Linux 64-bit Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
-    'WPE Linux 64-bit Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
-}
-
-
-class BuildStepsTest(unittest.TestCase):
-    @staticmethod
-    def generateTests():
-        for builder in c['builders']:
-            builderName = builder['name'].encode('ascii', 'ignore')
-            setattr(BuildStepsTest, 'test_builder %s' % builderName, BuildStepsTest.createTest(builder))
-
-    @staticmethod
-    def createTest(builder):
-        def doTest(self):
-            buildSteps = []
-            for step in builder['factory'].steps:
-                buildSteps.append(step[0].name)
-            self.assertTrue(builder['name'] in expected_build_steps, "Missing expected result for builder: %s\n Actual result is %s" % (builder['name'], buildSteps))
-            self.assertListEqual(expected_build_steps[builder['name']], buildSteps)
-
-        return doTest
-
-    def test_unnecessary_expected_results(self):
-        builders = set()
-        for builder in c['builders']:
-            builders.add(builder['name'].encode('ascii', 'ignore'))
-
-        for builder in expected_build_steps:
-            self.assertTrue(builder in builders, "Builder %s doesn't exist, but has unnecessary expected results" % builder)
-
-
-class RunAndUploadPerfTestsTest(unittest.TestCase):
-    def assertResults(self, rc, expected_text):
-        cmd = StubRemoteCommand(rc, expected_text)
-        step = RunAndUploadPerfTests()
-        step.commandComplete(cmd)
-        actual_results = step.evaluateCommand(cmd)
-        actual_text = str(step.getText2(cmd, actual_results)[0])
-        self.assertEqual(expected_text, actual_text)
-
-    def test_success(self):
-        self.assertResults(0, "perf-test")
-
-    def test_tests_failed(self):
-        self.assertResults(5, "5 perf tests failed")
-
-    def test_build_bad_build(self):
-        self.assertResults(255, "build not up to date")
-
-    def test_build_bad_source_json(self):
-        self.assertResults(254, "slave config JSON error")
-
-    def test_build_bad_marge(self):
-        self.assertResults(253, "output JSON merge error")
-
-    def test_build_bad_failed_uploading(self):
-        self.assertResults(252, "upload error")
-
-    def test_build_bad_preparation(self):
-        self.assertResults(251, "system dependency error")
-
-    def test_buildbot_timeout(self):
-        self.assertResults(-1, "timeout")
-
-
-class RunBenchmarkTest(unittest.TestCase):
-    def assertResults(self, rc, expected_text):
-        cmd = StubRemoteCommand(rc, expected_text)
-        step = RunBenchmarkTests()
-        step.commandComplete(cmd)
-        actual_results = step.evaluateCommand(cmd)
-        actual_text = str(step.getText2(cmd, actual_results)[0])
-        self.assertEqual(expected_text, actual_text)
-
-    def test_success(self):
-        self.assertResults(0, "benchmark-test")
-
-    def test_tests_failed(self):
-        self.assertResults(7, "7 benchmark tests failed")
-
-
-# FIXME: We should run this file as part of test-webkitpy.
-# Unfortunately test-webkitpy currently requires that unittests
-# be located in a directory with a valid module name.
-# 'build.webkit.org-config' is not a valid module name (due to '.' and '-')
-# so for now this is a stand-alone test harness.
-if __name__ == '__main__':
-    BuildBotConfigLoader().load_config('master.cfg')
-    BuildStepsConstructorTest.generateTests()
-    BuildStepsTest.generateTests()
-    unittest.main()

Copied: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py (from rev 225547, trunk/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py) (0 => 225548)


--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py	                        (rev 0)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py	2017-12-05 22:34:38 UTC (rev 225548)
@@ -0,0 +1,571 @@
+#! /usr/bin/python
+
+import sys
+import os
+import StringIO
+import unittest
+import make_passwords_json
+import json
+
+from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
+from buildbot.steps import source
+from steps import (CheckOutSource, Run32bitJSCTests, RunAndUploadPerfTests, RunBenchmarkTests,
+                   RunJavaScriptCoreTests, RunLLINTCLoopTests, RunTest262Tests, RunUnitTests, RunWebKitTests)
+
+# Show DepricationWarnings come from buildbot - it isn't default with Python 2.7 or newer.
+# See https://bugs.webkit.org/show_bug.cgi?id=90161 for details.
+import warnings
+warnings.simplefilter('default')
+
+
+class BuildBotConfigLoader(object):
+    def _add_webkitpy_to_sys_path(self):
+        # When files are passed to the python interpreter on the command line (e.g. python test.py) __file__ is a relative path.
+        absolute_file_path = os.path.abspath(__file__)
+        webkit_org_config_dir = os.path.dirname(absolute_file_path)
+        build_slave_support_dir = os.path.dirname(webkit_org_config_dir)
+        webkit_tools_dir = os.path.dirname(build_slave_support_dir)
+        scripts_dir = os.path.join(webkit_tools_dir, 'Scripts')
+        sys.path.append(scripts_dir)
+
+    def _mock_open(self, filename):
+        if filename == 'passwords.json':
+            return StringIO.StringIO(json.dumps(make_passwords_json.create_mock_slave_passwords_dict()))
+        return __builtins__.open(filename)
+
+    def _add_dependant_modules_to_sys_modules(self):
+        from webkitpy.thirdparty.autoinstalled import buildbot
+        sys.modules['buildbot'] = buildbot
+
+    def load_config(self, master_cfg_path):
+        # Before we can use webkitpy.thirdparty, we need to fix our path to include webkitpy.
+        # FIXME: If we're ever run by test-webkitpy we won't need this step.
+        self._add_webkitpy_to_sys_path()
+        # master.cfg expects the buildbot module to be in sys.path.
+        self._add_dependant_modules_to_sys_modules()
+
+        # master.cfg expects a passwords.json file which is not checked in.  Fake it by mocking open().
+        globals()['open'] = self._mock_open
+        # Because the master_cfg_path may have '.' in its name, we can't just use import, we have to use execfile.
+        # We pass globals() as both the globals and locals to mimic exectuting in the global scope, so
+        # that globals defined in master.cfg will be global to this file too.
+        execfile(master_cfg_path, globals(), globals())
+        globals()['open'] = __builtins__.open  # Stop mocking open().
+
+
+class RunWebKitTestsTest(unittest.TestCase):
+    def test_nrwt_leaks_parsing(self):
+        run_webkit_tests = RunWebKitTests()  # pylint is confused by the way we import the module ... pylint: disable-msg=E0602
+        log_text = """
+12:44:24.295 77706 13981 total leaks found for a total of 197,936 bytes.
+12:44:24.295 77706 1 unique leaks found.
+"""
+        expected_incorrect_lines = [
+            '13981 total leaks found for a total of 197,936 bytes.',
+            '1 unique leaks found.',
+        ]
+        run_webkit_tests._parseRunWebKitTestsOutput(log_text)
+        self.assertEqual(run_webkit_tests.incorrectLayoutLines, expected_incorrect_lines)
+
+    def test_nrwt_missing_results(self):
+        run_webkit_tests = RunWebKitTests()  # pylint is confused by the way we import the module ... pylint: disable-msg=E0602
+        log_text = """
+Expected to fail, but passed: (2)
+  animations/additive-transform-animations.html
+  animations/cross-fade-webkit-mask-box-image.html
+
+Unexpected flakiness: text-only failures (2)
+  fast/events/touch/touch-inside-iframe.html [ Failure Pass ]
+  http/tests/inspector-enabled/console-clear-arguments-on-frame-navigation.html [ Failure Pass ]
+
+Unexpected flakiness: timeouts (1)
+  svg/text/foreignObject-repaint.xml [ Timeout Pass ]
+
+Regressions: Unexpected missing results (1)
+  svg/custom/zero-path-square-cap-rendering2.svg [ Missing ]
+
+Regressions: Unexpected text-only failures (1)
+  svg/custom/zero-path-square-cap-rendering2.svg [ Failure ]
+"""
+        run_webkit_tests._parseRunWebKitTestsOutput(log_text)
+        self.assertEqual(set(run_webkit_tests.incorrectLayoutLines),
+            set(['2 new passes', '3 flakes', '1 missing results', '1 failures']))
+
+
+class StubStdio(object):
+    def __init__(self, stdio):
+        self._stdio = stdio
+
+    def getText(self):
+        return self._stdio
+
+
+class StubRemoteCommand(object):
+    def __init__(self, rc, stdio):
+        self.rc = rc
+        self.logs = {'stdio': StubStdio(stdio)}
+
+
+class RunJavaScriptCoreTestsTest(unittest.TestCase):
+    def assertResults(self, expected_result, expected_text, rc, stdio):
+        cmd = StubRemoteCommand(rc, stdio)
+        step = RunJavaScriptCoreTests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_text = step.getText2(cmd, actual_results)
+
+        self.assertEqual(expected_result, actual_results)
+        self.assertEqual(actual_text, expected_text)
+
+    def test_no_regressions_old_output(self):
+        self.assertResults(SUCCESS, ["jscore-test"], 0, """Results for Mozilla tests:
+    0 regressions found.
+    0 tests fixed.
+    OK.""")
+
+    def test_no_failure_new_output(self):
+        self.assertResults(SUCCESS, ["jscore-test"], 0, """Results for JSC stress tests:
+    0 failures found.
+    OK.""")
+
+    def test_mozilla_failure_old_output(self):
+        self.assertResults(FAILURE, ["1 JSC test failed"], 1, """Results for Mozilla tests:
+    1 regression found.
+    0 tests fixed.""")
+
+    def test_mozilla_failures_old_output(self):
+        self.assertResults(FAILURE, ["2 JSC tests failed"], 1, """Results for Mozilla tests:
+    2 regressions found.
+    0 tests fixed.""")
+
+    def test_jsc_stress_failure_new_output(self):
+        self.assertResults(FAILURE, ["1 JSC test failed"], 1,  """Results for JSC stress tests:
+    1 failure found.""")
+
+    def test_jsc_stress_failures_new_output(self):
+        self.assertResults(FAILURE, ["5 JSC tests failed"], 1,  """Results for JSC stress tests:
+    5 failures found.""")
+
+
+class RunTest262TestsTest(unittest.TestCase):
+    def assertResults(self, expected_result, expected_text, rc, stdio):
+        cmd = StubRemoteCommand(rc, stdio)
+        step = RunTest262Tests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_text = step.getText2(cmd, actual_results)
+
+        self.assertEqual(expected_result, actual_results)
+        self.assertEqual(actual_text, expected_text)
+
+    def test_no_regressions_output(self):
+        self.assertResults(SUCCESS, ["test262-test"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc
+
+Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
+Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
+""")
+
+    def test_failure_output(self):
+        self.assertResults(FAILURE, ["1 Test262 test failed"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc
+
+Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
+Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
+test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict: ERROR: Unexpected exit code: 0
+FAIL: test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
+""")
+
+    def test_failures_output(self):
+        self.assertResults(FAILURE, ["2 Test262 tests failed"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc
+
+Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
+test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default: ERROR: Unexpected exit code: 0
+FAIL: test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default
+Running test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
+test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict: ERROR: Unexpected exit code: 0
+FAIL: test262.yaml/test262/test/annexB/built-ins/Date/prototype/getYear/length.js.default-strict
+""")
+
+
+class RunLLINTCLoopTestsTest(unittest.TestCase):
+    def assertResults(self, expected_result, expected_text, rc, stdio):
+        cmd = StubRemoteCommand(rc, stdio)
+        step = RunLLINTCLoopTests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_text = step.getText2(cmd, actual_results)
+
+        self.assertEqual(expected_result, actual_results)
+        self.assertEqual(actual_text, expected_text)
+
+    def test_failures(self):
+        self.assertResults(FAILURE, ['5 regressions found.'], 1,  '    5 regressions found.')
+
+    def test_failure(self):
+        self.assertResults(FAILURE, ['1 regression found.'], 1,  '    1 regression found.')
+
+    def test_no_failure(self):
+        self.assertResults(SUCCESS, ['webkit-jsc-cloop-test'], 0,  '    0 regressions found.')
+
+
+class Run32bitJSCTestsTest(unittest.TestCase):
+    def assertResults(self, expected_result, expected_text, rc, stdio):
+        cmd = StubRemoteCommand(rc, stdio)
+        step = Run32bitJSCTests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_text = step.getText2(cmd, actual_results)
+
+        self.assertEqual(expected_result, actual_results)
+        self.assertEqual(actual_text, expected_text)
+
+    def test_failures(self):
+        self.assertResults(FAILURE, ['5 regressions found.'], 1,  '    5 failures found.')
+
+    def test_failure(self):
+        self.assertResults(FAILURE, ['1 regression found.'], 1,  '    1 failure found.')
+
+    def test_no_failure(self):
+        self.assertResults(SUCCESS, ['webkit-32bit-jsc-test'], 0,  '    0 failures found.')
+
+
+class RunUnitTestsTest(unittest.TestCase):
+    def assertFailures(self, expected_failure_count, stdio):
+        if expected_failure_count:
+            rc = 1
+            expected_results = FAILURE
+            plural_suffix = "" if expected_failure_count == 1 else "s"
+            expected_text = '%d unit test%s failed or timed out' % (expected_failure_count, plural_suffix)
+        else:
+            rc = 0
+            expected_results = SUCCESS
+            expected_text = 'run-api-tests'
+
+        cmd = StubRemoteCommand(rc, stdio)
+        step = RunUnitTests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_failure_count = step.failedTestCount
+        actual_text = step.getText(cmd, actual_results)[0]
+
+        self.assertEqual(expected_results, actual_results)
+        self.assertEqual(expected_failure_count, actual_failure_count)
+        self.assertEqual(expected_text, actual_text)
+
+    def test_no_failures_or_timeouts(self):
+        self.assertFailures(0, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[  PASSED  ] 1 test.
+""")
+
+    def test_one_failure(self):
+        self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[  PASSED  ] 1 test.
+Tests that failed:
+  WebKit2.WebKit2.CanHandleRequest
+""")
+
+    def test_multiple_failures(self):
+        self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[  PASSED  ] 1 test.
+Tests that failed:
+  WebKit2.WebKit2.CanHandleRequest
+  WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
+  WebKit2.WebKit2.HitTestResultNodeHandle
+  WebKit2.WebKit2.InjectedBundleBasic
+""")
+
+    def test_one_timeout(self):
+        self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[  PASSED  ] 1 test.
+Tests that timed out:
+  WebKit2.WebKit2.CanHandleRequest
+""")
+
+    def test_multiple_timeouts(self):
+        self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[  PASSED  ] 1 test.
+Tests that timed out:
+  WebKit2.WebKit2.CanHandleRequest
+  WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
+  WebKit2.WebKit2.HitTestResultNodeHandle
+  WebKit2.WebKit2.InjectedBundleBasic
+""")
+
+    def test_multiple_failures_and_timeouts(self):
+        self.assertFailures(8, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN      ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[       OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[  PASSED  ] 1 test.
+Tests that failed:
+  WebKit2.WebKit2.CanHandleRequest
+  WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
+  WebKit2.WebKit2.HitTestResultNodeHandle
+Tests that timed out:
+  WebKit2.WebKit2.InjectedBundleBasic
+  WebKit2.WebKit2.LoadCanceledNoServerRedirectCallback
+  WebKit2.WebKit2.MouseMoveAfterCrash
+  WebKit2.WebKit2.ResponsivenessTimerDoesntFireEarly
+  WebKit2.WebKit2.WebArchive
+""")
+
+
+class SVNMirrorTest(unittest.TestCase):
+    def setUp(self):
+        self.config = json.load(open('config.json'))
+
+    def get_SVNMirrorFromConfig(self, builderName):
+        SVNMirror = None
+        for builder in self.config['builders']:
+            if builder['name'] == builderName:
+                SVNMirror = builder.pop('SVNMirror', 'https://svn.webkit.org/repository/webkit/')
+        return SVNMirror
+
+    def test_CheckOutSource(self):
+        # SVN mirror feature isn't unittestable now with source.oldsource.SVN(==source.SVN) , only with source.svn.SVN(==SVN)
+        # https://bugs.webkit.org/show_bug.cgi?id=85887
+        if issubclass(CheckOutSource, source.SVN):
+            return
+
+        # Compare CheckOutSource.baseURL with SVNMirror (or with the default URL) in config.json for all builders
+        for builder in c['builders']:
+            for buildStepFactory, kwargs in builder['factory'].steps:
+                if str(buildStepFactory).split('.')[-1] == 'CheckOutSource':
+                        CheckOutSourceInstance = buildStepFactory(**kwargs)
+                        self.assertEqual(CheckOutSourceInstance.baseURL, self.get_SVNMirrorFromConfig(builder['name']))
+
+
+class BuildStepsConstructorTest(unittest.TestCase):
+    # "Passing a BuildStep subclass to factory.addStep is deprecated. Please pass a BuildStep instance instead.  Support will be dropped in v0.8.7."
+    # It checks if all builder's all buildsteps can be insantiated after migration.
+    # https://bugs.webkit.org/show_bug.cgi?id=89001
+    # http://buildbot.net/buildbot/docs/0.8.6p1/manual/customization.html#writing-buildstep-constructors
+
+    @staticmethod
+    def generateTests():
+        for builderNumber, builder in enumerate(c['builders']):
+            for stepNumber, step in enumerate(builder['factory'].steps):
+                builderName = builder['name'].encode('ascii', 'ignore')
+                setattr(BuildStepsConstructorTest, 'test_builder%02d_step%02d' % (builderNumber, stepNumber), BuildStepsConstructorTest.createTest(builderName, step))
+
+    @staticmethod
+    def createTest(builderName, step):
+        def doTest(self):
+            try:
+                buildStepFactory, kwargs = step
+                buildStepFactory(**kwargs)
+            except TypeError as e:
+                buildStepName = str(buildStepFactory).split('.')[-1]
+                self.fail("Error during instantiation %s buildstep for %s builder: %s\n" % (buildStepName, builderName, e))
+        return doTest
+
+
+expected_build_steps = {
+    'Apple Win 7 Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple Win 7 Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple Win Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple Win Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+
+    'Apple El Capitan 32-bit JSC (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-32bit-jsc-test'],
+    'Apple El Capitan CMake Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+    'Apple El Capitan Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple El Capitan Debug JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
+    'Apple El Capitan Debug Test262 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],
+    'Apple El Capitan Debug WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple El Capitan Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple El Capitan LLINT CLoop (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'],
+    'Apple El Capitan Release (32-bit Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+    'Apple El Capitan Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple El Capitan Release WK2 (Perf)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'perf-test'],
+    'Apple El Capitan Release JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
+    'Apple El Capitan Release Test262 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],
+    'Apple El Capitan Release WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple El Capitan Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+
+    'Apple High Sierra 32-bit JSC (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-32bit-jsc-test'],
+    'Apple High Sierra (Leaks)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple High Sierra Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple High Sierra Debug JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
+    'Apple High Sierra Debug WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple High Sierra Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple High Sierra LLINT CLoop (BuildAndTest)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'],
+    'Apple High Sierra Release (32-bit Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+    'Apple High Sierra Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple High Sierra Release JSC (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
+    'Apple High Sierra Release WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple High Sierra Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+
+    'Apple Sierra Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple Sierra Debug WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple Sierra Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple Sierra Release (32-bit Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+    'Apple Sierra Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple Sierra Release WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+    'Apple Sierra Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+
+    'Apple iOS 11 Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+    'Apple iOS 11 Simulator Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple iOS 11 Simulator Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+
+    'Apple iOS 11 Simulator Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'Apple iOS 11 Simulator Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
+
+    'JSCOnly Linux ARMv7 Thumb2 Release': ['configure build', 'wait-for-svn-server', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
+    'JSCOnly Linux ARMv7 Traditional Release': ['configure build', 'wait-for-svn-server', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
+    'JSCOnly Linux AArch64 Release': ['configure build', 'wait-for-svn-server', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
+
+    'GTK Linux 32-bit Release': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'jscore-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'API tests'],
+    'GTK Linux 64-bit Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'GTK Linux 64-bit Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
+    'GTK Linux 64-bit Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'GTK Linux 64-bit Release (Perf)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'perf-test', 'benchmark-test'],
+    'GTK Linux 64-bit Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
+    'GTK Linux 64-bit Release Wayland (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
+    'GTK Linux 64-bit Release Ubuntu LTS (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+    'GTK Linux 64-bit Release Debian Stable (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+    'GTK Linux ARM Release': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'jscore-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'API tests'],
+
+    'WinCairo 64-Bit Release': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
+
+    'WPE Linux 64-bit Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
+    'WPE Linux 64-bit Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
+}
+
+
+class BuildStepsTest(unittest.TestCase):
+    @staticmethod
+    def generateTests():
+        for builder in c['builders']:
+            builderName = builder['name'].encode('ascii', 'ignore')
+            setattr(BuildStepsTest, 'test_builder %s' % builderName, BuildStepsTest.createTest(builder))
+
+    @staticmethod
+    def createTest(builder):
+        def doTest(self):
+            buildSteps = []
+            for step in builder['factory'].steps:
+                buildSteps.append(step[0].name)
+            self.assertTrue(builder['name'] in expected_build_steps, "Missing expected result for builder: %s\n Actual result is %s" % (builder['name'], buildSteps))
+            self.assertListEqual(expected_build_steps[builder['name']], buildSteps)
+
+        return doTest
+
+    def test_unnecessary_expected_results(self):
+        builders = set()
+        for builder in c['builders']:
+            builders.add(builder['name'].encode('ascii', 'ignore'))
+
+        for builder in expected_build_steps:
+            self.assertTrue(builder in builders, "Builder %s doesn't exist, but has unnecessary expected results" % builder)
+
+
+class RunAndUploadPerfTestsTest(unittest.TestCase):
+    def assertResults(self, rc, expected_text):
+        cmd = StubRemoteCommand(rc, expected_text)
+        step = RunAndUploadPerfTests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_text = str(step.getText2(cmd, actual_results)[0])
+        self.assertEqual(expected_text, actual_text)
+
+    def test_success(self):
+        self.assertResults(0, "perf-test")
+
+    def test_tests_failed(self):
+        self.assertResults(5, "5 perf tests failed")
+
+    def test_build_bad_build(self):
+        self.assertResults(255, "build not up to date")
+
+    def test_build_bad_source_json(self):
+        self.assertResults(254, "slave config JSON error")
+
+    def test_build_bad_marge(self):
+        self.assertResults(253, "output JSON merge error")
+
+    def test_build_bad_failed_uploading(self):
+        self.assertResults(252, "upload error")
+
+    def test_build_bad_preparation(self):
+        self.assertResults(251, "system dependency error")
+
+    def test_buildbot_timeout(self):
+        self.assertResults(-1, "timeout")
+
+
+class RunBenchmarkTest(unittest.TestCase):
+    def assertResults(self, rc, expected_text):
+        cmd = StubRemoteCommand(rc, expected_text)
+        step = RunBenchmarkTests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_text = str(step.getText2(cmd, actual_results)[0])
+        self.assertEqual(expected_text, actual_text)
+
+    def test_success(self):
+        self.assertResults(0, "benchmark-test")
+
+    def test_tests_failed(self):
+        self.assertResults(7, "7 benchmark tests failed")
+
+
+# FIXME: We should run this file as part of test-webkitpy.
+# Unfortunately test-webkitpy currently requires that unittests
+# be located in a directory with a valid module name.
+# 'build.webkit.org-config' is not a valid module name (due to '.' and '-')
+# so for now this is a stand-alone test harness.
+if __name__ == '__main__':
+    BuildBotConfigLoader().load_config('master.cfg')
+    BuildStepsConstructorTest.generateTests()
+    BuildStepsTest.generateTests()
+    unittest.main()

Modified: trunk/Tools/ChangeLog (225547 => 225548)


--- trunk/Tools/ChangeLog	2017-12-05 22:28:20 UTC (rev 225547)
+++ trunk/Tools/ChangeLog	2017-12-05 22:34:38 UTC (rev 225548)
@@ -1,5 +1,15 @@
 2017-12-05  Aakash Jain  <[email protected]>
 
+        [build.webkit.org] Rename mastercfg_unittest to steps_unittest
+        https://bugs.webkit.org/show_bug.cgi?id=180432
+
+        Rubber-stamped by Alexey Proskuryakov.
+
+        * BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py: Removed.
+        * BuildSlaveSupport/build.webkit.org-config/steps_unittest.py: Copied from BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py.
+
+2017-12-05  Aakash Jain  <[email protected]>
+
         [build.webkit.org] Fix style issues in mastercfg_unittest.py
         https://bugs.webkit.org/show_bug.cgi?id=180439
 
_______________________________________________
webkit-changes mailing list
[email protected]
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to