Deleted: trunk/Tools/Scripts/webkitpy/tool/bot/commitqueuetask_unittest.py (261318 => 261319)
--- trunk/Tools/Scripts/webkitpy/tool/bot/commitqueuetask_unittest.py 2020-05-07 18:21:48 UTC (rev 261318)
+++ trunk/Tools/Scripts/webkitpy/tool/bot/commitqueuetask_unittest.py 2020-05-07 18:24:04 UTC (rev 261319)
@@ -1,583 +0,0 @@
-# Copyright (c) 2010 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from datetime import datetime
-import logging
-import unittest
-
-from webkitpy.common.net import bugzilla
-from webkitpy.common.net.layouttestresults import LayoutTestResults
-from webkitpy.common.system.executive import ScriptError
-from webkitpy.common.system.outputcapture import OutputCapture
-from webkitpy.layout_tests.models import test_results
-from webkitpy.layout_tests.models import test_failures
-from webkitpy.thirdparty.mock import Mock
-from webkitpy.tool.bot.commitqueuetask import *
-from webkitpy.tool.mocktool import MockTool
-
-_log = logging.getLogger(__name__)
-
-
-class MockCommitQueue(CommitQueueTaskDelegate):
- def __init__(self, error_plan):
- self._error_plan = error_plan
- self._failure_status_id = 0
- self._flaky_tests = []
-
- def run_command(self, command):
- _log.info("run_webkit_patch: %s" % command)
- if self._error_plan:
- error = self._error_plan.pop(0)
- if error:
- raise error
-
- def command_passed(self, success_message, patch):
- _log.info("command_passed: success_message='%s' patch='%s'" % (
- success_message, patch.id()))
-
- def command_failed(self, failure_message, script_error, patch):
- _log.info("command_failed: failure_message='%s' script_error='%s' patch='%s'" % (
- failure_message, script_error, patch.id()))
- self._failure_status_id += 1
- return self._failure_status_id
-
- def refetch_patch(self, patch):
- return patch
-
- def test_results(self):
- return LayoutTestResults(test_results=[], did_exceed_test_failure_limit=False)
-
- def report_flaky_tests(self, patch, flaky_results, results_archive):
- current_flaky_tests = [result.test_name for result in flaky_results]
- self._flaky_tests += current_flaky_tests
- _log.info("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'" % (patch.id(), current_flaky_tests, results_archive.filename))
-
- def get_reported_flaky_tests(self):
- return self._flaky_tests
-
- def archive_last_test_results(self, patch):
- _log.info("archive_last_test_results: patch='%s'" % patch.id())
- archive = Mock()
- archive.filename = "mock-archive-%s.zip" % patch.id()
- return archive
-
- def build_style(self):
- return "release"
-
- def did_pass_testing_ews(self, patch):
- return False
-
-
-class FailingTestCommitQueue(MockCommitQueue):
- def __init__(self, error_plan, test_failure_plan):
- MockCommitQueue.__init__(self, error_plan)
- self._test_run_counter = -1 # Special value to indicate tests have never been run.
- self._test_failure_plan = test_failure_plan
-
- def run_command(self, command):
- if command[0] == "build-and-test":
- self._test_run_counter += 1
- MockCommitQueue.run_command(self, command)
-
- def _mock_test_result(self, testname):
- return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
-
- def test_results(self):
- # Doesn't make sense to ask for the test_results until the tests have run at least once.
- assert(self._test_run_counter >= 0)
- failures_for_run = self._test_failure_plan[self._test_run_counter]
- assert(isinstance(failures_for_run, list))
- results = LayoutTestResults(test_results=list(map(self._mock_test_result, failures_for_run)), did_exceed_test_failure_limit=(len(failures_for_run) >= 10))
- return results
-
-
-class PatchAnalysisResult(object):
- FAIL = "Fail"
- DEFER = "Defer"
- PASS = "Pass"
-
-
-class MockSimpleTestPlanCommitQueue(MockCommitQueue):
- def __init__(self, first_test_failures, second_test_failures, clean_test_failures):
- MockCommitQueue.__init__(self, [])
- self._did_run_clean_tests = False
- self._patch_test_results = [first_test_failures, second_test_failures]
- self._clean_test_results = [clean_test_failures]
- self._current_test_results = []
-
- def run_command(self, command):
- MockCommitQueue.run_command(self, command)
- if command[0] == "build-and-test":
- if "--no-clean" in command:
- self._current_test_results = self._patch_test_results.pop(0)
- else:
- self._current_test_results = self._clean_test_results.pop(0)
- self._did_run_clean_tests = True
-
- if self._current_test_results:
- raise ScriptError("MOCK test failure")
-
- def _mock_test_result(self, testname):
- return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
-
- def test_results(self):
- assert(isinstance(self._current_test_results, list))
- return LayoutTestResults(test_results=list(map(self._mock_test_result, self._current_test_results)), did_exceed_test_failure_limit=(len(self._current_test_results) >= 10))
-
- def did_run_clean_tests(self):
- return self._did_run_clean_tests
-
-
-# We use GoldenScriptError to make sure that the code under test throws the
-# correct (i.e., golden) exception.
-class GoldenScriptError(ScriptError):
- pass
-
-
-_lots_of_failing_tests = list(map(lambda num: "test-%s.html" % num, range(0, 100)))
-
-
-class CommitQueueTaskTest(unittest.TestCase):
- def _run_and_expect_patch_analysis_result(self, commit_queue, expected_analysis_result, expected_reported_flaky_tests=[], expect_clean_tests_to_run=False, expected_failure_status_id=0):
- tool = MockTool(log_executive=True)
- patch = tool.bugs.fetch_attachment(10000)
- task = CommitQueueTask(commit_queue, patch)
-
- try:
- result = task.run()
- if result:
- analysis_result = PatchAnalysisResult.PASS
- else:
- analysis_result = PatchAnalysisResult.DEFER
- except ScriptError:
- analysis_result = PatchAnalysisResult.FAIL
-
- self.assertEqual(analysis_result, expected_analysis_result)
- self.assertEqual(frozenset(commit_queue.get_reported_flaky_tests()), frozenset(expected_reported_flaky_tests))
- self.assertEqual(commit_queue.did_run_clean_tests(), expect_clean_tests_to_run)
-
- # The failure status only means anything if we actually failed.
- if expected_analysis_result == PatchAnalysisResult.FAIL:
- self.assertEqual(task.failure_status_id, expected_failure_status_id)
- self.assertIsInstance(task.results_from_patch_test_run(patch), LayoutTestResults)
-
- def _run_through_task(self, commit_queue, expected_logs, expected_exception=None, expect_retry=False):
- self.maxDiff = None
- tool = MockTool(log_executive=True)
- patch = tool.bugs.fetch_attachment(10000)
- task = CommitQueueTask(commit_queue, patch)
- success = OutputCapture().assert_outputs(self, task.run, expected_logs=expected_logs, expected_exception=expected_exception)
- if not expected_exception:
- self.assertEqual(success, not expect_retry)
- return task
-
- def test_success_case(self):
- commit_queue = MockCommitQueue([])
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_passed: success_message='Updated working directory' patch='10000'
-run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
-command_passed: success_message='Applied patch' patch='10000'
-run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
-command_passed: success_message='ChangeLog validated' patch='10000'
-run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
-command_passed: success_message='Built patch' patch='10000'
-run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive', '--build-style=release']
-command_passed: success_message='Passed tests' patch='10000'
-run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
-command_passed: success_message='Landed patch' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs)
-
- def test_fast_success_case(self):
- commit_queue = MockCommitQueue([])
- commit_queue.did_pass_testing_ews = lambda patch: True
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_passed: success_message='Updated working directory' patch='10000'
-run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
-command_passed: success_message='Applied patch' patch='10000'
-run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
-command_passed: success_message='ChangeLog validated' patch='10000'
-run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
-command_passed: success_message='Built patch' patch='10000'
-run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
-command_passed: success_message='Landed patch' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs)
-
- def test_clean_failure(self):
- commit_queue = MockCommitQueue([
- ScriptError("MOCK clean failure"),
- ])
- expected_logs = """run_webkit_patch: ['clean']
-command_failed: failure_message='Unable to clean working directory' script_error='MOCK clean failure' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs, expect_retry=True)
-
- def test_update_failure(self):
- commit_queue = MockCommitQueue([
- None,
- ScriptError("MOCK update failure"),
- ])
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_failed: failure_message='Unable to update working directory' script_error='MOCK update failure' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs, expect_retry=True)
-
- def test_apply_failure(self):
- commit_queue = MockCommitQueue([
- None,
- None,
- GoldenScriptError("MOCK apply failure"),
- ])
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_passed: success_message='Updated working directory' patch='10000'
-run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
-command_failed: failure_message='Patch does not apply' script_error='MOCK apply failure' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
-
- def test_validate_changelog_failure(self):
- commit_queue = MockCommitQueue([
- None,
- None,
- None,
- GoldenScriptError("MOCK validate failure"),
- ])
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_passed: success_message='Updated working directory' patch='10000'
-run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
-command_passed: success_message='Applied patch' patch='10000'
-run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
-command_failed: failure_message='ChangeLog did not pass validation' script_error='MOCK validate failure' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
-
- def test_build_failure(self):
- commit_queue = MockCommitQueue([
- None,
- None,
- None,
- None,
- GoldenScriptError("MOCK build failure"),
- ])
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_passed: success_message='Updated working directory' patch='10000'
-run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
-command_passed: success_message='Applied patch' patch='10000'
-run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
-command_passed: success_message='ChangeLog validated' patch='10000'
-run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
-command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
-run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=release']
-command_passed: success_message='Able to build without patch' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
-
- def test_red_build_failure(self):
- commit_queue = MockCommitQueue([
- None,
- None,
- None,
- None,
- ScriptError("MOCK build failure"),
- ScriptError("MOCK clean build failure"),
- ])
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_passed: success_message='Updated working directory' patch='10000'
-run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
-command_passed: success_message='Applied patch' patch='10000'
-run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
-command_passed: success_message='ChangeLog validated' patch='10000'
-run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
-command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
-run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=release']
-command_failed: failure_message='Unable to build without patch' script_error='MOCK clean build failure' patch='10000'
-"""
- self._run_through_task(commit_queue, expected_logs, expect_retry=True)
-
- def test_land_failure(self):
- commit_queue = MockCommitQueue([
- None,
- None,
- None,
- None,
- None,
- None,
- GoldenScriptError("MOCK land failure"),
- ])
- expected_logs = """run_webkit_patch: ['clean']
-command_passed: success_message='Cleaned working directory' patch='10000'
-run_webkit_patch: ['update']
-command_passed: success_message='Updated working directory' patch='10000'
-run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
-command_passed: success_message='Applied patch' patch='10000'
-run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
-command_passed: success_message='ChangeLog validated' patch='10000'
-run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
-command_passed: success_message='Built patch' patch='10000'
-run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive', '--build-style=release']
-command_passed: success_message='Passed tests' patch='10000'
-run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
-command_failed: failure_message='Unable to land patch' script_error='MOCK land failure' patch='10000'
-"""
- # FIXME: This should really be expect_retry=True for a better user experiance.
- self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
-
- def test_failed_archive(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1"],
- second_test_failures=[],
- clean_test_failures=[])
-
- # It's possible for the delegate to fail to archive layout tests,
- # but we shouldn't try to report flaky tests when that happens.
- commit_queue.archive_last_test_results = lambda patch: None
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS)
-
- def test_double_flaky_test_failure(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1"],
- second_test_failures=["Fail2"],
- clean_test_failures=["Fail1"])
-
- # The (subtle) point of this test is that report_flaky_tests does not get
- # called for this run.
- # Note also that there is no attempt to run the tests w/o the patch.
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expected_reported_flaky_tests=["Fail1", "Fail2"])
-
- def test_test_failure(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1"],
- second_test_failures=["Fail1"],
- clean_test_failures=[])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_failure_status_id=1)
-
- def test_red_test_failure(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1"],
- second_test_failures=["Fail1"],
- clean_test_failures=["Fail1"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS, expect_clean_tests_to_run=True)
-
- def test_first_failure_limit(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=_lots_of_failing_tests,
- second_test_failures=[],
- clean_test_failures=[])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_failure_status_id=1)
-
- def test_first_failure_limit_with_some_tree_redness(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=_lots_of_failing_tests,
- second_test_failures=["Fail1", "Fail2", "Fail3"],
- clean_test_failures=["Fail1", "Fail2", "Fail3"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_failure_status_id=1)
-
- def test_second_failure_limit(self):
- # There need to be some failures in the first set of tests, or it won't even make it to the second test.
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1", "Fail2", "Fail3"],
- second_test_failures=_lots_of_failing_tests,
- clean_test_failures=["Fail1", "Fail2", "Fail3"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_failure_status_id=2)
-
- def test_tree_failure_limit_with_patch_that_potentially_fixes_some_redness(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1", "Fail2", "Fail3"],
- second_test_failures=["Fail1", "Fail2", "Fail3"],
- clean_test_failures=_lots_of_failing_tests)
-
- # Unfortunately there are cases where the clean build will randomly fail enough tests to hit the failure limit.
- # With that in mind, we can't actually know that this patch is good or bad until we see a clean run that doesn't
- # exceed the failure limit.
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True)
-
- def test_first_and_second_failure_limit(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=_lots_of_failing_tests,
- second_test_failures=_lots_of_failing_tests,
- clean_test_failures=[])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_failure_status_id=1)
-
- def test_first_and_clean_failure_limit(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=_lots_of_failing_tests,
- second_test_failures=[],
- clean_test_failures=_lots_of_failing_tests)
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True)
-
- def test_first_second_and_clean_failure_limit(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=_lots_of_failing_tests,
- second_test_failures=_lots_of_failing_tests,
- clean_test_failures=_lots_of_failing_tests)
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True)
-
- def test_red_tree_patch_rejection(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1", "Fail2"],
- second_test_failures=["Fail1", "Fail2"],
- clean_test_failures=["Fail1"])
-
- # failure_status_id should be of the test with patch (1), not the test without patch (2).
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_failure_status_id=1)
-
- def test_two_flaky_tests(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1"],
- second_test_failures=["Fail2"],
- clean_test_failures=["Fail1", "Fail2"])
-
- # FIXME: This should pass, but as of right now, it defers.
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expected_reported_flaky_tests=["Fail1", "Fail2"])
-
- def test_one_flaky_test(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1"],
- second_test_failures=[],
- clean_test_failures=[])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS, expected_reported_flaky_tests=["Fail1"])
-
- def test_very_flaky_patch(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
- second_test_failures=["Fail6", "Fail7", "Fail8", "Fail9", "Fail10"],
- clean_test_failures=[])
-
- # FIXME: This should actually fail, but right now it defers
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expected_reported_flaky_tests=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6", "Fail7", "Fail8", "Fail9", "Fail10"])
-
- def test_very_flaky_patch_with_some_tree_redness(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
- second_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail6", "Fail7", "Fail8", "Fail9", "Fail10"],
- clean_test_failures=["PreExistingFail1", "PreExistingFail2"])
-
- # FIXME: This should actually fail, but right now it defers
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6", "Fail7", "Fail8", "Fail9", "Fail10"])
-
- def test_different_test_failures(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6"],
- second_test_failures=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
- clean_test_failures=[])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail6"], expected_failure_status_id=1)
-
- def test_different_test_failures_with_some_tree_redness(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6"],
- second_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
- clean_test_failures=["PreExistingFail1", "PreExistingFail2"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail6"], expected_failure_status_id=1)
-
- def test_different_test_failures_with_some_tree_redness_and_some_fixes(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["PreExistingFail1", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6"],
- second_test_failures=["PreExistingFail1", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
- clean_test_failures=["PreExistingFail1", "PreExistingFail2"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail6"], expected_failure_status_id=1)
-
- def test_mildly_flaky_patch(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1"],
- second_test_failures=["Fail2"],
- clean_test_failures=[])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=False, expected_reported_flaky_tests=["Fail1", "Fail2"])
-
- def test_mildly_flaky_patch_with_some_tree_redness(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1"],
- second_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail2"],
- clean_test_failures=["PreExistingFail1", "PreExistingFail2"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail1", "Fail2"])
-
- def test_mildly_flaky_patch_with_some_tree_redness_and_flakiness(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1"],
- second_test_failures=["PreExistingFail1", "PreExistingFail2"],
- clean_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail2"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail1"])
-
- def test_tree_more_red_than_patch(self):
- commit_queue = MockSimpleTestPlanCommitQueue(
- first_test_failures=["Fail1", "Fail2", "Fail3"],
- second_test_failures=["Fail1", "Fail2", "Fail3"],
- clean_test_failures=["Fail1", "Fail2", "Fail3", "Fail4"])
-
- self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS, expect_clean_tests_to_run=True)
-
- def _expect_validate(self, patch, is_valid):
- class MockDelegate(object):
- def refetch_patch(self, patch):
- return patch
-
- task = CommitQueueTask(MockDelegate(), patch)
- self.assertEqual(task.validate(), is_valid)
-
- def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, committer="fake"):
- bug = bugzilla.Bug(bug_dict, None)
- patch = bugzilla.Attachment(attachment_dict, bug)
- patch._committer = committer
- return patch
-
- def test_validate(self):
- self._expect_validate(self._mock_patch(), True)
- self._expect_validate(self._mock_patch({'is_obsolete': True}), False)
- self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}), False)
- self._expect_validate(self._mock_patch(committer=None), False)
- self._expect_validate(self._mock_patch({'review': '-'}), False)
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py (261318 => 261319)
--- trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py 2020-05-07 18:21:48 UTC (rev 261318)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py 2020-05-07 18:24:04 UTC (rev 261319)
@@ -46,20 +46,6 @@
from webkitpy.tool.mocktool import MockTool, MockOptions
-class TestCommitQueue(CommitQueue):
- def __init__(self, tool=None):
- CommitQueue.__init__(self)
- if tool:
- self.bind_to_tool(tool)
- self._options = MockOptions(confirm=False, parent_command="commit-queue", port=None)
-
- def begin_work_queue(self):
- output_capture = OutputCapture()
- output_capture.capture_output()
- CommitQueue.begin_work_queue(self)
- output_capture.restore_output()
-
-
class TestQueue(AbstractPatchQueue):
name = "test-queue"
@@ -167,281 +153,6 @@
raise CheckoutNeedsUpdate([], 1, "", None)
-class AlwaysCommitQueueTool(object):
- def __init__(self):
- self.status_server = MockStatusServer()
-
- def command_by_name(self, name):
- return CommitQueue
-
-
-class SecondThoughtsCommitQueue(TestCommitQueue):
- def __init__(self, tool=None):
- self._reject_patch = False
- TestCommitQueue.__init__(self, tool)
-
- def run_command(self, command):
- # We want to reject the patch after the first validation,
- # so wait to reject it until after some other command has run.
- self._reject_patch = True
- return CommitQueue.run_command(self, command)
-
- def refetch_patch(self, patch):
- if not self._reject_patch:
- return self._tool.bugs.fetch_attachment(patch.id())
-
- attachment_dictionary = {
- "id": patch.id(),
- "bug_id": patch.bug_id(),
- "name": "Rejected",
- "is_obsolete": True,
- "is_patch": False,
- "review": "-",
- "reviewer_email": "[email protected]",
- "commit-queue": "-",
- "committer_email": "[email protected]",
- "attacher_email": "Contributer1",
- }
- return Attachment(attachment_dictionary, None)
-
-
-class CommitQueueTest(QueuesTest):
- def _mock_test_result(self, testname):
- return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
-
- def test_commit_queue(self):
- tool = MockTool()
- tool.filesystem.write_text_file('/tmp/layout-test-results/full_results.json', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem.
- tool.filesystem.write_text_file('/tmp/layout-test-results/webkit_unit_tests_output.xml', '')
- expected_logs = {
- "begin_work_queue": self._default_begin_work_queue_logs("commit-queue"),
- "process_work_item": """Running: webkit-patch clean --port=mac
-MOCK: update_status: commit-queue Cleaned working directory
-Running: webkit-patch update --port=mac
-MOCK: update_status: commit-queue Updated working directory
-Running: webkit-patch apply-attachment --no-update --non-interactive 10000 --port=mac
-MOCK: update_status: commit-queue Applied patch
-Running: webkit-patch validate-changelog --check-oops --non-interactive 10000 --port=mac
-MOCK: update_status: commit-queue ChangeLog validated
-Running: webkit-patch build --no-clean --no-update --build-style=release --port=mac
-MOCK: update_status: commit-queue Built patch
-Running: webkit-patch build-and-test --no-clean --no-update --test --non-interactive --build-style=release --port=mac
-MOCK: update_status: commit-queue Passed tests
-Running: webkit-patch land-attachment --force-clean --non-interactive --parent-command=commit-queue 10000 --port=mac
-MOCK: update_status: commit-queue Landed patch
-MOCK: update_status: commit-queue Pass
-MOCK: release_work_item: commit-queue 10000
-""",
- "handle_script_error": "ScriptError error message\n\nMOCK output\n",
- "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '10000' with comment 'Rejecting attachment 10000 from commit-queue.\n\nMock error message'\n",
- }
- self.assert_queue_outputs(CommitQueue(), tool=tool, expected_logs=expected_logs)
-
- def test_commit_queue_failure(self):
- expected_logs = {
- "begin_work_queue": self._default_begin_work_queue_logs("commit-queue"),
- "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory
-MOCK: update_status: commit-queue Updated working directory
-MOCK: update_status: commit-queue Patch does not apply
-MOCK setting flag 'commit-queue' to '-' on attachment '10000' with comment 'Rejecting attachment 10000 from commit-queue.\n\nMOCK script error
-Full output: http://dummy_url'
-MOCK: update_status: commit-queue Fail
-MOCK: release_work_item: commit-queue 10000
-""",
- "handle_script_error": "ScriptError error message\n\nMOCK output\n",
- "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '10000' with comment 'Rejecting attachment 10000 from commit-queue.\n\nMock error message'\n",
- }
- queue = CommitQueue()
-
- def mock_run_webkit_patch(command):
- if command[0] == 'clean' or command[0] == 'update':
- # We want cleaning to succeed so we can error out on a step
- # that causes the commit-queue to reject the patch.
- return
- raise ScriptError('MOCK script error')
-
- queue.run_webkit_patch = mock_run_webkit_patch
- self.assert_queue_outputs(queue, expected_logs=expected_logs)
-
- def test_commit_queue_failure_with_failing_tests(self):
- expected_logs = {
- "begin_work_queue": self._default_begin_work_queue_logs("commit-queue"),
- "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory
-MOCK: update_status: commit-queue Updated working directory
-MOCK: update_status: commit-queue Patch does not apply
-MOCK setting flag 'commit-queue' to '-' on attachment '10000' with comment 'Rejecting attachment 10000 from commit-queue.\n\nNew failing tests:
-mock_test_name.html
-Full output: http://dummy_url'
-MOCK: update_status: commit-queue Fail
-MOCK: release_work_item: commit-queue 10000
-""",
- "handle_script_error": "ScriptError error message\n\nMOCK output\n",
- "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '10000' with comment 'Rejecting attachment 10000 from commit-queue.\n\nMock error message'\n",
- }
-
- class MockCommitQueueTask(CommitQueueTask):
- def results_from_patch_test_run(self, patch):
- return LayoutTestResults([test_results.TestResult("mock_test_name.html", failures=[test_failures.FailureTextMismatch()])], did_exceed_test_failure_limit=False)
-
- queue = CommitQueue(MockCommitQueueTask)
-
- def mock_run_webkit_patch(command):
- if command[0] == 'clean' or command[0] == 'update':
- # We want cleaning to succeed so we can error out on a step
- # that causes the commit-queue to reject the patch.
- return
- raise ScriptError('MOCK script error')
-
- queue.run_webkit_patch = mock_run_webkit_patch
- self.assert_queue_outputs(queue, expected_logs=expected_logs)
-
- def test_revert(self):
- tool = MockTool()
- tool.filesystem.write_text_file('/tmp/layout-test-results/full_results.json', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem.
- tool.filesystem.write_text_file('/tmp/layout-test-results/webkit_unit_tests_output.xml', '')
- tool.buildbot.light_tree_on_fire()
- expected_logs = {
- "begin_work_queue": self._default_begin_work_queue_logs("commit-queue"),
- "process_work_item": """Running: webkit-patch clean --port=%(port)s
-MOCK: update_status: commit-queue Cleaned working directory
-Running: webkit-patch update --port=%(port)s
-MOCK: update_status: commit-queue Updated working directory
-Running: webkit-patch apply-attachment --no-update --non-interactive 10000 --port=%(port)s
-MOCK: update_status: commit-queue Applied patch
-Running: webkit-patch validate-changelog --check-oops --non-interactive 10000 --port=%(port)s
-MOCK: update_status: commit-queue ChangeLog validated
-Running: webkit-patch build --no-clean --no-update --build-style=release --port=%(port)s
-MOCK: update_status: commit-queue Built patch
-Running: webkit-patch build-and-test --no-clean --no-update --test --non-interactive --build-style=release --port=%(port)s
-MOCK: update_status: commit-queue Passed tests
-Running: webkit-patch land-attachment --force-clean --non-interactive --parent-command=commit-queue 10000 --port=%(port)s
-MOCK: update_status: commit-queue Landed patch
-MOCK: update_status: commit-queue Pass
-MOCK: release_work_item: commit-queue 10000
-""" % {"port": "mac"},
- "handle_script_error": "ScriptError error message\n\nMOCK output\n",
- "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '10000' with comment 'Rejecting attachment 10000 from commit-queue.\n\nMock error message'\n",
- }
- self.assert_queue_outputs(CommitQueue(), tool=tool, expected_logs=expected_logs)
-
- def test_revert_lands(self):
- tool = MockTool()
- tool.buildbot.light_tree_on_fire()
- revert_patch = tool.bugs.fetch_attachment(10005) # _patch6, a revert patch.
- assert(revert_patch.is_revert())
- expected_logs = {
- "begin_work_queue": self._default_begin_work_queue_logs("commit-queue"),
- "process_work_item": """Running: webkit-patch clean --port=%(port)s
-MOCK: update_status: commit-queue Cleaned working directory
-Running: webkit-patch update --port=%(port)s
-MOCK: update_status: commit-queue Updated working directory
-Running: webkit-patch apply-attachment --no-update --non-interactive 10005 --port=%(port)s
-MOCK: update_status: commit-queue Applied patch
-Running: webkit-patch validate-changelog --check-oops --non-interactive 10005 --port=%(port)s
-MOCK: update_status: commit-queue ChangeLog validated
-Running: webkit-patch land-attachment --force-clean --non-interactive --parent-command=commit-queue 10005 --port=%(port)s
-MOCK: update_status: commit-queue Landed patch
-MOCK: update_status: commit-queue Pass
-MOCK: release_work_item: commit-queue 10005
-""" % {"port": "mac"},
- "handle_script_error": "ScriptError error message\n\nMOCK output\n",
- "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '10005' with comment 'Rejecting attachment 10005 from commit-queue.\n\nMock error message'\n",
- }
- self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=revert_patch, expected_logs=expected_logs)
-
- def test_non_valid_patch(self):
- tool = MockTool()
- patch = tool.bugs.fetch_attachment(10007) # _patch8, resolved bug, without review flag, not marked obsolete (maybe already landed)
- expected_logs = {
- "begin_work_queue": self._default_begin_work_queue_logs("commit-queue"),
- "process_work_item": """MOCK: update_status: commit-queue Error: commit-queue did not process patch. Reason: Bug is already closed.
-MOCK: release_work_item: commit-queue 10007
-""",
- }
- self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=patch, expected_logs=expected_logs)
-
- def test_auto_retry(self):
- queue = CommitQueue()
- options = Mock()
- options.parent_command = "commit-queue"
- tool = AlwaysCommitQueueTool()
- sequence = NeedsUpdateSequence(None)
-
- expected_logs = """Commit failed because the checkout is out of date. Please update and try again.
-MOCK: update_status: commit-queue Tests passed, but commit failed (checkout out of date). Updating, then landing without building or re-running tests.
-"""
- state = {'patch': None}
- OutputCapture().assert_outputs(self, sequence.run_and_handle_errors, [tool, options, state], expected_exception=TryAgain, expected_logs=expected_logs)
-
- self.assertTrue(options.update)
- self.assertFalse(options.build)
- self.assertFalse(options.test)
-
- def test_manual_reject_during_processing(self):
- queue = SecondThoughtsCommitQueue(MockTool())
- queue.begin_work_queue()
- queue._tool.filesystem.write_text_file('/tmp/layout-test-results/full_results.json', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem.
- queue._tool.filesystem.write_text_file('/tmp/layout-test-results/webkit_unit_tests_output.xml', '')
- queue._options = Mock()
- queue._options.port = None
- expected_logs = """Running: webkit-patch clean --port=mac
-MOCK: update_status: commit-queue Cleaned working directory
-MOCK: update_status: commit-queue Error: commit-queue did not process patch. Reason: Patch is obsolete.
-MOCK: release_work_item: commit-queue 10000
-"""
- self.maxDiff = None
- OutputCapture().assert_outputs(self, queue.process_work_item, [QueuesTest.mock_work_item], expected_logs=expected_logs)
-
- def test_report_flaky_tests(self):
- queue = TestCommitQueue(MockTool())
- expected_logs = """MOCK bug comment: bug_id=50002, cc=None, see_also=None
---- Begin comment ---
-The commit-queue just saw foo/bar.html flake (text diff) while processing attachment 10000 on bug 50000.
-Port: MockPort Platform: MockPlatform 1.0
---- End comment ---
-
-MOCK add_attachment_to_bug: bug_id=50002, description=Failure diff from bot filename=failure.diff mimetype=None
-MOCK bug comment: bug_id=50002, cc=None, see_also=None
---- Begin comment ---
-The commit-queue just saw bar/baz.html flake (text diff) while processing attachment 10000 on bug 50000.
-Port: MockPort Platform: MockPlatform 1.0
---- End comment ---
-
-bar/baz-diffs.txt does not exist in results archive, uploading entire archive.
-MOCK add_attachment_to_bug: bug_id=50002, description=Archive of layout-test-results from bot filename=layout-test-results.zip mimetype=None
-MOCK bug comment: bug_id=50000, cc=None, see_also=None
---- Begin comment ---
-The commit-queue encountered the following flaky tests while processing attachment 10000:
-
-foo/bar.html bug 50002 (author: [email protected])
-bar/baz.html bug 50002 (author: [email protected])
-The commit-queue is continuing to process your patch.
---- End comment ---
-
-"""
- test_names = ["foo/bar.html", "bar/baz.html"]
- test_results = [self._mock_test_result(name) for name in test_names]
-
- class MockZipFile(object):
- def __init__(self):
- self.fp = StringIO()
-
- def read(self, path):
- return ""
-
- def namelist(self):
- # This is intentionally missing one diffs.txt to exercise the "upload the whole zip" codepath.
- return ['foo/bar-diffs.txt']
-
- OutputCapture().assert_outputs(self, queue.report_flaky_tests, [QueuesTest.mock_work_item, test_results, MockZipFile()], expected_logs=expected_logs)
-
- def test_did_pass_testing_ews(self):
- tool = MockTool()
- patch = tool.bugs.fetch_attachment(10000)
- queue = TestCommitQueue(tool)
- self.assertFalse(queue.did_pass_testing_ews(patch))
-
-
class StyleQueueTest(QueuesTest):
def test_style_queue_with_style_exception(self):
expected_logs = {
