Diff
Modified: trunk/Tools/ChangeLog (252271 => 252272)
--- trunk/Tools/ChangeLog 2019-11-08 23:40:31 UTC (rev 252271)
+++ trunk/Tools/ChangeLog 2019-11-09 00:03:56 UTC (rev 252272)
@@ -1,3 +1,53 @@
+2019-11-08 Jonathan Bedard <jbed...@apple.com>
+
+ results.webkit.org: List failing tests for criteria
+ https://bugs.webkit.org/show_bug.cgi?id=203905
+
+ Rubber-stamped by Aakash Jain.
+
+ * resultsdbpy/resultsdbpy/controller/api_routes.py:
+ (APIRoutes.__init__): Add FailureController and /failures endpoint.
+ * resultsdbpy/resultsdbpy/controller/failure_controller.py: Added.
+ (FailureController): Added FailureController object to drive /failure endpoint.
+ (FailureController.__init__):
+ (FailureController.failures): Return a json list of failures for a range.
+ * resultsdbpy/resultsdbpy/controller/failure_controller_unittest.py: Added.
+ (FailureControllerTest): Test the /failure API.
+ (FailureControllerTest.setup_webserver):
+ (FailureControllerTest.test_failures_collapsed):
+ (FailureControllerTest.test_unexpected_failures_collapsed):
+ (FailureControllerTest.test_failures):
+ (FailureControllerTest.test_unexpected_failures):
+ (FailureControllerTest.test_failure_by_time):
+ * resultsdbpy/resultsdbpy/model/failure_context.py: Added.
+ (FailureContext): Database access to tables recording test failures.
+ (FailureContext.TestFailuresBase):
+ (FailureContext.TestFailuresBase.unpack):
+ (FailureContext.TestFailuresByCommit): Save test failures sorted by commit.
+ (FailureContext.TestFailuresByStartTime): Save test failures sorted by start time.
+ (FailureContext.UnexpectedTestFailuresByCommit): Save unexpected test failures
+ sorted by commit.
+ (FailureContext.UnexpectedTestFailuresByStartTime): Save unexpected test failures
+ sorted by commit.
+ (FailureContext.__init__):
+ (FailureContext.register): Register test failures to each of the 4 tables for an upload.
+ (FailureContext._failures): Return a list of test failures from a table.
+ (FailureContext.failures_by_commit): Return a list of test failures from a table sorted
+ by commit.
+ (FailureContext.failures_by_start_time): Return a list of test failures from a table
+ sorted by commit.
+ * resultsdbpy/resultsdbpy/model/failure_context_unittest.py: Added.
+ (FailureContextTest):
+ (FailureContextTest.init_database):
+ (FailureContextTest.test_failures_collapsed):
+ (FailureContextTest.test_unexpected_failures_collapsed):
+ (FailureContextTest.test_failures):
+ (FailureContextTest.test_unexpected_failures):
+ * resultsdbpy/resultsdbpy/model/model.py:
+ (Model.__init__):
+ * resultsdbpy/resultsdbpy/view/templates/documentation.html: Add documentation
+ for the /failure endpoint.
+
2019-11-08 Alex Christensen <achristen...@webkit.org>
Fix flaky tests in http/tests/cache/disk-cache
Modified: trunk/Tools/resultsdbpy/resultsdbpy/controller/api_routes.py (252271 => 252272)
--- trunk/Tools/resultsdbpy/resultsdbpy/controller/api_routes.py 2019-11-08 23:40:31 UTC (rev 252271)
+++ trunk/Tools/resultsdbpy/resultsdbpy/controller/api_routes.py 2019-11-09 00:03:56 UTC (rev 252272)
@@ -26,6 +26,7 @@
from resultsdbpy.controller.archive_controller import ArchiveController
from resultsdbpy.controller.commit_controller import CommitController
from resultsdbpy.controller.ci_controller import CIController
+from resultsdbpy.controller.failure_controller import FailureController
from resultsdbpy.controller.suite_controller import SuiteController
from resultsdbpy.controller.test_controller import TestController
from resultsdbpy.controller.upload_controller import UploadController
@@ -42,6 +43,7 @@
self.suite_controller = SuiteController(suite_context=model.suite_context)
self.test_controller = TestController(test_context=model.test_context)
+ self.failure_controller = FailureController(failure_context=model.failure_context)
self.ci_controller = CIController(ci_context=model.ci_context, upload_context=model.upload_context)
self.archive_controller = ArchiveController(commit_controller=self.commit_controller, archive_context=model.archive_context, upload_context=model.upload_context)
@@ -68,6 +70,8 @@
self.add_url_rule('/results/<path:suite>', 'suite-results', self.suite_controller.find_run_results, methods=('GET',))
self.add_url_rule('/results/<path:suite>/<path:test>', 'test-results', self.test_controller.find_test_result, methods=('GET',))
+ self.add_url_rule('/failures/<path:suite>', 'suite-failures', self.failure_controller.failures, methods=('GET',))
+
self.add_url_rule('/urls/queue', 'queue-urls', self.ci_controller.urls_for_queue_endpoint, methods=('GET',))
self.add_url_rule('/urls', 'build-urls', self.ci_controller.urls_for_builds_endpoint, methods=('GET',))
Added: trunk/Tools/resultsdbpy/resultsdbpy/controller/failure_controller.py (0 => 252272)
--- trunk/Tools/resultsdbpy/resultsdbpy/controller/failure_controller.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/controller/failure_controller.py 2019-11-09 00:03:56 UTC (rev 252272)
@@ -0,0 +1,95 @@
+# Copyright (C) 2019 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from flask import abort, jsonify
+from resultsdbpy.controller.commit_controller import uuid_range_for_query, HasCommitContext
+from resultsdbpy.controller.configuration import Configuration
+from resultsdbpy.controller.configuration_controller import configuration_for_query
+from resultsdbpy.controller.suite_controller import time_range_for_query
+from resultsdbpy.flask_support.util import AssertRequest, query_as_kwargs, limit_for_query, boolean_query
+
+
+class FailureController(HasCommitContext):
+ DEFAULT_LIMIT = 5000
+
+ def __init__(self, failure_context):
+ super(FailureController, self).__init__(failure_context.commit_context)
+ self.failure_context = failure_context
+
+ @query_as_kwargs()
+ @uuid_range_for_query()
+ @limit_for_query(DEFAULT_LIMIT)
+ @configuration_for_query()
+ @time_range_for_query()
+ def failures(
+ self, suite=None,
+ configurations=None, recent=None,
+ branch=None, begin=None, end=None,
+ begin_query_time=None, end_query_time=None,
+ unexpected=None, collapsed=None,
+ limit=None, **kwargs
+ ):
+ AssertRequest.is_type(['GET'])
+ AssertRequest.query_kwargs_empty(**kwargs)
+
+ recent = boolean_query(*recent)[0] if recent else True
+ unexpected = boolean_query(*unexpected)[0] if unexpected else True
+ collapsed = boolean_query(*collapsed)[0] if collapsed else True
+
+ if not suite:
+ abort(400, description='No suite specified')
+
+ with self.failure_context:
+ query_dict = dict(
+ suite=suite,
+ configurations=configurations, recent=recent,
+ branch=branch[0], begin=begin, end=end,
+ begin_query_time=begin_query_time, end_query_time=end_query_time,
+ limit=limit, unexpected=unexpected, collapsed=collapsed,
+ )
+ num_specified_commits = sum([1 if element else 0 for element in [begin, end]])
+ num_specified_timestamps = sum([1 if element else 0 for element in [begin_query_time, end_query_time]])
+
+ if num_specified_commits >= num_specified_timestamps:
+ find_function = self.failure_context.failures_by_commit
+
+ def sort_function(result):
+ return result['uuid']
+
+ else:
+ find_function = self.failure_context.failures_by_start_time
+
+ def sort_function(result):
+ return result['start_time']
+
+ if collapsed:
+ response = set()
+ response.update(find_function(**query_dict))
+ return jsonify(sorted(response))
+
+ response = []
+ for config, results in find_function(**query_dict).items():
+ response.append(dict(
+ configuration=Configuration.Encoder().default(config),
+ results=sorted(results, key=sort_function),
+ ))
+ return jsonify(response)
Added: trunk/Tools/resultsdbpy/resultsdbpy/controller/failure_controller_unittest.py (0 => 252272)
--- trunk/Tools/resultsdbpy/resultsdbpy/controller/failure_controller_unittest.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/controller/failure_controller_unittest.py 2019-11-09 00:03:56 UTC (rev 252272)
@@ -0,0 +1,108 @@
+# Copyright (C) 2019 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import time
+
+from fakeredis import FakeStrictRedis
+from redis import StrictRedis
+from resultsdbpy.controller.api_routes import APIRoutes
+from resultsdbpy.flask_support.flask_testcase import FlaskTestCase
+from resultsdbpy.model.cassandra_context import CassandraContext
+from resultsdbpy.model.mock_cassandra_context import MockCassandraContext
+from resultsdbpy.model.mock_model_factory import MockModelFactory
+from resultsdbpy.model.test_context import Expectations
+from resultsdbpy.model.wait_for_docker_test_case import WaitForDockerTestCase
+
+
+class FailureControllerTest(FlaskTestCase, WaitForDockerTestCase):
+ KEYSPACE = 'failure_controller_test_keyspace'
+
+ @classmethod
+ def setup_webserver(cls, app, redis=StrictRedis, cassandra=CassandraContext):
+ cassandra.drop_keyspace(keyspace=cls.KEYSPACE)
+ model = MockModelFactory.create(redis=redis(), cassandra=cassandra(keyspace=cls.KEYSPACE, create_keyspace=True))
+ app.register_blueprint(APIRoutes(model))
+
+ MockModelFactory.add_mock_results(model, test_results=dict(
+ details=dict(link='dummy-link'),
+ run_stats=dict(tests_skipped=0),
+ results={
+ 'fast': {
+ 'encoding': {
+ 'css-cached-bom.html': dict(expected='PASS', actual='FAIL', time=1.2),
+ 'css-charset-default.xhtml': dict(expected='FAIL', actual='FAIL', time=1.2),
+ 'css-charset.html': dict(expected='FAIL', actual='PASS', time=1.2),
+ 'css-link-charset.html': dict(expected='PASS', actual='PAS', time=1.2),
+ }
+ }
+ },
+ ))
+ MockModelFactory.process_results(model)
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_failures_collapsed(self, client, **kwargs):
+ response = client.get(self.URL + '/api/failures/layout-tests?platform=iOS&style=Debug&unexpected=False')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()), 2)
+ self.assertEqual(response.json(), ['fast/encoding/css-cached-bom.html', 'fast/encoding/css-charset-default.xhtml'])
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_unexpected_failures_collapsed(self, client, **kwargs):
+ response = client.get(self.URL + '/api/failures/layout-tests?platform=iOS&style=Debug')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()), 1)
+ self.assertEqual(response.json(), ['fast/encoding/css-cached-bom.html'])
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_failures(self, client, **kwargs):
+ response = client.get(self.URL + '/api/failures/layout-tests?platform=Mac&style=Debug&unexpected=False&collapsed=False')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()), 2)
+ self.assertEqual(len(response.json()[0]['results']), 5)
+ self.assertEqual(response.json()[0]['results'][0]['fast/encoding/css-cached-bom.html'], 'FAIL')
+ self.assertEqual(response.json()[0]['results'][0]['fast/encoding/css-charset-default.xhtml'], 'FAIL')
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_unexpected_failures(self, client, **kwargs):
+ response = client.get(self.URL + '/api/failures/layout-tests?platform=Mac&style=Debug&collapsed=False')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()), 2)
+ self.assertEqual(len(response.json()[0]['results']), 5)
+ self.assertEqual(response.json()[0]['results'][0]['fast/encoding/css-cached-bom.html'], 'FAIL')
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_failure_by_time(self, client, **kwargs):
+ response = client.get(f'{self.URL}/api/failures/layout-tests?platform=iOS&style=Debug&recent=False&after_time={time.time() - 60 * 60}&collapsed=False')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(response.json()), 2)
+ print(response.json())
+ for i in range(2):
+ self.assertEqual(len(response.json()[i]['results']), 5)
+ last_start_time = 0
+ for result in response.json()[i]['results']:
+ self.assertGreaterEqual(result['start_time'], last_start_time)
+ last_start_time = result['start_time']
Added: trunk/Tools/resultsdbpy/resultsdbpy/model/failure_context.py (0 => 252272)
--- trunk/Tools/resultsdbpy/resultsdbpy/model/failure_context.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/model/failure_context.py 2019-11-09 00:03:56 UTC (rev 252272)
@@ -0,0 +1,195 @@
+# Copyright (C) 2019 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import calendar
+import collections
+import json
+import time
+
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model
+from datetime import datetime
+from resultsdbpy.controller.commit import Commit
+from resultsdbpy.model.commit_context import CommitContext
+from resultsdbpy.model.configuration_context import ClusteredByConfiguration
+from resultsdbpy.model.upload_context import UploadCallbackContext
+from resultsdbpy.model.test_context import Expectations
+
+
+class FailureContext(UploadCallbackContext):
+ DEFAULT_LIMIT = 100
+
+ class TestFailuresBase(ClusteredByConfiguration):
+ suite = columns.Text(partition_key=True, required=True)
+ branch = columns.Text(partition_key=True, required=True)
+ tests = columns.Text(required=True)
+
+ def unpack(self):
+ results = json.loads(self.tests) if self.tests else {}
+ for test in results.keys():
+ results[test] = Expectations.state_ids_to_string([results[test]])
+ results['uuid'] = self.uuid
+ results['start_time'] = calendar.timegm(self.start_time.timetuple())
+ return results
+
+ class TestFailuresByCommit(TestFailuresBase):
+ __table_name__ = 'test_failures_by_commit'
+ uuid = columns.BigInt(primary_key=True, required=True, clustering_order='DESC')
+ sdk = columns.Text(primary_key=True, required=True)
+ start_time = columns.DateTime(primary_key=True, required=True)
+
+ class TestFailuresByStartTime(TestFailuresBase):
+ __table_name__ = 'test_failures_by_start_time'
+ start_time = columns.DateTime(primary_key=True, required=True, clustering_order='DESC')
+ sdk = columns.Text(primary_key=True, required=True)
+ uuid = columns.BigInt(primary_key=True, required=True)
+
+ class UnexpectedTestFailuresByCommit(TestFailuresBase):
+ __table_name__ = 'unexpected_test_failures_by_commit'
+ uuid = columns.BigInt(primary_key=True, required=True, clustering_order='DESC')
+ sdk = columns.Text(primary_key=True, required=True)
+ start_time = columns.DateTime(primary_key=True, required=True)
+
+ class UnexpectedTestFailuresByStartTime(TestFailuresBase):
+ __table_name__ = 'unexpected_test_failures_by_start_time'
+ start_time = columns.DateTime(primary_key=True, required=True, clustering_order='DESC')
+ sdk = columns.Text(primary_key=True, required=True)
+ uuid = columns.BigInt(primary_key=True, required=True)
+
+ def __init__(self, *args, **kwargs):
+ super(FailureContext, self).__init__('test-failures', *args, **kwargs)
+
+ with self:
+ self.cassandra.create_table(self.TestFailuresByCommit)
+ self.cassandra.create_table(self.TestFailuresByStartTime)
+ self.cassandra.create_table(self.UnexpectedTestFailuresByCommit)
+ self.cassandra.create_table(self.UnexpectedTestFailuresByStartTime)
+
+ def register(self, configuration, commits, suite, test_results, timestamp=None):
+ try:
+ if not isinstance(suite, str):
+ raise TypeError(f'Expected type {str}, got {type(suite)}')
+
+ timestamp = timestamp or time.time()
+ if isinstance(timestamp, datetime):
+ timestamp = calendar.timegm(timestamp.timetuple())
+
+ with self:
+ uuid = self.commit_context.uuid_for_commits(commits)
+ ttl = int((uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER) + self.ttl_seconds - time.time()) if self.ttl_seconds else None
+
+ def callback(test, result, failures, unexpected):
+ failed_result = Expectations.string_to_state_ids(result.get('actual', ''))
+ expected = set(Expectations.string_to_state_ids(result.get('expected', '')))
+ if Expectations.STRING_TO_STATE_ID[Expectations.FAIL] in expected:
+ expected.add(Expectations.STRING_TO_STATE_ID[Expectations.TEXT])
+ expected.add(Expectations.STRING_TO_STATE_ID[Expectations.AUDIO])
+ expected.add(Expectations.STRING_TO_STATE_ID[Expectations.IMAGE])
+
+ unexpected_result = set(failed_result) - expected
+ if failed_result:
+ worst = min(failed_result)
+ if worst < Expectations.STRING_TO_STATE_ID[Expectations.WARNING]:
+ failures[test] = min(worst, failures.get(test, Expectations.STRING_TO_STATE_ID[Expectations.PASS]))
+
+ if unexpected_result:
+ worst = min(unexpected_result)
+ if worst < Expectations.STRING_TO_STATE_ID[Expectations.WARNING]:
+ unexpected[test] = min(worst, unexpected.get(test, Expectations.STRING_TO_STATE_ID[Expectations.PASS]))
+
+ with self.cassandra.batch_query_context():
+ for branch in self.commit_context.branch_keys_for_commits(commits):
+ failures = {}
+ unexpected = {}
+
+ Expectations.iterate_through_nested_results(
+ test_results.get('results'),
+ lambda test, result: callback(test, result, failures=failures, unexpected=unexpected),
+ )
+
+ for table in [self.TestFailuresByCommit, self.TestFailuresByStartTime]:
+ self.configuration_context.insert_row_with_configuration(
+ table.__table_name__, configuration=configuration, suite=suite,
+ branch=branch, uuid=uuid, ttl=ttl,
+ sdk=configuration.sdk or '?', start_time=timestamp,
+ tests=json.dumps(failures),
+ )
+
+ for table in [self.UnexpectedTestFailuresByCommit, self.UnexpectedTestFailuresByStartTime]:
+ self.configuration_context.insert_row_with_configuration(
+ table.__table_name__, configuration=configuration, suite=suite,
+ branch=branch, uuid=uuid, ttl=ttl,
+ sdk=configuration.sdk or '?', start_time=timestamp,
+ tests=json.dumps(unexpected),
+ )
+
+ except Exception as e:
+ return self.partial_status(e)
+ return self.partial_status()
+
+ def _failures(
+ self, all_table, unexpected_table, configurations, suite, recent=True,
+ branch=None, begin=None, end=None,
+ begin_query_time=None, end_query_time=None,
+ unexpected=True, collapsed=True, limit=DEFAULT_LIMIT,
+ ):
+ table = unexpected_table if unexpected else all_table
+ if not isinstance(suite, str):
+ raise TypeError(f'Expected type {str} for suite, got {type(suite)}')
+
+ def get_time(time):
+ if isinstance(time, datetime):
+ return time
+ elif time:
+ return datetime.utcfromtimestamp(int(time))
+ return None
+
+ with self:
+ if collapsed:
+ result = set()
+ else:
+ result = {}
+
+ for configuration in configurations:
+ for config, values in self.configuration_context.select_from_table_with_configurations(
+ table.__table_name__, configurations=[configuration], recent=recent,
+ suite=suite, sdk=configuration.sdk, branch=branch or self.commit_context.DEFAULT_BRANCH_KEY,
+ uuid__gte=CommitContext.convert_to_uuid(begin),
+ uuid__lte=CommitContext.convert_to_uuid(end, CommitContext.timestamp_to_uuid()),
+ start_time__gte=get_time(begin_query_time), start_time__lte=get_time(end_query_time),
+ limit=limit,
+ ).items():
+ if collapsed:
+ for value in values:
+ for test in value.unpack():
+ if test not in ['uuid', 'start_time']:
+ result.add(test)
+ else:
+ result.update({config: [value.unpack() for value in values]})
+
+ return result
+
+ def failures_by_commit(self, *args, **kwargs):
+ return self._failures(self.TestFailuresByCommit, self.UnexpectedTestFailuresByCommit, *args, **kwargs)
+
+ def failures_by_start_time(self, *args, **kwargs):
+ return self._failures(self.TestFailuresByStartTime, self.UnexpectedTestFailuresByStartTime, *args, **kwargs)
Added: trunk/Tools/resultsdbpy/resultsdbpy/model/failure_context_unittest.py (0 => 252272)
--- trunk/Tools/resultsdbpy/resultsdbpy/model/failure_context_unittest.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/model/failure_context_unittest.py 2019-11-09 00:03:56 UTC (rev 252272)
@@ -0,0 +1,111 @@
+# Copyright (C) 2019 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import time
+
+from fakeredis import FakeStrictRedis
+from redis import StrictRedis
+from resultsdbpy.controller.configuration import Configuration
+from resultsdbpy.model.cassandra_context import CassandraContext
+from resultsdbpy.model.mock_cassandra_context import MockCassandraContext
+from resultsdbpy.model.mock_model_factory import MockModelFactory
+from resultsdbpy.model.mock_repository import MockSVNRepository
+from resultsdbpy.model.test_context import Expectations
+from resultsdbpy.model.wait_for_docker_test_case import WaitForDockerTestCase
+
+
+class FailureContextTest(WaitForDockerTestCase):
+ KEYSPACE = 'failure_context_test_keyspace'
+
+ def init_database(self, redis=StrictRedis, cassandra=CassandraContext):
+ cassandra.drop_keyspace(keyspace=self.KEYSPACE)
+ self.model = MockModelFactory.create(redis=redis(), cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True))
+ MockModelFactory.add_mock_results(self.model, test_results=dict(
+ details=dict(link='dummy-link'),
+ run_stats=dict(tests_skipped=0),
+ results={
+ 'fast': {
+ 'encoding': {
+ 'css-cached-bom.html': dict(expected='PASS', actual='FAIL', time=1.2),
+ 'css-charset-default.xhtml': dict(expected='FAIL', actual='FAIL', time=1.2),
+ 'css-charset.html': dict(expected='FAIL', actual='PASS', time=1.2),
+ 'css-link-charset.html': dict(expected='PASS', actual='PASS', time=1.2),
+ }
+ }
+ },
+ ))
+ MockModelFactory.process_results(self.model)
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_failures_collapsed(self, redis=StrictRedis, cassandra=CassandraContext):
+ self.init_database(redis=redis, cassandra=cassandra)
+ results = self.model.failure_context.failures_by_commit(
+ configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
+ suite='layout-tests', recent=True, unexpected=False,
+ )
+
+ self.assertEqual(len(results), 2)
+ self.assertEqual(results, set(['fast/encoding/css-cached-bom.html', 'fast/encoding/css-charset-default.xhtml']))
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_unexpected_failures_collapsed(self, redis=StrictRedis, cassandra=CassandraContext):
+ self.init_database(redis=redis, cassandra=cassandra)
+ results = self.model.failure_context.failures_by_commit(
+ configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
+ suite='layout-tests', recent=True,
+ )
+
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results, set(['fast/encoding/css-cached-bom.html']))
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_failures(self, redis=StrictRedis, cassandra=CassandraContext):
+ self.init_database(redis=redis, cassandra=cassandra)
+ results = self.model.failure_context.failures_by_commit(
+ configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
+ suite='layout-tests', recent=True, collapsed=False, unexpected=False,
+ )
+
+ self.assertEqual(len(results), 1)
+ self.assertEqual(len(list(results.values())[0]), 5)
+ self.assertEqual(list(results.values())[0][0], {
+ 'fast/encoding/css-cached-bom.html': 'FAIL',
+ 'fast/encoding/css-charset-default.xhtml': 'FAIL',
+ 'start_time': list(results.values())[0][0]['start_time'],
+ 'uuid': 153802947900
+ })
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_unexpected_failures(self, redis=StrictRedis, cassandra=CassandraContext):
+ self.init_database(redis=redis, cassandra=cassandra)
+ results = self.model.failure_context.failures_by_commit(
+ configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
+ suite='layout-tests', recent=True, collapsed=False
+ )
+
+ self.assertEqual(len(results), 1)
+ self.assertEqual(len(list(results.values())[0]), 5)
+ self.assertEqual(list(results.values())[0][0], {
+ 'fast/encoding/css-cached-bom.html': 'FAIL',
+ 'start_time': list(results.values())[0][0]['start_time'],
+ 'uuid': 153802947900
+ })
Modified: trunk/Tools/resultsdbpy/resultsdbpy/model/model.py (252271 => 252272)
--- trunk/Tools/resultsdbpy/resultsdbpy/model/model.py 2019-11-08 23:40:31 UTC (rev 252271)
+++ trunk/Tools/resultsdbpy/resultsdbpy/model/model.py 2019-11-09 00:03:56 UTC (rev 252272)
@@ -30,6 +30,7 @@
from resultsdbpy.model.upload_context import UploadContext
from resultsdbpy.model.suite_context import SuiteContext
from resultsdbpy.model.test_context import TestContext
+from resultsdbpy.model.failure_context import FailureContext
class Model(object):
@@ -66,6 +67,11 @@
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
+ self.failure_context = FailureContext(
+ configuration_context=self.configuration_context,
+ commit_context=self.commit_context,
+ ttl_seconds=self.default_ttl_seconds,
+ )
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
@@ -72,7 +78,7 @@
ttl_seconds=self.default_ttl_seconds,
)
- for context in [self.suite_context, self.test_context, self.ci_context]:
+ for context in [self.suite_context, self.test_context, self.ci_context, self.failure_context]:
self.upload_context.register_upload_callback(context.name, context.register)
self.archive_context = ArchiveContext(
Modified: trunk/Tools/resultsdbpy/resultsdbpy/view/templates/documentation.html (252271 => 252272)
--- trunk/Tools/resultsdbpy/resultsdbpy/view/templates/documentation.html 2019-11-08 23:40:31 UTC (rev 252271)
+++ trunk/Tools/resultsdbpy/resultsdbpy/view/templates/documentation.html 2019-11-09 00:03:56 UTC (rev 252272)
@@ -314,7 +314,7 @@
documentEndpoint(
'/api/results/<suite>/<test>',
['GET'],
- ['Branch', 'Branch', 'Configuration', 'Limit', 'Repository', 'Time', 'UUID'],
+ ['Branch', 'Configuration', 'Limit', 'Repository', 'Time', 'UUID'],
[
`Access results for a specific test on a specific commit with a specific configuration. This endpoint only returns results for a single test. Each result is stored in dictionary formatted like this:`,
codeBlock('{\n' +
@@ -343,7 +343,43 @@
`where <configuration-object-a> and <configuration-object-b> are both ${localLink(['Query Parameters', 'Configuration'], 'configuration objects')}' and <run-a1>, <run-a2>, <run-b1> and <run-b2> are all the afformentioned single test result dictionary.`,
],
),
- ], 'CI Links': [
+ ], 'Failure Analysis': [
+ `Results databases provide a few APIs to assist in the investigation of test failures. These analysis endpoints aggregate data from multiple test runs for consumption by both humans and automated systems.`,
+ documentEndpoint(
+ '/api/failures/<suite>',
+ ['GET'],
+ ['Aggregation', 'Branch', 'Configuration', 'Limit', 'Repository', 'Time', 'UUID'],
+ [
+ `Returns a list of tests which failed during test runs matching the specified criteria. When collapsed, these results will be a sorted list looking like this:`,
+ codeBlock('[\n' +
+ ' "suite.sub-1.test-1",\n' +
+ ' "suite.sub-1.test-2",\n' +
+ ' "suite.sub-2.test-1"\n' +
+ ']'),
+ `When uncollapsed, these results will be separated by the upload that generated them. These results are laid out much like the ${localLink(['API', '/api/results/<suite>'], '/api/results/<suite>')} and ${localLink(['API', '/api/results/<suite>/<test>'], '/api/results/<suite>/<test>')} endpoints.`,
+ codeBlock('[\n' +
+ ' {\n' +
+ ' "configuration": <configuration-object-a>,\n' +
+ ' "results": {\n' +
+ ' "start_time": <UTC time test run started>,\n' +
+ ' "uuid": <UUID for test run>,\n' +
+ ' "suite.sub-1.test-1": "FAIL",\n' +
+ ' "suite.sub-1.test-2": "FAIL",\n' +
+ ' }\n' +
+ ' }, {\n' +
+ ' "configuration": <configuration-object-b>,\n' +
+ ' "results": {\n' +
+ ' "start_time": <UTC time test run started>,\n' +
+ ' "uuid": <UUID for test run>,\n' +
+ ' "suite.sub-1.test-1": "FAIL",\n' +
+ ' "suite.sub-2.test-1": "CRASH",\n' +
+ ' }\n' +
+ ' }\n' +
+ ']'),
+ `where <configuration-object-a> and <configuration-object-b> are both ${localLink(['Query Parameters', 'Configuration'], 'configuration objects')}.`,
+ ],
+ ),
+ ],'CI Links': [
`Results database instances are usually storing test results from some sort of continuous integration system. While the results database doesn't assume any particular continuous integration system, it does make some basic assumptions. The results database assumes that every upload has a URL associated with it and that every upload was run on a specific machine. Additionally, the results database assumes that for a given configuration, there is a corresponding 'queue' that all continuous integration runs with that specific configuration are associated with`,
`If these assumptions aren't true for a particular instance of the results database, or if URLs are not included in upload data, the continuous integration endpoints may be dead links. That should not effect the operation or usage of the results database.`,
documentEndpoint(
@@ -418,6 +454,14 @@
),
],
}, 'Query Parameters': {
+ 'Aggregation': [
+ `Some endpoints in the results database aggregate data from multiple test runs. Such endpoints accept query parameters that control how this aggregation is preformed. The first of these is the collapsed parameter, which is set to True by default in aggregation endpoints:`,
+ codeBlock('collapsed=False'),
+ `The collapsed parameter indicates that a single result will be returned for all uploads which match the specified criteria. If false, aggregation endpoints will return the results which would have otherwise been aggregated.`,
+ `Because the results database distinguishes between expected and unexpected failures, endpoints performing aggregation will often filter out expected failures, and flag unexpected passes. To modify the behavior of these algorithms, these endpoints will support the unexpected flag:`,
+ codeBlock('unexpected=False'),
+ `By default, this flag is 'True' and endpoints will ignore tests which matched their expected behavior. If set to 'False', endpoints will return results for all failing tests, regardless of what their expectation is.`,
+ ],
'Branch': [
`Most data in the results database is partitioned by branch, and it is generally expected that results on seperate branches are independent of one another. By default, endpoints that support the branch query will assume that the branch is master for git repositories and trunk for SVN repositories if no value is specified. If multiple values for branch are specified, only the first will be respected. A request which intended to search for results on only results on the safari-607 branch would use a query like this:`,
codeBlock('branch=safari-607'),
