- Revision
- 262876
- Author
- [email protected]
- Date
- 2020-06-10 17:07:51 -0700 (Wed, 10 Jun 2020)
Log Message
results.webkit.org: Health check should access redis and Cassandra
https://bugs.webkit.org/show_bug.cgi?id=212899
<rdar://problem/64118077>
Reviewed by Stephanie Lewis.
* resultsdbpy/resultsdbpy/example/main.py:
(health): Check the health status of the model.
* resultsdbpy/resultsdbpy/model/model.py:
(Model.HealthTable):
(Model.__init__): Construct a Cassandra table used to check the health of our Cassandra connection.
(Model.healthy): Make both a read and write connection to our two database dependencies so that if
our connection to those databases fails, an instance will be forced to restart.
* resultsdbpy/resultsdbpy/model/model_unittest.py: Added.
(ModelTest):
(ModelTest.init_database):
(ModelTest.test_health):
Modified Paths
Added Paths
Diff
Modified: trunk/Tools/ChangeLog (262875 => 262876)
--- trunk/Tools/ChangeLog 2020-06-11 00:00:11 UTC (rev 262875)
+++ trunk/Tools/ChangeLog 2020-06-11 00:07:51 UTC (rev 262876)
@@ -1,3 +1,23 @@
+2020-06-10 Jonathan Bedard <[email protected]>
+
+ results.webkit.org: Health check should access redis and Cassandra
+ https://bugs.webkit.org/show_bug.cgi?id=212899
+ <rdar://problem/64118077>
+
+ Reviewed by Stephanie Lewis.
+
+ * resultsdbpy/resultsdbpy/example/main.py:
+ (health): Check the health status of the model.
+ * resultsdbpy/resultsdbpy/model/model.py:
+ (Model.HealthTable):
+ (Model.__init__): Construct a Cassandra table used to check the health of our Cassandra connection.
+ (Model.healthy): Make both a read and write connection to our two database dependencies so that if
+ our connection to those databases fails, an instance will be forced to restart.
+ * resultsdbpy/resultsdbpy/model/model_unittest.py: Added.
+ (ModelTest):
+ (ModelTest.init_database):
+ (ModelTest.test_health):
+
2020-06-10 Keith Miller <[email protected]>
run-jsc-stress-tests should skip tests marked as slow in quick mode.
Modified: trunk/Tools/resultsdbpy/resultsdbpy/example/main.py (262875 => 262876)
--- trunk/Tools/resultsdbpy/resultsdbpy/example/main.py 2020-06-11 00:00:11 UTC (rev 262875)
+++ trunk/Tools/resultsdbpy/resultsdbpy/example/main.py 2020-06-11 00:07:51 UTC (rev 262876)
@@ -45,6 +45,8 @@
@app.route('/__health', methods=('GET',))
def health():
+ if not model.healthy(writable=True):
+ abort(503, description='Health check failed, invalid database connections')
return 'ok'
Modified: trunk/Tools/resultsdbpy/resultsdbpy/model/model.py (262875 => 262876)
--- trunk/Tools/resultsdbpy/resultsdbpy/model/model.py 2020-06-11 00:00:11 UTC (rev 262875)
+++ trunk/Tools/resultsdbpy/resultsdbpy/model/model.py 2020-06-11 00:07:51 UTC (rev 262876)
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 Apple Inc. All rights reserved.
+# Copyright (C) 2019, 2020 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -23,6 +23,8 @@
import traceback
import sys
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model as CassandraModel
from resultsdbpy.model.archive_context import ArchiveContext
from resultsdbpy.model.ci_context import CIContext
from resultsdbpy.model.commit_context import CommitContext
@@ -38,6 +40,11 @@
TTL_WEEK = 7 * TTL_DAY
TTL_YEAR = 365 * TTL_DAY
+ class HealthTable(CassandraModel):
+ __table_name__ = 'health'
+ key = columns.Text(partition_key=True, required=True)
+ value = columns.Text(required=True)
+
def __init__(self, redis, cassandra, repositories=[], default_ttl_seconds=TTL_YEAR * 5, archive_ttl_seconds=TTL_WEEK * 8, async_processing=False):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
@@ -49,6 +56,10 @@
self.redis = redis
self.cassandra = cassandra
+
+ with self.cassandra:
+ self.cassandra.create_table(self.HealthTable)
+
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
@@ -90,6 +101,28 @@
ttl_seconds=self.archive_ttl_seconds,
)
+ def healthy(self, writable=True):
+ if writable:
+ self.redis.set('health-check', b'healthy', ex=5 * 60)
+ value = self.redis.get('health-check')
+ if value is not None and value != b'healthy':
+ return False
+
+ with self.cassandra:
+ if writable:
+ self.cassandra.insert_row(
+ self.HealthTable.__table_name__,
+ key='health-check', value='healthy',
+ ttl=5 * 60,
+ )
+ values = [element.value for element in self.cassandra.select_from_table(
+ self.HealthTable.__table_name__, limit=100, key='health-check',
+ )]
+ if values and values != ['healthy']:
+ return False
+
+ return True
+
def do_work(self):
if not self._async_processing:
raise RuntimeError('No work to be done, asynchronous processing disabled')
Copied: trunk/Tools/resultsdbpy/resultsdbpy/model/model_unittest.py (from rev 262875, trunk/Tools/resultsdbpy/resultsdbpy/example/main.py) (0 => 262876)
--- trunk/Tools/resultsdbpy/resultsdbpy/model/model_unittest.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/model/model_unittest.py 2020-06-11 00:07:51 UTC (rev 262876)
@@ -0,0 +1,45 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from fakeredis import FakeStrictRedis
+from redis import StrictRedis
+from resultsdbpy.model.cassandra_context import CassandraContext
+from resultsdbpy.model.mock_cassandra_context import MockCassandraContext
+from resultsdbpy.model.mock_model_factory import MockModelFactory
+from resultsdbpy.model.wait_for_docker_test_case import WaitForDockerTestCase
+
+
+class ModelTest(WaitForDockerTestCase):
+ KEYSPACE = 'model_test_keyspace'
+
+ def init_database(self, redis=StrictRedis, cassandra=CassandraContext, async_processing=False):
+ cassandra.drop_keyspace(keyspace=self.KEYSPACE)
+ self.model = MockModelFactory.create(
+ redis=redis(), cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True),
+ async_processing=async_processing,
+ )
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_health(self, redis=StrictRedis, cassandra=CassandraContext):
+ self.init_database(redis=redis, cassandra=cassandra)
+ self.assertTrue(self.model.healthy())
+ self.assertTrue(self.model.healthy(writable=False))