Title: [262876] trunk/Tools
Revision
262876
Author
[email protected]
Date
2020-06-10 17:07:51 -0700 (Wed, 10 Jun 2020)

Log Message

results.webkit.org: Health check should access redis and Cassandra
https://bugs.webkit.org/show_bug.cgi?id=212899
<rdar://problem/64118077>

Reviewed by Stephanie Lewis.

* resultsdbpy/resultsdbpy/example/main.py:
(health): Check the health status of the model.
* resultsdbpy/resultsdbpy/model/model.py:
(Model.HealthTable):
(Model.__init__): Construct a Cassandra table used to check the health of our Cassandra connection.
(Model.healthy): Make both a read and write connection to our two database dependencies so that if
our connection to those databases fails, an instance will be forced to restart.
* resultsdbpy/resultsdbpy/model/model_unittest.py: Added.
(ModelTest):
(ModelTest.init_database):
(ModelTest.test_health):

Modified Paths

Added Paths

Diff

Modified: trunk/Tools/ChangeLog (262875 => 262876)


--- trunk/Tools/ChangeLog	2020-06-11 00:00:11 UTC (rev 262875)
+++ trunk/Tools/ChangeLog	2020-06-11 00:07:51 UTC (rev 262876)
@@ -1,3 +1,23 @@
+2020-06-10  Jonathan Bedard  <[email protected]>
+
+        results.webkit.org: Health check should access redis and Cassandra
+        https://bugs.webkit.org/show_bug.cgi?id=212899
+        <rdar://problem/64118077>
+
+        Reviewed by Stephanie Lewis.
+
+        * resultsdbpy/resultsdbpy/example/main.py:
+        (health): Check the health status of the model.
+        * resultsdbpy/resultsdbpy/model/model.py:
+        (Model.HealthTable):
+        (Model.__init__): Construct a Cassandra table used to check the health of our Cassandra connection.
+        (Model.healthy): Make both a read and write connection to our two database dependencies so that if
+        our connection to those databases fails, an instance will be forced to restart.
+        * resultsdbpy/resultsdbpy/model/model_unittest.py: Added.
+        (ModelTest):
+        (ModelTest.init_database):
+        (ModelTest.test_health):
+
 2020-06-10  Keith Miller  <[email protected]>
 
         run-jsc-stress-tests should skip tests marked as slow in quick mode.

Modified: trunk/Tools/resultsdbpy/resultsdbpy/example/main.py (262875 => 262876)


--- trunk/Tools/resultsdbpy/resultsdbpy/example/main.py	2020-06-11 00:00:11 UTC (rev 262875)
+++ trunk/Tools/resultsdbpy/resultsdbpy/example/main.py	2020-06-11 00:07:51 UTC (rev 262876)
@@ -45,6 +45,8 @@
 
 @app.route('/__health', methods=('GET',))
 def health():
+    if not model.healthy(writable=True):
+        abort(503, description='Health check failed, invalid database connections')
     return 'ok'
 
 

Modified: trunk/Tools/resultsdbpy/resultsdbpy/model/model.py (262875 => 262876)


--- trunk/Tools/resultsdbpy/resultsdbpy/model/model.py	2020-06-11 00:00:11 UTC (rev 262875)
+++ trunk/Tools/resultsdbpy/resultsdbpy/model/model.py	2020-06-11 00:07:51 UTC (rev 262876)
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 Apple Inc. All rights reserved.
+# Copyright (C) 2019, 2020 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -23,6 +23,8 @@
 import traceback
 import sys
 
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model as CassandraModel
 from resultsdbpy.model.archive_context import ArchiveContext
 from resultsdbpy.model.ci_context import CIContext
 from resultsdbpy.model.commit_context import CommitContext
@@ -38,6 +40,11 @@
     TTL_WEEK = 7 * TTL_DAY
     TTL_YEAR = 365 * TTL_DAY
 
+    class HealthTable(CassandraModel):
+        __table_name__ = 'health'
+        key = columns.Text(partition_key=True, required=True)
+        value = columns.Text(required=True)
+
     def __init__(self, redis, cassandra, repositories=[], default_ttl_seconds=TTL_YEAR * 5, archive_ttl_seconds=TTL_WEEK * 8, async_processing=False):
         if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
             raise ValueError('TTL must be at least 4 weeks')
@@ -49,6 +56,10 @@
 
         self.redis = redis
         self.cassandra = cassandra
+
+        with self.cassandra:
+            self.cassandra.create_table(self.HealthTable)
+
         self.commit_context = CommitContext(redis, cassandra)
         for repository in repositories:
             self.commit_context.register_repository(repository)
@@ -90,6 +101,28 @@
             ttl_seconds=self.archive_ttl_seconds,
         )
 
+    def healthy(self, writable=True):
+        if writable:
+            self.redis.set('health-check', b'healthy', ex=5 * 60)
+        value = self.redis.get('health-check')
+        if value is not None and value != b'healthy':
+            return False
+
+        with self.cassandra:
+            if writable:
+                self.cassandra.insert_row(
+                    self.HealthTable.__table_name__,
+                    key='health-check', value='healthy',
+                    ttl=5 * 60,
+                )
+            values = [element.value for element in self.cassandra.select_from_table(
+                self.HealthTable.__table_name__, limit=100, key='health-check',
+            )]
+            if values and values != ['healthy']:
+                return False
+
+        return True
+
     def do_work(self):
         if not self._async_processing:
             raise RuntimeError('No work to be done, asynchronous processing disabled')

Copied: trunk/Tools/resultsdbpy/resultsdbpy/model/model_unittest.py (from rev 262875, trunk/Tools/resultsdbpy/resultsdbpy/example/main.py) (0 => 262876)


--- trunk/Tools/resultsdbpy/resultsdbpy/model/model_unittest.py	                        (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/model/model_unittest.py	2020-06-11 00:07:51 UTC (rev 262876)
@@ -0,0 +1,45 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from fakeredis import FakeStrictRedis
+from redis import StrictRedis
+from resultsdbpy.model.cassandra_context import CassandraContext
+from resultsdbpy.model.mock_cassandra_context import MockCassandraContext
+from resultsdbpy.model.mock_model_factory import MockModelFactory
+from resultsdbpy.model.wait_for_docker_test_case import WaitForDockerTestCase
+
+
+class ModelTest(WaitForDockerTestCase):
+    KEYSPACE = 'model_test_keyspace'
+
+    def init_database(self, redis=StrictRedis, cassandra=CassandraContext, async_processing=False):
+        cassandra.drop_keyspace(keyspace=self.KEYSPACE)
+        self.model = MockModelFactory.create(
+            redis=redis(), cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True),
+            async_processing=async_processing,
+        )
+
+    @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+    def test_health(self, redis=StrictRedis, cassandra=CassandraContext):
+        self.init_database(redis=redis, cassandra=cassandra)
+        self.assertTrue(self.model.healthy())
+        self.assertTrue(self.model.healthy(writable=False))
_______________________________________________
webkit-changes mailing list
[email protected]
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to