Diff
Modified: trunk/Tools/ChangeLog (258354 => 258355)
--- trunk/Tools/ChangeLog 2020-03-12 21:01:46 UTC (rev 258354)
+++ trunk/Tools/ChangeLog 2020-03-12 21:06:33 UTC (rev 258355)
@@ -1,3 +1,24 @@
+2020-03-12 Jonathan Bedard <[email protected]>
+
+ resultsdpy: Add script to run local instance
+ https://bugs.webkit.org/show_bug.cgi?id=208746
+
+ Rubber-stamped by Aakash Jain.
+
+ * resultsdbpy/resultsdbpy/example: Added.
+ * resultsdbpy/resultsdbpy/example/__init__.py: Added.
+ * resultsdbpy/resultsdbpy/example/environment.py: Added.
+ (Environment): Organize environment variables used by results database.
+ (ModelFromEnvironment): Construct the database model from the environment.
+ (main): Create and drop tables, if required.
+ * resultsdbpy/resultsdbpy/example/main.py: Added.
+ (health): Web-server health check.
+ (handle_errors):
+ (main): Run results database web-server.
+ * resultsdbpy/resultsdbpy/example/worker.py: Added.
+ (main): Asynchronously process uploaded results.
+ * resultsdbpy/resultsdbpy/run: Added.
+
2020-03-12 Wenson Hsieh <[email protected]>
run-webkit-tests --use-gpu-process should enable GPU Process for canvas
Added: trunk/Tools/resultsdbpy/resultsdbpy/example/__init__.py (0 => 258355)
--- trunk/Tools/resultsdbpy/resultsdbpy/example/__init__.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/example/__init__.py 2020-03-12 21:06:33 UTC (rev 258355)
@@ -0,0 +1,21 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Added: trunk/Tools/resultsdbpy/resultsdbpy/example/environment.py (0 => 258355)
--- trunk/Tools/resultsdbpy/resultsdbpy/example/environment.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/example/environment.py 2020-03-12 21:06:33 UTC (rev 258355)
@@ -0,0 +1,159 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import sys
+
+from cassandra.auth import PlainTextAuthProvider
+from fakeredis import FakeStrictRedis
+from redis import StrictRedis
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from resultsdbpy.model.cassandra_context import CassandraContext
+from resultsdbpy.model.casserole import CasseroleNodes, CasseroleRedis
+from resultsdbpy.model.ci_context import BuildbotURLFactory, BuildbotEightURLFactory
+from resultsdbpy.model.mock_cassandra_context import MockCassandraContext
+from resultsdbpy.model.model import Model
+from resultsdbpy.model.repository import WebKitRepository
+from resultsdbpy.model.partitioned_redis import PartitionedRedis
+
+MOCK_CASSANDRA = 'mock'
+KEYSPACE = 'results_database_example'
+
+
+class Environment(object):
+ REDIS_PORT = 6379
+
+ def __init__(self, cqleng_allow_schema_management=False):
+ self.port = int(os.environ.get('PORT', 5000))
+
+ # The base environment provides defaults for database information, although that can be explicitly overridden
+ self.cassandra_server = os.environ.get('CASSANDRA_SERVER', 'localhost')
+ self.keyspace = os.environ.get('KEYSPACE', KEYSPACE)
+
+ self.redis_host = os.environ.get('REDIS_HOST', 'localhost')
+ self.redis_port = os.environ.get('REDIS_PORT', self.REDIS_PORT)
+
+ self.drop_keyspace = cqleng_allow_schema_management and bool(int(os.environ.get('DROP_KEYSPACE', 0)))
+ self.create_keyspace = cqleng_allow_schema_management and bool(int(os.environ.get('CREATE_KEYSPACE', 1)))
+ self.cqleng_allow_schema_management = cqleng_allow_schema_management
+
+ os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1' if self.cqleng_allow_schema_management else '0'
+
+ for key, value in self.__dict__.items():
+ if value is None:
+ raise EnvironmentError(f'{key.upper()} is not defined in the environment')
+
+ self.redis_url = os.environ.get('REDIS_URL', None)
+ self.redis_password = os.environ.get('REDIS_PASSWORD', None)
+
+ self.cassandra_username = None
+ self.cassandra_password = None
+
+ if not self.cqleng_allow_schema_management:
+ self.cassandra_username = os.environ.get('CASSANDRA_USER')
+ self.cassandra_password = os.environ.get('CASSANDRA_USER_PASSWORD')
+
+ if not self.cassandra_username or not self.cassandra_password:
+ self.cassandra_username = os.environ.get('CASSANDRA_ADMIN')
+ self.cassandra_password = os.environ.get('CASSANDRA_ADMIN_PASSWORD')
+
+ @staticmethod
+ def obfuscate_password(password):
+ if password:
+ return '*' * len(password)
+ return '-'
+
+ def __str__(self):
+ result = ''
+ order = [
+ 'port',
+ 'cassandra_server', 'cqleng_allow_schema_management',
+ 'keyspace', 'drop_keyspace', 'create_keyspace',
+ 'cassandra_username', 'cassandra_password',
+ 'redis_url',
+ 'redis_host', 'redis_port', 'redis_password',
+ ]
+ for key in self.__dict__.keys():
+ if key not in order:
+ order.append(key)
+ for key in order:
+ value = self.__dict__.get(key)
+ if value and ('password' in key or 'token' in key or key.endswith('key')):
+ value = self.obfuscate_password(value)
+ result += f" {key.upper()}: {value}\n"
+ return result[:-1]
+
+
+class ModelFromEnvironment(Model):
+ TTL_SECONDS = Model.TTL_YEAR * 5
+
+ def __init__(self, environment):
+ nodes = ['localhost']
+ cassandra_class = CassandraContext
+ if environment.cassandra_server == MOCK_CASSANDRA:
+ cassandra_class = MockCassandraContext
+
+ auth_provider = None
+ if environment.cassandra_username and environment.cassandra_password:
+ auth_provider = PlainTextAuthProvider(environment.cassandra_username, environment.cassandra_password)
+
+ if environment.drop_keyspace:
+ cassandra_class.drop_keyspace(nodes=nodes, keyspace=environment.keyspace, auth_provider=auth_provider)
+
+ cassandra = cassandra_class(
+ nodes=nodes,
+ keyspace=environment.keyspace,
+ auth_provider=auth_provider,
+ create_keyspace=environment.create_keyspace,
+ replication_map=None,
+ )
+ if environment.redis_host == 'mock':
+ redis = FakeStrictRedis()
+ elif environment.redis_url:
+ redis = StrictRedis.from_url(environment.redis_url)
+ else:
+ redis = StrictRedis(
+ port=environment.redis_port,
+ host=environment.redis_host,
+ password=environment.redis_password,
+ )
+ redis.ping() # Force a connection with the redis database
+
+ webkit = WebKitRepository()
+ super(ModelFromEnvironment, self).__init__(
+ redis=redis, cassandra=cassandra,
+ repositories=[webkit],
+ default_ttl_seconds=self.TTL_SECONDS,
+ async_processing=True,
+ )
+
+
+def main():
+ environment = Environment(cqleng_allow_schema_management=True)
+ print(f'Environment for setup:\n{environment}')
+ ModelFromEnvironment(environment)
+
+
+if __name__ == '__main__':
+ main()
Added: trunk/Tools/resultsdbpy/resultsdbpy/example/main.py (0 => 258355)
--- trunk/Tools/resultsdbpy/resultsdbpy/example/main.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/example/main.py 2020-03-12 21:06:33 UTC (rev 258355)
@@ -0,0 +1,65 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import hashlib
+import json
+
+from example.environment import Environment, ModelFromEnvironment
+from flask import abort, Flask, request
+from resultsdbpy.controller.api_routes import APIRoutes
+from resultsdbpy.view.view_routes import ViewRoutes
+
+
+environment = Environment()
+print(f'Environment for web-app:\n{environment}')
+
+model = ModelFromEnvironment(environment)
+app = Flask(__name__)
+
+
+api_routes = APIRoutes(model=model, import_name=__name__)
+view_routes = ViewRoutes(
+ title='Example Results Database',
+ model=model, controller=api_routes, import_name=__name__,
+)
+
+
[email protected]('/__health', methods=('GET',))
+def health():
+ return 'ok'
+
+
[email protected](401)
[email protected](404)
[email protected](405)
+def handle_errors(error):
+ if request.path.startswith('/api/'):
+ return api_routes.error_response(error)
+ return view_routes.error(error=error)
+
+
+app.register_blueprint(api_routes)
+app.register_blueprint(view_routes)
+
+
+def main():
+ app.run(host='0.0.0.0', port=environment.port)
Added: trunk/Tools/resultsdbpy/resultsdbpy/example/worker.py (0 => 258355)
--- trunk/Tools/resultsdbpy/resultsdbpy/example/worker.py (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/example/worker.py 2020-03-12 21:06:33 UTC (rev 258355)
@@ -0,0 +1,42 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import random
+import time
+from example.environment import Environment, ModelFromEnvironment
+
+
+def main():
+ environment = Environment()
+ print(f'Environment for worker:\n{environment}')
+
+ model = ModelFromEnvironment(environment)
+
+ random.seed()
+ while True:
+ model.do_work()
+ time.sleep(random.randint(15, 45)) # Stagger workers.
+
+
+if __name__ == '__main__':
+ main()
Added: trunk/Tools/resultsdbpy/resultsdbpy/run (0 => 258355)
--- trunk/Tools/resultsdbpy/resultsdbpy/run (rev 0)
+++ trunk/Tools/resultsdbpy/resultsdbpy/run 2020-03-12 21:06:33 UTC (rev 258355)
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+import threading
+
+sys.dont_write_bytecode = True
+
+from model.docker import Docker
+from multiprocessing import Process
+from redis import StrictRedis
+
+
+class DockerIfNeededContext(object):
+
+ def __init__(self, args):
+ self.docker = Docker.instance() if 'localhost' in [args.cassandra_server, args.redis_server] else None
+
+ def __enter__(self):
+ if self.docker:
+ self.docker.__enter__()
+
+ def __exit__(self, *args, **kwargs):
+ if self.docker:
+ self.docker.__exit__(*args, **kwargs)
+
+
+def has_gunicorn():
+ return not subprocess.run(['python3', '/usr/local/bin/gunicorns', '--version'], capture_output=True).returncode
+
+
+def parse_arguments():
+ parser = argparse.ArgumentParser(description='Run a simple instance of the results database for local development')
+ parser.add_argument(
+ '--local-cassandra', help='Use a local Cassandra instance',
+ dest='cassandra_server', action='', const='localhost',
+ )
+ parser.add_argument(
+ '--mock-cassandra', help='Use a mock Cassandra instance',
+ dest='cassandra_server', action='', const='mock',
+ )
+ parser.set_defaults(cassandra_server='localhost' if Docker.installed() else 'mock')
+
+ parser.add_argument(
+ '--local-redis', help='Use a local Redis instance',
+ dest='redis_server', action='', const='localhost',
+ )
+ parser.add_argument(
+ '--mock-redis', help='Use a mock Redis instance',
+ dest='redis_server', action='', const='mock',
+ )
+ parser.set_defaults(redis_server='localhost' if Docker.installed() else 'mock')
+
+ parser.add_argument(
+ '--drop-keyspace', help='Drop the existing keyspace before running the program',
+ dest='drop_keyspace', action='',
+ )
+ parser.add_argument(
+ '--no-drop-keyspace', help='Do NOT drop the existing keyspace before running the program',
+ dest='drop_keyspace', action='',
+ )
+ parser.set_defaults(drop_keyspace=False)
+
+ parser.add_argument(
+ '--clear-redis', help='Clear redis before running the application',
+ dest='clear_redis', action='',
+ )
+ parser.add_argument(
+ '--no-clear-redis',
+ help='Do not clear redis before running the application',
+ dest='clear_redis', action='',
+ )
+ parser.set_defaults(clear_redis=True)
+
+ parser.add_argument(
+ '--gunicorn', help='Use the gunicorn webserver',
+ dest='gunicorn', action='',
+ )
+ parser.add_argument(
+ '--no-gunicorn',
+ help='Do not use the gunicorn webserver',
+ dest='gunicorn', action='',
+ )
+ parser.set_defaults(gunicorn=has_gunicorn())
+
+ return parser.parse_args()
+
+
+def do_work():
+ from example.worker import main
+ main()
+
+
+def web_app():
+ from example.main import main
+ return main()
+
+
+def gunicorn():
+ if not has_gunicorn():
+ print('gunicorn is not installed, please run `pip3 install gunicorn`')
+ return 1
+
+ # Slight delay so that the worker is started before the webserver
+ time.sleep(2)
+ return subprocess.run([
+ 'python3', '/usr/local/bin/gunicorn',
+ 'example.main:app',
+ '--log-file=-', '--reload',
+ ]).returncode
+
+
+def main():
+ args = parse_arguments()
+ with DockerIfNeededContext(args):
+ os.environ['CASSANDRA_SERVER'] = args.cassandra_server
+ os.environ['REDIS_HOST'] = args.redis_server
+ os.environ['DROP_KEYSPACE'] = '1' if args.drop_keyspace else '0'
+ os.environ['CREATE_KEYSPACE'] = '1' # Unconditionally create keyspace for testing
+
+ if args.clear_redis and args.redis_server != 'mock':
+ StrictRedis(host=args.redis_server).flushdb()
+
+ if args.gunicorn:
+ if 'mock' in [args.redis_server, args.cassandra_server]:
+ raise RuntimeError('Cannot run with gunicorn and mock databases, asynchronous workers rely on the real databases')
+
+ from example.environment import main as environment_main
+ environment_main()
+
+ if not args.gunicorn:
+ worker = threading.Thread(target=do_work)
+ worker.daemon = True
+ worker.start()
+
+ return web_app()
+
+ worker = Process(target=do_work)
+ worker.start()
+
+ try:
+ return gunicorn()
+ finally:
+ worker.terminate()
+ worker.join()
+
+if __name__ == '__main__':
+ sys.exit(main())
Property changes on: trunk/Tools/resultsdbpy/resultsdbpy/run
___________________________________________________________________
Added: svn:executable
+*
\ No newline at end of property
