Diff
Deleted: trunk/Tools/CISupport/build-webkit-org/run-buildbot-test.py (276867 => 276868)
--- trunk/Tools/CISupport/build-webkit-org/run-buildbot-test.py 2021-05-01 01:19:18 UTC (rev 276867)
+++ trunk/Tools/CISupport/build-webkit-org/run-buildbot-test.py 2021-05-01 01:20:56 UTC (rev 276868)
@@ -1,347 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2017 Igalia S.L.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-import sys
-import signal
-import os
-import argparse
-import subprocess
-import tempfile
-import shutil
-import socket
-import json
-import traceback
-import multiprocessing
-from time import sleep
-
-test_buildbot_master_tac = """
-import os
-from twisted.application import service
-try:
- from buildbot.master.bot import BuildMaster
-except:
- from buildbot.master import BuildMaster
-
-basedir = os.path.dirname(os.path.realpath(__file__))
-configfile = r'master.cfg'
-
-application = service.Application('buildmaster')
-BuildMaster(basedir, configfile).setServiceParent(application)
-"""
-
-worker_buildbot_master_tac = """
-import os
-from twisted.application import service
-from buildslave.bot import BuildSlave
-
-basedir = os.path.dirname(os.path.realpath(__file__))
-buildmaster_host = 'localhost'
-port = 17000
-slavename = '{}'
-passwd = '1234'
-keepalive = 600
-usepty = 1
-
-application = service.Application('buildslave')
-BuildSlave(buildmaster_host, port, slavename, passwd, basedir, keepalive, usepty).setServiceParent(application)
-"""
-
-
-def check_tcp_port_open(address, port):
- s = socket.socket()
- try:
- s.connect((address, port))
- return True
- except:
- return False
-
-
-def upgrade_db_needed(log):
- try:
- with open(log) as f:
- for l in f:
- if 'upgrade the database' in l:
- return True
- except:
- return False
- return False
-
-
-def create_tempdir(tmpdir=None):
- if tmpdir is not None:
- if not os.path.isdir(tmpdir):
- raise ValueError('{} is not a directory'.format(tmpdir))
- return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(tmpdir), 'tmp'))
- return tempfile.mkdtemp()
-
-
-def print_if_error_stdout_stderr(cmd, retcode, stdout=None, stderr=None, extramsg=None):
- if retcode != 0:
- if type(cmd) == type([]):
- cmd = ' '.join(cmd)
- print('WARNING: "{cmd}" returned {retcode} status code'.format(cmd=cmd, retcode=retcode))
- if stdout is not None:
- print(stdout)
- if stderr is not None:
- print(stderr)
- if extramsg is not None:
- print(extramsg)
-
-
-def setup_master_workdir(configdir, base_workdir):
- master_workdir = os.path.join(base_workdir, 'master')
- print('Copying files from {} to {} ...'.format(configdir, master_workdir))
- shutil.copytree(configdir, master_workdir)
- print('Generating buildbot files at {} ...'.format(master_workdir))
- with open(os.path.join(master_workdir, 'buildbot.tac'), 'w') as f:
- f.write(test_buildbot_master_tac)
- mkpwd_cmd = ['./make_passwords_json.py']
- mkpwd_process = subprocess.Popen(mkpwd_cmd, cwd=master_workdir,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- stdout, stderr = mkpwd_process.communicate()
- print_if_error_stdout_stderr(mkpwd_cmd, mkpwd_process.returncode, stdout, stderr)
- return master_workdir
-
-
-def wait_for_master_ready(master_workdir):
- master_ready_check_counter = 0
- while True:
- if os.path.isfile(os.path.join(master_workdir, '.master-is-ready')):
- return
- if master_ready_check_counter > 60:
- raise RuntimeError('ERROR: Aborting after waiting 60 seconds for the master to start.')
- sleep(1)
- master_ready_check_counter += 1
-
-
-def start_master(master_workdir):
- # This is started via multiprocessing. We set a new process group here
- # to be able to reliably kill this subprocess and all of its child on clean.
- os.setsid()
- buildmasterlog = os.path.join(master_workdir, 'buildmaster.log')
- dbupgraded = False
- retry = True
- if check_tcp_port_open('localhost', 8710):
- print('ERROR: There is some process already listening in port 8170')
- return 1
- while retry:
- retry = False
- print('Starting the twistd process ...')
- twistd_cmd = ['twistd', '-l', buildmasterlog, '-noy', 'buildbot.tac']
- twistd_process = subprocess.Popen(twistd_cmd, cwd=master_workdir,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- while twistd_process.poll() is None:
- if check_tcp_port_open('localhost', 8710):
- print('Test buildmaster ready!.\n\n'
- + ' - See buildmaster log:\n'
- + ' tail -f {}\n'.format(buildmasterlog)
- + ' - Open a browser to:\n'
- + ' http://localhost:8710\n'
- + ' - Credentials for triggering manual builds:\n'
- + ' login: commit...@webkit.org\n'
- + ' password: committerpassword\n')
- with open(os.path.join(master_workdir, '.master-is-ready'), 'w') as f:
- f.write('ready')
- twistd_process.wait()
- return 0
- sleep(1)
- stdout, stderr = twistd_process.communicate()
- if twistd_process.returncode == 0 and upgrade_db_needed(buildmasterlog) and not dbupgraded:
- retry = True
- dbupgraded = True
- print('Upgrading the database ...')
- upgrade_cmd = ['buildbot', 'upgrade-master', master_workdir]
- upgrade_process = subprocess.Popen(upgrade_cmd, cwd=master_workdir,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- stdout, stderr = upgrade_process.communicate()
- print_if_error_stdout_stderr(upgrade_cmd, upgrade_process.returncode, stdout, stderr)
- else:
- print_if_error_stdout_stderr(twistd_cmd, twistd_process.returncode, stdout, stderr,
- 'Check the log at {}'.format(buildmasterlog))
- return 0
-
-
-def get_list_workers(master_workdir):
- password_list = os.path.join(master_workdir, 'passwords.json')
- with open(password_list) as f:
- passwords = json.load(f)
- list_workers = []
- for worker in passwords:
- list_workers.append(str(worker))
- return list_workers
-
-
-def start_worker(base_workdir, worker):
- # This is started via multiprocessing. We set a new process group here
- # to be able to reliably kill this subprocess and all of its child on clean.
- os.setsid()
- worker_workdir = os.path.join(base_workdir, worker)
- os.mkdir(worker_workdir)
- with open(os.path.join(worker_workdir, 'buildbot.tac'), 'w') as f:
- f.write(worker_buildbot_master_tac.format(worker))
- twistd_cmd = ['twistd', '-l', 'worker.log', '-noy', 'buildbot.tac']
- twistd_worker_process = subprocess.Popen(twistd_cmd, cwd=worker_workdir,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- try:
- stdout, stderr = twistd_worker_process.communicate()
- except:
- twistd_worker_process.kill()
- return
- print_if_error_stdout_stderr(twistd_cmd, twistd_worker_process.returncode, stdout, stderr,
- 'Check the log at {}'.format(os.path.join(worker_workdir, 'worker.log')))
-
-
-def clean(temp_dir):
- if os.path.isdir(temp_dir):
- print('\n\nCleaning {} ... \n'.format(temp_dir))
- # shutil.rmtree can fail if we hold an open file descriptor on temp_dir
- # (which is very likely when cleaning) or if temp_dir is a NFS mount.
- # Use rm instead that always works.
- rm = subprocess.Popen(['rm', '-fr', temp_dir])
- rm.wait()
-
-
-def cmd_exists(cmd):
- return any(os.access(os.path.join(path, cmd), os.X_OK)
- for path in os.environ['PATH'].split(os.pathsep))
-
-
-def check_buildbot_installed():
- if cmd_exists('twistd') and cmd_exists('buildbot'):
- return
- raise RuntimeError('Buildbot is not installed.')
-
-
-def setup_virtualenv(base_workdir_temp):
- if cmd_exists('virtualenv'):
- print('Setting up virtualenv at {} ... '.format(base_workdir_temp))
- virtualenv_cmd = ['virtualenv', '-p', 'python2', 'venv']
- virtualenv_process = subprocess.Popen(virtualenv_cmd, cwd=base_workdir_temp,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- stdout, stderr = virtualenv_process.communicate()
- print_if_error_stdout_stderr(virtualenv_cmd, virtualenv_process.returncode, stdout, stderr)
- virtualenv_bindir = os.path.join(base_workdir_temp, 'venv', 'bin')
- virtualenv_pip = os.path.join(virtualenv_bindir, 'pip')
- if not os.access(virtualenv_pip, os.X_OK):
- print('Something went wrong setting up virtualenv'
- 'Trying to continue using the system version of buildbot')
- return
- print('Setting up buildbot dependencies on the virtualenv ... ')
- # The idea is to install the very same version of buildbot and its
- # dependencies than the ones used for running https://build.webkit.org/about
- pip_cmd = [virtualenv_pip, 'install',
- 'buildbot==0.8.6p1',
- 'buildbot-slave==0.8.6p1',
- 'twisted==12.1.0',
- 'jinja2==2.6',
- 'sqlalchemy==0.7.8',
- 'sqlalchemy-migrate==0.12.0']
- pip_process = subprocess.Popen(pip_cmd, cwd=base_workdir_temp,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- stdout, stderr = pip_process.communicate()
- print_if_error_stdout_stderr(pip_cmd, pip_process.returncode, stdout, stderr)
- os.environ['PATH'] = virtualenv_bindir + ':' + os.environ['PATH']
- return
- print('WARNING: virtualenv not installed. '
- 'Trying to continue using the system version of buildbot')
-
-
-def configdir_is_valid(configdir):
- return(os.path.isdir(configdir) and
- os.path.isfile(os.path.join(configdir, 'config.json')) and
- os.path.isfile(os.path.join(configdir, 'master.cfg')) and
- os.access(os.path.join(configdir, 'make_passwords_json.py'), os.X_OK))
-
-
-def main(configdir, basetempdir=None, no_clean=False, no_workers=False, use_system_version=False):
- configdir = os.path.abspath(os.path.realpath(configdir))
- if not configdir_is_valid(configdir):
- raise ValueError('The configdir {} dont contains the buildmaster files expected by this script'.format(configdir))
- base_workdir_temp = os.path.abspath(os.path.realpath(create_tempdir(basetempdir)))
- if base_workdir_temp.startswith(configdir):
- raise ValueError('The temporal working directory {} cant be located inside configdir {}'.format(base_workdir_temp, configdir))
- try:
- if not use_system_version:
- setup_virtualenv(base_workdir_temp)
- check_buildbot_installed()
- master_workdir = setup_master_workdir(configdir, base_workdir_temp)
- master_runner = multiprocessing.Process(target=start_master, args=(master_workdir,))
- master_runner.start()
- wait_for_master_ready(master_workdir)
- if no_workers:
- print(' - To manually attach a build worker use this info:\n'
- + ' TCP port for the worker-to-master connection: 17000\n'
- + ' worker-id: the one defined at {}\n'.format(os.path.join(master_workdir, 'passwords.json'))
- + ' password: 1234\n')
- else:
- worker_runners = []
- for worker in get_list_workers(master_workdir):
- worker_runner = multiprocessing.Process(target=start_worker, args=(base_workdir_temp, worker,))
- worker_runner.start()
- worker_runners.append(worker_runner)
- print(' - Workers started!.\n'
- + ' Check the log for each one at {}/${worker-name-id}/worker.log\n'.format(base_workdir_temp)
- + ' tail -f {}/*/worker.log\n'.format(base_workdir_temp))
- for worker_runner in worker_runners:
- worker_runner.join()
- master_runner.join()
- except:
- traceback.print_exc()
- finally:
- try:
- # The children may exit between the check and the kill call.
- # Ignore any exception raised here.
- for c in multiprocessing.active_children():
- # Send the signal to the whole process group.
- # Otherwise some twistd sub-childs can remain alive.
- os.killpg(os.getpgid(c.pid), signal.SIGKILL)
- except:
- pass
- if not no_clean:
- clean(base_workdir_temp)
- sys.exit(0)
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser()
- parser.add_argument('--config-dir', help='Path to the directory of the build master config files. '
- 'Defauls to the directory where this script is located.',
- dest='configdir', type=str,
- default=os.path.dirname(__file__))
- parser.add_argument('--base-temp-dir', help='Path where the temporal working directory will be created. '
- 'Note: To trigger test builds with the test workers you need enough free space on that path.',
- dest='basetempdir', default=None, type=str)
- parser.add_argument('--no-clean', help='Do not clean the temporal working dir on exit.',
- dest='no_clean', action='')
- parser.add_argument('--no-workers', help='Do not start the test workers.',
- dest='no_workers', action='')
- parser.add_argument('--use-system-version', help='Instead of setting up a virtualenv with the buildbot version '
- 'used by build.webkit.org, use the buildbot version installed on this system.',
- dest='use_system_version', action='')
- args = parser.parse_args()
- main(args.configdir, args.basetempdir, args.no_clean, args.no_workers, args.use_system_version)
Modified: trunk/Tools/CISupport/ews-build/loadConfig.py (276867 => 276868)
--- trunk/Tools/CISupport/ews-build/loadConfig.py 2021-05-01 01:19:18 UTC (rev 276867)
+++ trunk/Tools/CISupport/ews-build/loadConfig.py 2021-05-01 01:20:56 UTC (rev 276868)
@@ -27,6 +27,7 @@
from buildbot.scheduler import AnyBranchScheduler, Periodic, Dependent, Triggerable, Nightly
from buildbot.schedulers.trysched import Try_Userpass
+from buildbot.schedulers.forcesched import ForceScheduler, StringParameter, FixedParameter, CodebaseParameter
from buildbot.worker import Worker
from buildbot.util import identifiers as buildbot_identifiers
@@ -84,7 +85,24 @@
scheduler['userpass'] = [(os.getenv('BUILDBOT_TRY_USERNAME', 'sampleuser'), os.getenv('BUILDBOT_TRY_PASSWORD', 'samplepass'))]
c['schedulers'].append(schedulerClass(**scheduler))
+ if is_test_mode_enabled:
+ forceScheduler = ForceScheduler(
+ name="force_build",
+ buttonName="Force Build",
+ builderNames=[str(builder['name']) for builder in config['builders']],
+ # Disable default enabled input fields: branch, repository, project, additional properties
+ codebases=[CodebaseParameter("",
+ revision=FixedParameter(name="revision", default=""),
+ repository=FixedParameter(name="repository", default=""),
+ project=FixedParameter(name="project", default=""),
+ branch=FixedParameter(name="branch", default=""))],
+ # Add custom properties needed
+ properties=[StringParameter(name="patch_id", label="Patch attachment id number (not bug number)", required=True, maxsize=7),
+ StringParameter(name="ews_revision", label="WebKit git sha1 hash to checkout before trying patch (optional)", required=False, maxsize=40)],
+ )
+ c['schedulers'].append(forceScheduler)
+
def prioritizeBuilders(buildmaster, builders):
# Prioritize builder queues over tester queues
builders.sort(key=lambda b: 'build' in b.name.lower(), reverse=True)
Modified: trunk/Tools/CISupport/ews-build/steps.py (276867 => 276868)
--- trunk/Tools/CISupport/ews-build/steps.py 2021-05-01 01:19:18 UTC (rev 276867)
+++ trunk/Tools/CISupport/ews-build/steps.py 2021-05-01 01:20:56 UTC (rev 276868)
@@ -314,7 +314,10 @@
def start(self):
patch = self._get_patch()
if not patch:
- self.finished(FAILURE)
+ # Forced build, don't have patch_id raw data on the request, need to fech it.
+ patch_id = self.getProperty('patch_id', '')
+ self.command = ['/bin/sh', '-c', 'curl -L "https://bugs.webkit.org/attachment.cgi?id={}" -o .buildbot-diff && {}'.format(patch_id, ' '.join(self.command))]
+ shell.ShellCommand.start(self)
return None
patch_reviewer_name = self.getProperty('patch_reviewer_full_name', '')
Added: trunk/Tools/CISupport/start-local-buildbot-server (0 => 276868)
--- trunk/Tools/CISupport/start-local-buildbot-server (rev 0)
+++ trunk/Tools/CISupport/start-local-buildbot-server 2021-05-01 01:20:56 UTC (rev 276868)
@@ -0,0 +1,398 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2017, 2021 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+import sys
+import signal
+import os
+import argparse
+import subprocess
+import tempfile
+import shutil
+import socket
+import json
+import traceback
+import multiprocessing
+from time import sleep
+
+buildbot_server_tac = """
+import os
+from twisted.application import service
+from buildbot.master import BuildMaster
+
+basedir = os.path.dirname(os.path.realpath(__file__))
+configfile = r'%(config_file)s'
+
+application = service.Application('buildmaster')
+BuildMaster(basedir, configfile).setServiceParent(application)
+"""
+
+buildbot_worker_tac = """
+import os
+from buildbot_worker.bot import Worker
+from twisted.application import service
+
+basedir = os.path.dirname(os.path.realpath(__file__))
+application = service.Application('buildbot-worker')
+
+buildmaster_host = 'localhost'
+port = %(server_pb_port)s
+workername = '%(worker_name)s'
+passwd = 'password'
+keepalive = 600
+
+s = Worker(buildmaster_host, port, workername, passwd, basedir, keepalive)
+s.setServiceParent(application)
+"""
+
+
+def check_tcp_port_open(address, port):
+ s = socket.socket()
+ try:
+ s.connect((address, port))
+ return True
+ except ConnectionRefusedError:
+ return False
+
+
+def create_tempdir(tmpdir=None):
+ if tmpdir is not None:
+ if not os.path.isdir(tmpdir):
+ raise ValueError('{} is not a directory'.format(tmpdir))
+ return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(tmpdir), 'tmp'))
+ return tempfile.mkdtemp()
+
+
+def print_if_error_stdout_stderr(cmd, retcode, stdout=None, stderr=None, extramsg=None):
+ if retcode != 0:
+ if type(cmd) is list:
+ cmd = ' '.join(cmd)
+ print('WARNING: "{cmd}" returned {retcode} status code'.format(cmd=cmd, retcode=retcode))
+ if stdout is not None:
+ print('STDOUT:\n' + stdout.decode('utf-8'))
+ if stderr is not None:
+ print('STDERR:\n' + stderr.decode('utf-8'))
+ if extramsg is not None:
+ print(extramsg)
+
+
+def cmd_exists(cmd):
+ return any(os.access(os.path.join(path, cmd), os.X_OK)
+ for path in os.environ['PATH'].split(os.pathsep))
+
+
+class BuildbotTestRunner(object):
+
+ def __init__(self, configdir):
+ self._configdir = os.path.abspath(os.path.realpath(configdir))
+ if not os.path.isdir(self._configdir):
+ raise RuntimeError('The configdir {} is not a directory'.format(self._configdir))
+ if not os.path.isfile(os.path.join(self._configdir, 'config.json')):
+ raise RuntimeError('The configdir {} does not contain a config.json file'.format(self._configdir))
+
+ number_config_files = 0
+ for file in os.listdir(self._configdir):
+ if file.endswith('.cfg'):
+ self._server_config_file_name = file
+ number_config_files += 1
+ if number_config_files == 0:
+ raise RuntimeError('The configdir {} does not contain a .cfg file'.format(self._configdir))
+ if number_config_files != 1:
+ raise RuntimeError('The configdir {} has more than one .cfg file'.format(self._configdir))
+
+ self._server_http_port, self._server_pb_port = self._get_config_tcp_ports()
+
+ def _get_config_tcp_ports(self):
+ pb_port = 17000
+ http_port = 8010
+ try:
+ with open(os.path.join(self._configdir, self._server_config_file_name)) as f:
+ for line in f:
+ if '=' in line:
+ if 'protocols' in line and 'pb' in line and 'port' in line:
+ pb = eval(line.split('=', 1)[1])
+ pb_port = int((pb['pb']['port']))
+ if 'www' in line and 'port' in line:
+ http = eval(line.split('=', 1)[1])
+ http_port = int((http['port']))
+ except:
+ print("Unable to detect http and pb ports from config. Using defaults")
+ return http_port, pb_port
+
+ def start(self, basetempdir=None, no_clean=False, number_workers=1, use_system_version=False):
+ try:
+ self._base_workdir_temp = os.path.abspath(os.path.realpath(create_tempdir(basetempdir)))
+ if self._base_workdir_temp.startswith(self._configdir):
+ raise ValueError('The temporal working directory {} cant be located inside configdir {}'.format(self._base_workdir_temp, self._configdir))
+ if not use_system_version:
+ self._setup_virtualenv()
+ if not (cmd_exists('twistd') and cmd_exists('buildbot')):
+ raise RuntimeError('Buildbot is not installed.')
+ self._setup_server_workdir()
+ server_runner = multiprocessing.Process(target=self._start_server)
+ server_runner.start()
+ self._wait_for_server_ready()
+ if number_workers == 0:
+ print(' - To manually attach a build worker use this info:\n'
+ + ' TCP port for the worker-to-server connection: {}\n'.format(self._server_pb_port)
+ + ' worker-id: the one defined at {}\n'.format(os.path.join(self._server_wordir, 'passwords.json'))
+ + ' password: password\n')
+ elif number_workers == 1:
+ worker = 'local-worker'
+ worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,))
+ worker_runner.start()
+ print(' - Worker started!.\n'
+ + ' Check the log for at {}/local-worker/worker.log\n'.format(self._base_workdir_temp)
+ + ' tail -f {}/local-worker/worker.log\n'.format(self._base_workdir_temp))
+ worker_runner.join()
+ else:
+ worker_runners = []
+ for worker in self._get_list_workers():
+ worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,))
+ worker_runner.start()
+ worker_runners.append(worker_runner)
+ print(' - Workers started!.\n'
+ + ' Check the log for each one at {}/WORKERNAMEID/worker.log\n'.format(self._base_workdir_temp)
+ + ' tail -f {}/*/worker.log\n'.format(self._base_workdir_temp))
+ for worker_runner in worker_runners:
+ worker_runner.join()
+ server_runner.join()
+ except KeyboardInterrupt:
+ pass # no print the exception
+ except:
+ traceback.print_exc()
+ finally:
+ try:
+ # The children may exit between the check and the kill call.
+ # Ignore any exception raised here.
+ for c in multiprocessing.active_children():
+ # Send the signal to the whole process group.
+ # Otherwise some twistd sub-childs can remain alive.
+ os.killpg(os.getpgid(c.pid), signal.SIGKILL)
+ except:
+ pass
+ if not no_clean:
+ self._clean()
+ sys.exit(0)
+
+ def _wait_for_server_ready(self):
+ server_ready_check_counter = 0
+ while True:
+ if os.path.isfile(self._server_ready_fd):
+ return
+ if server_ready_check_counter > 60:
+ print('ERROR: buildbot server has not started after waiting 60 seconds for it.')
+
+ if os.path.isfile(self._server_log):
+ print('The server.log file contains the following:')
+ with open(self._server_log, 'r') as f:
+ print(f.read())
+ else:
+ print('There is no server.log on the directory. That means a general failure starting buildbot.')
+
+ raise RuntimeError('buildbot server has not started after waiting 60 seconds for it.')
+ sleep(1)
+ server_ready_check_counter += 1
+
+ def _create_mock_worker_passwords_dict(self):
+ with open(os.path.join(self._server_wordir, 'config.json'), 'r') as config_json:
+ config_dict = json.load(config_json)
+ result = dict([(worker['name'], 'password') for worker in config_dict['workers']])
+ return result
+
+ def _setup_server_workdir(self):
+ self._server_wordir = os.path.join(self._base_workdir_temp, 'buildbot-server')
+ assert(not os.path.exists(self._server_wordir))
+ self._server_log = os.path.join(self._server_wordir, 'server.log')
+ self._server_ready_fd = os.path.join(self._server_wordir, '.server-is-ready')
+ print('Copying files from {} to {} ...'.format(self._configdir, self._server_wordir))
+ shutil.copytree(self._configdir, self._server_wordir)
+ print('Generating buildbot files at {} ...'.format(self._server_wordir))
+ with open(os.path.join(self._server_wordir, 'buildbot.tac'), 'w') as f:
+ f.write(buildbot_server_tac % {'config_file': self._server_config_file_name})
+ with open(os.path.join(self._server_wordir, 'passwords.json'), 'w') as passwords_file:
+ passwords_file.write(json.dumps(self._create_mock_worker_passwords_dict(), indent=4, sort_keys=True))
+
+ def _setup_virtualenv(self):
+ if cmd_exists('virtualenv'):
+ print('Setting up virtualenv at {} ... '.format(self._base_workdir_temp))
+ virtualenv_cmd = ['virtualenv', '-p', 'python3', 'venv']
+ virtualenv_process = subprocess.Popen(virtualenv_cmd, cwd=self._base_workdir_temp,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout, stderr = virtualenv_process.communicate()
+ print_if_error_stdout_stderr(virtualenv_cmd, virtualenv_process.returncode, stdout, stderr)
+ virtualenv_bindir = os.path.join(self._base_workdir_temp, 'venv', 'bin')
+ virtualenv_pip = os.path.join(virtualenv_bindir, 'pip')
+ if not os.access(virtualenv_pip, os.X_OK):
+ print('Something went wrong setting up virtualenv'
+ 'Trying to continue using the system version of buildbot')
+ return
+ print('Setting up buildbot dependencies on the virtualenv ... ')
+ # The idea is to install the very same version of buildbot and its
+ # dependencies than the ones used for running https://build.webkit.org/about
+ buildbot_version = '2.10.5'
+ pip_cmd = [virtualenv_pip, 'install',
+ 'buildbot=={}'.format(buildbot_version),
+ 'buildbot-console-view=={}'.format(buildbot_version),
+ 'buildbot-grid-view=={}'.format(buildbot_version),
+ 'buildbot-waterfall-view=={}'.format(buildbot_version),
+ 'buildbot-worker=={}'.format(buildbot_version),
+ 'buildbot-www=={}'.format(buildbot_version),
+ 'twisted==21.2.0',
+ 'requests==2.21.0',
+ 'lz4==1.1.0']
+ pip_process = subprocess.Popen(pip_cmd, cwd=self._base_workdir_temp,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout, stderr = pip_process.communicate()
+ print_if_error_stdout_stderr(pip_cmd, pip_process.returncode, stdout, stderr)
+ os.environ['PATH'] = virtualenv_bindir + ':' + os.environ['PATH']
+ return
+ print('WARNING: virtualenv not installed. '
+ 'Trying to continue using the system version of buildbot')
+
+ def _upgrade_db_needed(self):
+ with open(self._server_log) as f:
+ for l in f:
+ if 'upgrade the database' in l:
+ return True
+ return False
+
+ def _start_server(self):
+ # This is started via multiprocessing. We set a new process group here
+ # to be able to reliably kill this subprocess and all of its child on clean.
+ os.setsid()
+ dbupgraded = False
+ retry = True
+ if check_tcp_port_open('localhost', self._server_pb_port):
+ print('ERROR: There is some process already listening in port {}'.format(self._server_pb_port))
+ return 1
+ while retry:
+ retry = False
+ print('Starting the buildbot server process ...')
+ twistd_cmd = ['twistd', '-l', self._server_log, '-noy', 'buildbot.tac']
+ twistd_server_process = subprocess.Popen(twistd_cmd, cwd=self._server_wordir,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ while twistd_server_process.poll() is None:
+ if check_tcp_port_open('localhost', self._server_pb_port):
+ print('Test buildbot server ready!\n'
+ + 'Press CTRL-C to stop\n\n'
+ + ' - See buildbot server log:\n'
+ + ' tail -f {}\n\n'.format(self._server_log)
+ + ' - Open a browser to:\n'
+ + ' http://localhost:{}\n'.format(self._server_http_port))
+ with open(self._server_ready_fd, 'w') as f:
+ f.write('ready')
+ twistd_server_process.wait()
+ return 0
+ sleep(1)
+ stdout, stderr = twistd_server_process.communicate()
+ if twistd_server_process.returncode == 0 and self._upgrade_db_needed() and not dbupgraded:
+ retry = True
+ dbupgraded = True
+ print('Upgrading the database ...')
+ upgrade_cmd = ['buildbot', 'upgrade-master', self._server_wordir]
+ upgrade_process = subprocess.Popen(upgrade_cmd, cwd=self._server_wordir,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout, stderr = upgrade_process.communicate()
+ print_if_error_stdout_stderr(upgrade_cmd, upgrade_process.returncode, stdout, stderr)
+ else:
+ print_if_error_stdout_stderr(twistd_cmd, twistd_server_process.returncode, stdout, stderr,
+ 'Check the log at {}'.format(self._server_log))
+ return twistd_server_process.returncode
+
+ def _get_list_workers(self):
+ password_list = os.path.join(self._server_wordir, 'passwords.json')
+ with open(password_list) as f:
+ passwords = json.load(f)
+ list_workers = []
+ for worker in passwords:
+ list_workers.append(str(worker))
+ return list_workers
+
+ def _start_worker(self, worker):
+ # This is started via multiprocessing. We set a new process group here
+ # to be able to reliably kill this subprocess and all of its child on clean.
+ os.setsid()
+ worker_workdir = os.path.join(self._base_workdir_temp, worker)
+ os.mkdir(worker_workdir)
+ with open(os.path.join(worker_workdir, 'buildbot.tac'), 'w') as f:
+ f.write(buildbot_worker_tac % {'worker_name': worker, 'server_pb_port': self._server_pb_port})
+ twistd_cmd = ['twistd', '-l', 'worker.log', '-noy', 'buildbot.tac']
+ twistd_worker_process = subprocess.Popen(twistd_cmd, cwd=worker_workdir,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout, stderr = twistd_worker_process.communicate()
+ print_if_error_stdout_stderr(twistd_cmd, twistd_worker_process.returncode, stdout, stderr,
+ 'Check the log at {}'.format(os.path.join(worker_workdir, 'worker.log')))
+ return twistd_worker_process.returncode
+
+ def _clean(self):
+ if os.path.isdir(self._base_workdir_temp):
+ print('\n\nCleaning {} ... \n'.format(self._base_workdir_temp))
+ # shutil.rmtree can fail if we hold an open file descriptor on temp_dir
+ # (which is very likely when cleaning) or if temp_dir is a NFS mount.
+ # Use rm instead that always works.
+ rm = subprocess.Popen(['rm', '-fr', self._base_workdir_temp])
+ rm.wait()
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ configuration = parser.add_mutually_exclusive_group(required=True)
+ configuration.add_argument('--ews', action='', const='ews', dest='configuration',
+ help='Simulate the EWS buildbot server (ews-build.webkit.org)')
+ configuration.add_argument('--post-commit', action='', const='post_commit', dest='configuration',
+ help='Simulate the post-commit buildbot server (build.webkit.org)')
+ configuration.add_argument('--config-dir', default=None, dest='configdir', type=str,
+ help='Specify the directory that contains the buildbot config files')
+ parser.add_argument('--base-temp-dir', help='Path where the temporal working directory will be created. '
+ 'Note: To trigger test builds with the test workers you need enough free space on that path.',
+ dest='basetempdir', default=None, type=str)
+ parser.add_argument('--no-clean', help='Do not clean the temporal working dir on exit.',
+ dest='no_clean', action='')
+ workers = parser.add_mutually_exclusive_group(required=False)
+ workers.add_argument('--no-workers', help='Do not start any workers.',
+ dest='number_workers', action='', const=0)
+ workers.add_argument('--all-workers', help='Instead of starting only one worker that round-robins between all the queues, '
+ 'start multiple parallel workers as defined on the server config.',
+ dest='number_workers', action='', const=float('inf'))
+ parser.add_argument('--use-system-version', help='Instead of setting up a virtualenv with the buildbot version '
+ 'used by build.webkit.org, use the buildbot version installed on this system.',
+ dest='use_system_version', action='')
+ args = parser.parse_args()
+
+ if args.configuration == "ews":
+ configdir = os.path.join(os.path.dirname(__file__), 'ews-build')
+ elif args.configuration == "post_commit":
+ configdir = os.path.join(os.path.dirname(__file__), 'build-webkit-org')
+ else:
+ configdir = args.configdir
+
+ if args.number_workers is None:
+ args.number_workers = 1
+
+ buildbot_test_runner = BuildbotTestRunner(configdir)
+ buildbot_test_runner.start(args.basetempdir, args.no_clean, args.number_workers, args.use_system_version)
Property changes on: trunk/Tools/CISupport/start-local-buildbot-server
___________________________________________________________________
Added: svn:executable
+*
\ No newline at end of property
Modified: trunk/Tools/ChangeLog (276867 => 276868)
--- trunk/Tools/ChangeLog 2021-05-01 01:19:18 UTC (rev 276867)
+++ trunk/Tools/ChangeLog 2021-05-01 01:20:56 UTC (rev 276868)
@@ -1,3 +1,45 @@
+2021-04-30 Carlos Alberto Lopez Perez <clo...@igalia.com>
+
+ [tools] Make run-buildbot-test compatible with buildbot 2.10.5
+ https://bugs.webkit.org/show_bug.cgi?id=222540
+
+ Reviewed by Aakash Jain.
+
+ This renames the previous tool run-buildbot-test to start-buildbot-server-virtualenv
+ and it makes several changes to it:
+ - Use python3 and refactor the code.
+ - Make it also work with the EWS config (previously it only worked for the build.webkit.org config).
+ - Use newer buildbot configs.
+ - Instead of hardcoding values try to automatically detect the values from the config dir.
+ - Instead of starting Nth workers by default start only one round-robin worker (local-worker).
+
+ It also modifies the configuration of the EWS server to add a force scheduler in order
+ to allow to manually trigger builds (only in test mode).
+
+ * CISupport/build-webkit-org/run-buildbot-test.py: Removed.
+ * CISupport/ews-build/loadConfig.py:
+ (loadBuilderConfig):
+ * CISupport/ews-build/steps.py:
+ (ApplyPatch.start):
+ * CISupport/start-local-buildbot-server: Added.
+ (check_tcp_port_open):
+ (create_tempdir):
+ (print_if_error_stdout_stderr):
+ (cmd_exists):
+ (BuildbotTestRunner):
+ (BuildbotTestRunner.__init__):
+ (BuildbotTestRunner._get_config_tcp_ports):
+ (BuildbotTestRunner.start):
+ (BuildbotTestRunner._wait_for_server_ready):
+ (BuildbotTestRunner._create_mock_worker_passwords_dict):
+ (BuildbotTestRunner._setup_server_workdir):
+ (BuildbotTestRunner._setup_virtualenv):
+ (BuildbotTestRunner._upgrade_db_needed):
+ (BuildbotTestRunner._start_server):
+ (BuildbotTestRunner._get_list_workers):
+ (BuildbotTestRunner._start_worker):
+ (BuildbotTestRunner._clean):
+
2021-04-30 Brent Fulgham <bfulg...@apple.com>
[Cocoa] Always extend access to local process HTTP/3 cache directory
