- Revision
- 258611
- Author
- [email protected]
- Date
- 2020-03-17 17:01:42 -0700 (Tue, 17 Mar 2020)
Log Message
[ews] Resubmit patches to commit-queue which were cq- by commit-queue and later cq+
https://bugs.webkit.org/show_bug.cgi?id=208920
Reviewed by Jonathan Bedard.
* BuildSlaveSupport/ews-app/ews/common/buildbot.py:
(Buildbot.update_builder_name_to_id_mapping): Method to generate builder name to id mapping.
(Buildbot.fetch_pending_and_inprogress_builds): Method to fetch pendign and in-progress builds from buildbot.
(Buildbot.get_patches_in_queue): Method to fetch list of patches which are pending or are in-progress on buildbot.
* BuildSlaveSupport/ews-app/ews/common/bugzilla.py:
(Bugzilla.get_cq_plus_timestamp): Get UTC timestamp when cq+ flag was set.
* BuildSlaveSupport/ews-app/ews/fetcher.py:
(FetchLoop.run): Update builder name to id mapping. We just need to update it one-time, since we need the id only
for commit-queue which isn't expected to change.
(BugzillaPatchFetcher.fetch_commit_queue_patches):
(BugzillaPatchFetcher.send_patches_to_buildbot): Allow sending the patch again to commit-queue.
(BugzillaPatchFetcher.patches_to_send_to_commit_queue): Find patches which needs to be sent to commit-queue. Filter
out patches which are already waiting or in-progress on commit-queue, or whose recent build was completed after
setting cq+ flag on bugzilla.
Modified Paths
Diff
Modified: trunk/Tools/BuildSlaveSupport/ews-app/ews/common/bugzilla.py (258610 => 258611)
--- trunk/Tools/BuildSlaveSupport/ews-app/ews/common/bugzilla.py 2020-03-17 23:32:57 UTC (rev 258610)
+++ trunk/Tools/BuildSlaveSupport/ews-app/ews/common/bugzilla.py 2020-03-18 00:01:42 UTC (rev 258611)
@@ -33,6 +33,7 @@
from ews.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer
import ews.common.util as util
import ews.config as config
+import dateutil.parser
_log = logging.getLogger(__name__)
@@ -51,6 +52,21 @@
return attachment_json
@classmethod
+ def get_cq_plus_timestamp(cls, attachment_id):
+ attachment_json = Bugzilla._fetch_attachment_json(attachment_id)
+ if not attachment_json:
+ _log.warn('Unable to fetch attachment {}.'.format(attachment_id))
+ return None
+
+ for flag in attachment_json.get('flags'):
+ if flag.get('name') == 'commit-queue' and flag.get('status') == '+':
+ try:
+ return dateutil.parser.parse(flag.get('modification_date'))
+ except:
+ _log.error('Unable to parse timestamp: {}'.format(flag.get('modification_date')))
+ return None
+
+ @classmethod
def save_attachment(cls, attachment_id, attachment_data):
with open(Bugzilla.file_path_for_patch(attachment_id), 'w') as attachment_file:
attachment_file.write(attachment_data)
Modified: trunk/Tools/BuildSlaveSupport/ews-app/ews/common/buildbot.py (258610 => 258611)
--- trunk/Tools/BuildSlaveSupport/ews-app/ews/common/buildbot.py 2020-03-17 23:32:57 UTC (rev 258610)
+++ trunk/Tools/BuildSlaveSupport/ews-app/ews/common/buildbot.py 2020-03-18 00:01:42 UTC (rev 258611)
@@ -39,6 +39,7 @@
ALL_RESULTS = lrange(7)
SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, CANCELLED = ALL_RESULTS
icons_for_queues_mapping = {}
+ builder_name_to_id_mapping = {}
@classmethod
def send_patch_to_buildbot(cls, patch_path, send_to_commit_queue=False, properties=None):
@@ -99,9 +100,40 @@
shortname = builder.get('shortname')
Buildbot.icons_for_queues_mapping[shortname] = builder.get('icon')
- return Buildbot.icons_for_queues_mapping
+ @classmethod
+ def update_builder_name_to_id_mapping(cls):
+ url = ''.format(config.BUILDBOT_SERVER_HOST)
+ builders_data = util.fetch_data_from_url(url)
+ if not builders_data:
+ return
+ for builder in builders_data.json().get('builders', []):
+ name = builder.get('name')
+ Buildbot.builder_name_to_id_mapping[name] = builder.get('builderid')
@classmethod
+ def fetch_pending_and_inprogress_builds(cls, builder_full_name):
+ builderid = Buildbot.builder_name_to_id_mapping.get(builder_full_name)
+ if not builderid:
+ _log.error('Invalid builder: {}'.format(builder_full_name))
+ return {}
+ url = ''.format(config.BUILDBOT_SERVER_HOST, builderid)
+ builders_data = util.fetch_data_from_url(url)
+ if not builders_data:
+ return {}
+ return builders_data.json()
+
+ @classmethod
+ def get_patches_in_queue(cls, builder_full_name):
+ patch_ids = []
+ builds = cls.fetch_pending_and_inprogress_builds(builder_full_name)
+ for buildrequest in builds.get('buildrequests', []):
+ properties = buildrequest.get('properties')
+ if properties:
+ patch_ids.append(properties.get('patch_id')[0])
+ _log.debug('Patches in queue for {}: {}'.format(builder_full_name, patch_ids))
+ return patch_ids
+
+ @classmethod
def retry_build(cls, builder_id, build_number):
if not (util.is_valid_id(builder_id) and util.is_valid_id(build_number)):
return False
Modified: trunk/Tools/BuildSlaveSupport/ews-app/ews/fetcher.py (258610 => 258611)
--- trunk/Tools/BuildSlaveSupport/ews-app/ews/fetcher.py 2020-03-17 23:32:57 UTC (rev 258610)
+++ trunk/Tools/BuildSlaveSupport/ews-app/ews/fetcher.py 2020-03-18 00:01:42 UTC (rev 258611)
@@ -20,7 +20,9 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import datetime
import logging
+import pytz
import threading
import time
@@ -27,6 +29,7 @@
from ews.common.bugzilla import Bugzilla
from ews.common.buildbot import Buildbot
from ews.models.patch import Patch
+from ews.views.statusbubble import StatusBubble
_log = logging.getLogger(__name__)
@@ -39,6 +42,7 @@
thread.start()
def run(self):
+ Buildbot.update_builder_name_to_id_mapping()
while True:
Buildbot.update_icons_for_queues_mapping()
try:
@@ -69,7 +73,7 @@
patch_ids_commit_queue = BugzillaPatchFetcher.filter_valid_patches(patch_ids_commit_queue)
_log.debug('cq+ patches: {}'.format(patch_ids_commit_queue))
Patch.save_patches(patch_ids_commit_queue)
- patches_to_send = self.patches_to_send_to_buildbot(patch_ids_commit_queue, commit_queue=True)
+ patches_to_send = self.patches_to_send_to_commit_queue(patch_ids_commit_queue)
_log.info('{} cq+ patches, {} patches need to be sent to commit queue: {}'.format(len(patch_ids_commit_queue), len(patches_to_send), patches_to_send))
self.send_patches_to_buildbot(patches_to_send, send_to_commit_queue=True)
@@ -85,8 +89,8 @@
_log.warn('Patch is obsolete, skipping')
Patch.set_obsolete(patch_id)
continue
- if Patch.is_patch_sent_to_buildbot(patch_id, commit_queue=send_to_commit_queue):
- _log.error('Patch {} is already sent to buildbot/commit-queue.'.format(patch_id))
+ if not send_to_commit_queue and Patch.is_patch_sent_to_buildbot(patch_id):
+ _log.error('Patch {} is already sent to buildbot.'.format(patch_id))
continue
Patch.set_sent_to_buildbot(patch_id, True, commit_queue=send_to_commit_queue)
rc = Buildbot.send_patch_to_buildbot(bz_patch['path'],
@@ -99,9 +103,31 @@
Patch.set_sent_to_buildbot(patch_id, False, commit_queue=send_to_commit_queue)
#FIXME: send an email for this failure
- def patches_to_send_to_buildbot(self, patch_ids, commit_queue=False):
- return [patch_id for patch_id in patch_ids if not Patch.is_patch_sent_to_buildbot(patch_id, commit_queue)]
+ def patches_to_send_to_buildbot(self, patch_ids):
+ return [patch_id for patch_id in patch_ids if not Patch.is_patch_sent_to_buildbot(patch_id)]
+ def patches_to_send_to_commit_queue(self, patch_ids):
+ if not patch_ids:
+ return patch_ids
+ patches_in_queue = set(Buildbot.get_patches_in_queue('Commit-Queue'))
+ patch_ids = [patch_id for patch_id in set(patch_ids) if str(patch_id) not in patches_in_queue]
+
+ patch_ids_to_send = []
+ for patch_id in patch_ids:
+ patch = Patch.get_patch(patch_id)
+ recent_build, _ = StatusBubble().get_latest_build_for_queue(patch, 'commit')
+ if not recent_build:
+ patch_ids_to_send.append(patch_id)
+ continue
+ recent_build_timestamp = datetime.datetime.fromtimestamp(recent_build.complete_at, tz=pytz.UTC)
+ cq_timestamp = Bugzilla.get_cq_plus_timestamp(patch_id)
+ if not cq_timestamp:
+ patch_ids_to_send.append(patch_id)
+ continue
+ if cq_timestamp > recent_build_timestamp:
+ patch_ids_to_send.append(patch_id)
+ return patch_ids_to_send
+
@classmethod
def filter_valid_patches(cls, patch_ids):
return list(filter(lambda p: Patch.is_valid_patch_id(p), patch_ids))
Modified: trunk/Tools/ChangeLog (258610 => 258611)
--- trunk/Tools/ChangeLog 2020-03-17 23:32:57 UTC (rev 258610)
+++ trunk/Tools/ChangeLog 2020-03-18 00:01:42 UTC (rev 258611)
@@ -1,3 +1,25 @@
+2020-03-17 Aakash Jain <[email protected]>
+
+ [ews] Resubmit patches to commit-queue which were cq- by commit-queue and later cq+
+ https://bugs.webkit.org/show_bug.cgi?id=208920
+
+ Reviewed by Jonathan Bedard.
+
+ * BuildSlaveSupport/ews-app/ews/common/buildbot.py:
+ (Buildbot.update_builder_name_to_id_mapping): Method to generate builder name to id mapping.
+ (Buildbot.fetch_pending_and_inprogress_builds): Method to fetch pendign and in-progress builds from buildbot.
+ (Buildbot.get_patches_in_queue): Method to fetch list of patches which are pending or are in-progress on buildbot.
+ * BuildSlaveSupport/ews-app/ews/common/bugzilla.py:
+ (Bugzilla.get_cq_plus_timestamp): Get UTC timestamp when cq+ flag was set.
+ * BuildSlaveSupport/ews-app/ews/fetcher.py:
+ (FetchLoop.run): Update builder name to id mapping. We just need to update it one-time, since we need the id only
+ for commit-queue which isn't expected to change.
+ (BugzillaPatchFetcher.fetch_commit_queue_patches):
+ (BugzillaPatchFetcher.send_patches_to_buildbot): Allow sending the patch again to commit-queue.
+ (BugzillaPatchFetcher.patches_to_send_to_commit_queue): Find patches which needs to be sent to commit-queue. Filter
+ out patches which are already waiting or in-progress on commit-queue, or whose recent build was completed after
+ setting cq+ flag on bugzilla.
+
2020-03-17 Chris Dumez <[email protected]>
[WKTR] testRunner API that takes in a completion handler should use async IPC
