Modified: trunk/Tools/CISupport/ews-build/loadConfig.py (292784 => 292785)
--- trunk/Tools/CISupport/ews-build/loadConfig.py 2022-04-12 21:20:37 UTC (rev 292784)
+++ trunk/Tools/CISupport/ews-build/loadConfig.py 2022-04-12 21:36:14 UTC (rev 292785)
@@ -31,6 +31,8 @@
from buildbot.worker import Worker
from buildbot.util import identifiers as buildbot_identifiers
from buildbot.changes.filter import ChangeFilter
+from datetime import datetime, timezone
+from twisted.internet import defer
from factories import (APITestsFactory, BindingsFactory, BuildFactory, CommitQueueFactory, Factory, GTKBuildFactory,
GTKTestsFactory, JSCBuildFactory, JSCBuildAndTestsFactory, JSCTestsFactory, MergeQueueFactory, StressTestFactory,
@@ -113,9 +115,33 @@
c['schedulers'].append(forceScheduler)
+# Copied from https://github.com/buildbot/buildbot/blob/master/master/buildbot/util/async_sort.py
+@defer.inlineCallbacks
+def async_sort(l, key, max_parallel=10):
+ sem = defer.DeferredSemaphore(max_parallel)
+ try:
+ keys = yield defer.gatherResults([sem.run(key, i) for i in l])
+ except defer.FirstError as e:
+ raise e.subFailure.value
+
+ keys = {id(l[i]): v for i, v in enumerate(keys)}
+ l.sort(key=lambda x: keys[id(x)])
+
+
def prioritizeBuilders(buildmaster, builders):
- # Prioritize builder queues over tester queues
- builders.sort(key=lambda b: 'build' in b.name.lower(), reverse=True)
+ # Prioritize builder queues over tester queues.
+ # Otherwise, prioritize older requests.
+ # Inspired by https://docs.buildbot.net/latest/manual/customization.html#builder-priority-functions
+ @defer.inlineCallbacks
+ def key(b):
+ request_time = yield b.getOldestRequestTime()
+ return (
+ 'build' not in b.name.lower(),
+ bool(b.building) or bool(b.old_building),
+ request_time or datetime.now(timezone.utc),
+ )
+
+ async_sort(builders, key)
return builders
Modified: trunk/Tools/CISupport/ews-build/loadConfig_unittest.py (292784 => 292785)
--- trunk/Tools/CISupport/ews-build/loadConfig_unittest.py 2022-04-12 21:20:37 UTC (rev 292784)
+++ trunk/Tools/CISupport/ews-build/loadConfig_unittest.py 2022-04-12 21:36:14 UTC (rev 292785)
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (C) 2018-2021 Apple Inc. All rights reserved.
+# Copyright (C) 2018-2022 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -27,6 +27,9 @@
import os
import unittest
+from datetime import datetime, timedelta, timezone
+from twisted.internet import defer
+
import loadConfig
@@ -246,5 +249,41 @@
loadConfig.checkWorkersAndBuildersForConsistency({}, [self.ews101, {'name': 'ews102', 'platform': 'mac-sierra'}], [self.WK2Builder])
+class TestPrioritizeBuilders(unittest.TestCase):
+ class MockBuilder(object):
+ def __init__(self, name, building=False, oldestRequestTime=None):
+ self.name = name
+ self.building = building
+ self.old_building = False
+ self._oldestRequestTime = oldestRequestTime or datetime.now(timezone.utc)
+
+ def getOldestRequestTime(self):
+ return self._oldestRequestTime
+
+ def test_builders_over_testers(self):
+ builders = [
+ self.MockBuilder('macOS-BigSur-Debug-Build-EWS'),
+ self.MockBuilder('macOS-BigSur-Debug-WK1-Tests-EWS'),
+ self.MockBuilder('macOS-BigSur-Release-Build-EWS'),
+ ]
+ sorted_builders = loadConfig.prioritizeBuilders(None, builders)
+ self.assertEqual(
+ ['macOS-BigSur-Debug-Build-EWS', 'macOS-BigSur-Release-Build-EWS', 'macOS-BigSur-Debug-WK1-Tests-EWS'],
+ [builder.name for builder in sorted_builders],
+ )
+
+ def test_starvation(self):
+ builders = [
+ self.MockBuilder('Commit-Queue', oldestRequestTime=datetime.now(timezone.utc) - timedelta(seconds=30)),
+ self.MockBuilder('Merge-Queue', oldestRequestTime=datetime.now(timezone.utc) - timedelta(seconds=10)),
+ self.MockBuilder('Unsafe-Merge-Queue', oldestRequestTime=datetime.now(timezone.utc) - timedelta(seconds=60)),
+ ]
+ sorted_builders = loadConfig.prioritizeBuilders(None, builders)
+ self.assertEqual(
+ ['Unsafe-Merge-Queue', 'Commit-Queue', 'Merge-Queue'],
+ [builder.name for builder in sorted_builders],
+ )
+
+
if __name__ == '__main__':
unittest.main()
Modified: trunk/Tools/ChangeLog (292784 => 292785)
--- trunk/Tools/ChangeLog 2022-04-12 21:20:37 UTC (rev 292784)
+++ trunk/Tools/ChangeLog 2022-04-12 21:36:14 UTC (rev 292785)
@@ -1,3 +1,21 @@
+2022-04-12 Jonathan Bedard <jbed...@apple.com>
+
+ [Merge-Queue] Prioritize oldest builds first
+ https://bugs.webkit.org/show_bug.cgi?id=239094
+ <rdar://problem/91595070 >]
+
+ Reviewed by Stephanie Lewis.
+
+ * CISupport/ews-build/loadConfig.py:
+ (loadBuilderConfig):
+ (async_sort): Added.
+ (prioritizeBuilders): Prioritize the builder with the oldest request.
+ * CISupport/ews-build/loadConfig_unittest.py:
+ (TestPrioritizeBuilders):
+ (TestPrioritizeBuilders.MockBuilder):
+ (TestPrioritizeBuilders.test_builders_over_testers):
+ (TestPrioritizeBuilders.test_starvation):
+
2022-04-12 Angelos Oikonomopoulos <ange...@igalia.com>
[JSC] Report test flakiness to resultsdb
