- Revision
- 159052
- Author
- [email protected]
- Date
- 2013-11-11 09:37:06 -0800 (Mon, 11 Nov 2013)
Log Message
The feeder queue will not feed patches older than one week.
https://bugs.webkit.org/show_bug.cgi?id=107152
Patch by Tamas Gergely <[email protected]> on 2013-11-11
Reviewed by Csaba Osztrogonác.
The feeder queue will not feed patches older than one week.
Unit test added to this scenario.
* Scripts/webkitpy/common/net/bugzilla/bugzilla.py:
(BugzillaQueries._parse_attachment_ids_request_query):
Handle date parameter.
(BugzillaQueries):
(BugzillaQueries._fetch_attachment_ids_request_query):
Propagate date.
(BugzillaQueries.fetch_attachment_ids_from_review_queue):
Propagate date.
* Scripts/webkitpy/common/net/bugzilla/bugzilla_mock.py:
(MockBugzillaQueries.fetch_attachment_ids_from_review_queue):
Handle date.
* Scripts/webkitpy/common/net/bugzilla/bugzilla_unittest.py:
(test_request_page_parsing):
Unit test added.
* Scripts/webkitpy/tool/bot/feeders.py:
(EWSFeeder.feed):
Query patch ids with date parameter (now - 1 week).
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (159051 => 159052)
--- trunk/Tools/ChangeLog 2013-11-11 17:33:19 UTC (rev 159051)
+++ trunk/Tools/ChangeLog 2013-11-11 17:37:06 UTC (rev 159052)
@@ -1,5 +1,33 @@
2013-11-11 Tamas Gergely <[email protected]>
+ The feeder queue will not feed patches older than one week.
+ https://bugs.webkit.org/show_bug.cgi?id=107152
+
+ Reviewed by Csaba Osztrogonác.
+
+ The feeder queue will not feed patches older than one week.
+ Unit test added to this scenario.
+
+ * Scripts/webkitpy/common/net/bugzilla/bugzilla.py:
+ (BugzillaQueries._parse_attachment_ids_request_query):
+ Handle date parameter.
+ (BugzillaQueries):
+ (BugzillaQueries._fetch_attachment_ids_request_query):
+ Propagate date.
+ (BugzillaQueries.fetch_attachment_ids_from_review_queue):
+ Propagate date.
+ * Scripts/webkitpy/common/net/bugzilla/bugzilla_mock.py:
+ (MockBugzillaQueries.fetch_attachment_ids_from_review_queue):
+ Handle date.
+ * Scripts/webkitpy/common/net/bugzilla/bugzilla_unittest.py:
+ (test_request_page_parsing):
+ Unit test added.
+ * Scripts/webkitpy/tool/bot/feeders.py:
+ (EWSFeeder.feed):
+ Query patch ids with date parameter (now - 1 week).
+
+2013-11-11 Tamas Gergely <[email protected]>
+
Do not shut down ImageDiff server after every diff during the tests.
https://bugs.webkit.org/show_bug.cgi?id=124140
Modified: trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla.py (159051 => 159052)
--- trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla.py 2013-11-11 17:33:19 UTC (rev 159051)
+++ trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla.py 2013-11-11 17:37:06 UTC (rev 159052)
@@ -1,6 +1,7 @@
# Copyright (c) 2011 Google Inc. All rights reserved.
# Copyright (c) 2009 Apple Inc. All rights reserved.
# Copyright (c) 2010 Research In Motion Limited. All rights reserved.
+# Copyright (c) 2013 University of Szeged. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
@@ -180,16 +181,37 @@
return [int(bug_link_cell.find("a").string)
for bug_link_cell in soup('td', "first-child")]
- def _parse_attachment_ids_request_query(self, page):
+ def _parse_attachment_ids_request_query(self, page, since=None):
+ # Formats
digits = re.compile("\d+")
attachment_href = re.compile("attachment.cgi\?id=\d+&action=""
- attachment_links = SoupStrainer("a", href=""
- return [int(digits.search(tag["href"]).group(0))
+ # if no date is given, return all ids
+ if not since:
+ attachment_links = SoupStrainer("a", href=""
+ return [int(digits.search(tag["href"]).group(0))
for tag in BeautifulSoup(page, parseOnlyThese=attachment_links)]
- def _fetch_attachment_ids_request_query(self, query):
- return self._parse_attachment_ids_request_query(self._load_query(query))
+ # Parse the main table only
+ date_format = re.compile("\d{4}-\d{2}-\d{2} \d{2}:\d{2}")
+ mtab = SoupStrainer("table", {"class": "requests"})
+ soup = BeautifulSoup(page, parseOnlyThese=mtab)
+ patch_ids = []
+ for row in soup.findAll("tr"):
+ patch_tag = row.find("a", {"href": attachment_href})
+ if not patch_tag:
+ continue
+ patch_id = int(digits.search(patch_tag["href"]).group(0))
+ date_tag = row.find("td", text=date_format)
+ if date_tag and datetime.strptime(date_format.search(date_tag).group(0), "%Y-%m-%d %H:%M") < since:
+ _log.info("Patch is old: %d (%s)" % (patch_id, date_tag))
+ continue
+ patch_ids.append(patch_id)
+ return patch_ids
+
+ def _fetch_attachment_ids_request_query(self, query, since=None):
+ return self._parse_attachment_ids_request_query(self._load_query(query), since)
+
def _parse_quips(self, page):
soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES)
quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li")
@@ -252,9 +274,9 @@
# NOTE: This is the only client of _fetch_attachment_ids_request_query
# This method only makes one request to bugzilla.
- def fetch_attachment_ids_from_review_queue(self):
+ def fetch_attachment_ids_from_review_queue(self, since=None):
review_queue_url = "request.cgi?action=""
- return self._fetch_attachment_ids_request_query(review_queue_url)
+ return self._fetch_attachment_ids_request_query(review_queue_url, since)
# This only works if your account has edituser privileges.
# We could easily parse https://bugs.webkit.org/userprefs.cgi?tab=permissions to
Modified: trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla_mock.py (159051 => 159052)
--- trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla_mock.py 2013-11-11 17:33:19 UTC (rev 159051)
+++ trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla_mock.py 2013-11-11 17:37:06 UTC (rev 159052)
@@ -251,9 +251,12 @@
self._all_bugs())
return map(lambda bug: bug.id(), bugs_with_commit_queued_patches)
- def fetch_attachment_ids_from_review_queue(self):
+ def fetch_attachment_ids_from_review_queue(self, since=None):
unreviewed_patches = sum([bug.unreviewed_patches()
for bug in self._all_bugs()], [])
+ if since:
+ unreviewed_pacthes = [patch for patch in unreviewed_patches
+ if patch.attach_date() >= since]
return map(lambda patch: patch.id(), unreviewed_patches)
def fetch_patches_from_commit_queue(self):
Modified: trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla_unittest.py (159051 => 159052)
--- trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla_unittest.py 2013-11-11 17:33:19 UTC (rev 159051)
+++ trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla_unittest.py 2013-11-11 17:37:06 UTC (rev 159052)
@@ -429,6 +429,7 @@
def test_request_page_parsing(self):
queries = BugzillaQueries(None)
self.assertEqual([40511, 40722, 40723], queries._parse_attachment_ids_request_query(self._sample_request_page))
+ self.assertEqual([40722, 40723], queries._parse_attachment_ids_request_query(self._sample_request_page, datetime.datetime(2009, 10, 4, 11, 38, 44)))
def test_quip_page_parsing(self):
queries = BugzillaQueries(None)
Modified: trunk/Tools/Scripts/webkitpy/tool/bot/feeders.py (159051 => 159052)
--- trunk/Tools/Scripts/webkitpy/tool/bot/feeders.py 2013-11-11 17:33:19 UTC (rev 159051)
+++ trunk/Tools/Scripts/webkitpy/tool/bot/feeders.py 2013-11-11 17:37:06 UTC (rev 159052)
@@ -26,6 +26,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from datetime import datetime, timedelta
import logging
from webkitpy.common.config.committervalidator import CommitterValidator
@@ -90,7 +91,7 @@
AbstractFeeder.__init__(self, tool)
def feed(self):
- ids_needing_review = set(self._tool.bugs.queries.fetch_attachment_ids_from_review_queue())
+ ids_needing_review = set(self._tool.bugs.queries.fetch_attachment_ids_from_review_queue(datetime.today() - timedelta(7)))
new_ids = ids_needing_review.difference(self._ids_sent_to_server)
_log.info("Feeding EWS (%s, %s new)" % (pluralize("r? patch", len(ids_needing_review)), len(new_ids)))
for attachment_id in new_ids: # Order doesn't really matter for the EWS.
