Colin Watson has proposed merging lp:~cjwatson/launchpad/mirror-prober-redirects into lp:launchpad.
Commit message: Allow archive mirrors to return redirect responses. Requested reviews: Launchpad code reviewers (launchpad-reviewers) Related bugs: Bug #1836712 in Launchpad itself: "Launchpad mirror prober considers http redirects broken" https://bugs.launchpad.net/launchpad/+bug/1836712 For more details, see: https://code.launchpad.net/~cjwatson/launchpad/mirror-prober-redirects/+merge/370493 APT used to not support these, but this was changed over ten years ago. -- Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/mirror-prober-redirects into lp:launchpad.
=== modified file 'database/sampledata/current-dev.sql' --- database/sampledata/current-dev.sql 2019-05-22 14:57:45 +0000 +++ database/sampledata/current-dev.sql 2019-07-23 14:58:40 +0000 @@ -3443,6 +3443,7 @@ INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (8, 1, 'canonical-archive', 'http://archive.ubuntu.com/ubuntu/', NULL, NULL, NULL, NULL, 1, 70, 225, 1, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false); INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (9, 1, 'canonical-releases', 'http://releases.ubuntu.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false); INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (10, 1, 'random-releases-mirror', 'http://releases.random.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 10, NULL, NULL, false); +INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (11, 1, 'archive-redirect-mirror', 'http://localhost:11375/redirect-to-valid-mirror/', NULL, NULL, NULL, NULL, 1, 10, 9, 1, true, true, '2019-07-23 13:16:51.953354', NULL, 30, NULL, NULL, false); ALTER TABLE distributionmirror ENABLE TRIGGER ALL; === modified file 'database/sampledata/current.sql' --- database/sampledata/current.sql 2019-05-22 14:57:45 +0000 +++ database/sampledata/current.sql 2019-07-23 14:58:40 +0000 @@ -1,4 +1,4 @@ --- Copyright 2010-2017 Canonical Ltd. This software is licensed under the +-- Copyright 2010-2019 Canonical Ltd. This software is licensed under the -- GNU Affero General Public License version 3 (see the file LICENSE). -- Created using pg_dump (PostgreSQL) 9.3.5 @@ -3380,6 +3380,7 @@ INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (8, 1, 'canonical-archive', 'http://archive.ubuntu.com/ubuntu/', NULL, NULL, NULL, NULL, 1, 70, 225, 1, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false); INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (9, 1, 'canonical-releases', 'http://releases.ubuntu.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false); INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (10, 1, 'random-releases-mirror', 'http://releases.random.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 10, NULL, NULL, false); +INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (11, 1, 'archive-redirect-mirror', 'http://localhost:11375/redirect-to-valid-mirror/', NULL, NULL, NULL, NULL, 1, 10, 9, 1, true, true, '2019-07-23 13:16:51.953354', NULL, 30, NULL, NULL, false); ALTER TABLE distributionmirror ENABLE TRIGGER ALL; === modified file 'lib/lp/registry/doc/distribution-mirror.txt' --- lib/lp/registry/doc/distribution-mirror.txt 2019-05-22 14:57:45 +0000 +++ lib/lp/registry/doc/distribution-mirror.txt 2019-07-23 14:58:40 +0000 @@ -122,7 +122,8 @@ [(u'canonical-archive', '100 Mbps', 'OFFICIAL'), (u'archive-404-mirror', '512 Kbps', 'OFFICIAL'), (u'archive-mirror', '128 Kbps', 'OFFICIAL'), - (u'archive-mirror2', '128 Kbps', 'OFFICIAL')] + (u'archive-mirror2', '128 Kbps', 'OFFICIAL'), + (u'archive-redirect-mirror', '128 Kbps', 'OFFICIAL')] >>> [(mirror.name, mirror.speed.title, mirror.status.name) ... for mirror in ubuntu.cdimage_mirrors] @@ -242,7 +243,7 @@ >>> [mirror.name ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.ARCHIVE)] [u'canonical-archive', u'archive-404-mirror', u'archive-mirror', - u'archive-mirror2'] + u'archive-mirror2', u'archive-redirect-mirror'] >>> [mirror.name ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.RELEASE)] @@ -263,7 +264,8 @@ >>> [mirror.name ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.ARCHIVE)] - [u'canonical-archive', u'archive-404-mirror', u'archive-mirror2'] + [u'canonical-archive', u'archive-404-mirror', u'archive-mirror2', + u'archive-redirect-mirror'] >>> [mirror.name ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.RELEASE)] @@ -278,7 +280,7 @@ ... MirrorContent.ARCHIVE, ignore_last_probe=True) >>> [mirror.name for mirror in mirrors] [u'canonical-archive', u'archive-404-mirror', u'archive-mirror', - u'archive-mirror2'] + u'archive-mirror2', u'archive-redirect-mirror'] It's also possible to limit the number of mirrors we want to probe by passing an extra limit= argument to mirrorset.getMirrorsToProbe(). If @@ -306,9 +308,10 @@ archive-404-mirror NEVER archive-mirror TODAY archive-mirror2 2006-05-24... + archive-redirect-mirror NEVER >>> mirrors = mirrorset.getMirrorsToProbe( - ... MirrorContent.ARCHIVE, ignore_last_probe=True, limit=3) + ... MirrorContent.ARCHIVE, ignore_last_probe=True, limit=4) >>> for mirror in mirrors: ... last_probe = mirror.last_probe_record ... last_probe_date = 'NEVER' @@ -318,6 +321,7 @@ canonical-archive NEVER archive-404-mirror NEVER archive-mirror2 2006-05-24... + archive-redirect-mirror NEVER If when we finish probing a mirror, that mirror doesn't have any @@ -400,6 +404,7 @@ getExpectedPackagesPaths() and getExpectedSourcesPaths() methods of IDistributionMirror. + >>> mirror = mirrorset.getByName('archive-mirror2') >>> paths = mirror.getExpectedPackagesPaths() >>> [path for (series, pocket, component, path) in paths] [u'dists/breezy-autotest/main/binary-i386/Packages.gz', @@ -646,7 +651,7 @@ >>> print stderr INFO Creating lockfile: /var/lock/launchpad-distributionmirror-prober.lock INFO Probing Archive Mirrors - INFO Probed 4 mirrors. + INFO Probed 5 mirrors. INFO Disabling 1 mirror(s): http://launchpad.test/ubuntu/+mirror/archive-404-mirror INFO Re-enabling 1 mirror(s): === modified file 'lib/lp/registry/scripts/distributionmirror_prober.py' --- lib/lp/registry/scripts/distributionmirror_prober.py 2018-07-13 12:48:19 +0000 +++ lib/lp/registry/scripts/distributionmirror_prober.py 2019-07-23 14:58:40 +0000 @@ -1,4 +1,4 @@ -# Copyright 2009-2018 Canonical Ltd. This software is licensed under the +# Copyright 2009-2019 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). __metaclass__ = type @@ -688,7 +688,9 @@ callbacks = ArchiveMirrorProberCallbacks( mirror, series, pocket, component, url, logfile) unchecked_keys.append(url) - prober = ProberFactory(url) + # APT has supported redirects since 0.7.21 (2009-04-14), so allow + # them here too. + prober = RedirectAwareProberFactory(url) deferred = request_manager.run(prober.request_host, prober.probe) deferred.addCallbacks( === modified file 'lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt' --- lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt 2019-05-22 14:57:45 +0000 +++ lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt 2019-07-23 14:58:40 +0000 @@ -42,8 +42,10 @@ >>> print browser.title Mirrors :... >>> print_mirrors_by_countries(browser.contents) - Antarctica: [(u'Archive-mirror2', u'http', u'128 Kbps', - u'Six hours behind')] + Antarctica: + [(u'Archive-mirror2', u'http', u'128 Kbps', u'Six hours behind'), + (u'Archive-redirect-mirror', u'http', u'128 Kbps', + u'Last update unknown')] France: [(u'Archive-404-mirror', u'http', u'512 Kbps', u'Last update unknown'), (u'Archive-mirror', u'http', u'128 Kbps', u'Last update unknown')]
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
