[Launchpad-reviewers] [Merge] lp:~cjwatson/launchpad/git-permissions-check-api into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/git-permissions-check-api into lp:launchpad. Commit message: Add webservice APIs to allow checking a user's effective permissions on Git refs. Requested reviews: Launchpad code reviewers (launchpad-reviewers) Related bugs: Bug #1517559 in Launchpad itself: "git fine-grained permissions" https://bugs.launchpad.net/launchpad/+bug/1517559 For more details, see: https://code.launchpad.net/~cjwatson/launchpad/git-permissions-check-api/+merge/357842 I ended up pushing most of the guts of GitAPI._checkRefPermissions down to GitRepository, since we need almost all of the same logic except for looking up translated paths, handling authentication parameters, and the slightly different output encoding. -- Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/git-permissions-check-api into lp:launchpad. === modified file 'lib/lp/code/interfaces/gitref.py' --- lib/lp/code/interfaces/gitref.py 2018-10-22 13:10:21 + +++ lib/lp/code/interfaces/gitref.py 2018-10-25 19:12:27 + @@ -433,6 +433,19 @@ Other grants may apply via wildcard rules. """ +@operation_parameters( +person=Reference(title=_("Person to check"), schema=IPerson)) +@export_read_operation() +@operation_for_version("devel") +def checkPermissions(person): +"""Check a person's permissions on this reference. + +:param person: An `IPerson` to check. +:return: A list of zero or more of "create", "push", and +"force-push", indicating the requested person's effective +permissions on this reference. +""" + class IGitRef(IGitRefView, IGitRefEdit): """A reference in a Git repository.""" === modified file 'lib/lp/code/interfaces/gitrepository.py' --- lib/lp/code/interfaces/gitrepository.py 2018-10-23 16:15:37 + +++ lib/lp/code/interfaces/gitrepository.py 2018-10-25 19:12:27 + @@ -25,6 +25,7 @@ export_as_webservice_collection, export_as_webservice_entry, export_destructor_operation, +export_operation_as, export_read_operation, export_write_operation, exported, @@ -809,6 +810,35 @@ def setRules(rules, user): """Set the access rules for this repository.""" +def checkRefPermissions(person, ref_paths): +"""Check a person's permissions on some references in this repository. + +:param person: An `IPerson` to check, or +`GitGranteeType.REPOSITORY_OWNER` to check an anonymous +repository owner. +:param ref_paths: An iterable of reference paths. +:return: A dict mapping reference paths to sets of +`GitPermissionType`, corresponding to the requested person's +effective permissions on each of the requested references. +""" + +@operation_parameters( +person=Reference(title=_("Person to check"), schema=IPerson), +paths=List(title=_("Reference paths"), value_type=TextLine())) +@export_operation_as("checkRefPermissions") +@export_read_operation() +@operation_for_version("devel") +def api_checkRefPermissions(person, paths): +"""Check a person's permissions on some references in this repository. + +:param person: An `IPerson` to check. +:param paths: An iterable of reference paths. +:return: A dict mapping reference paths to lists of zero or more of +"create", "push", and "force-push", indicating the requested +person's effective permissions on each of the requested +references. +""" + @export_read_operation() @operation_for_version("devel") def canBeDeleted(): === modified file 'lib/lp/code/model/gitref.py' --- lib/lp/code/model/gitref.py 2018-10-22 12:43:55 + +++ lib/lp/code/model/gitref.py 2018-10-25 19:12:27 + @@ -65,6 +65,7 @@ IGitRef, IGitRefRemoteSet, ) +from lp.code.interfaces.gitrule import describe_git_permissions from lp.code.model.branchmergeproposal import ( BranchMergeProposal, BranchMergeProposalGetter, @@ -441,6 +442,12 @@ rule = self.repository.addRule(self.path, user) rule.setGrants(grants, user) +def checkPermissions(self, person): +"""See `IGitRef`.""" +return describe_git_permissions( +self.repository.checkRefPermissions( +person, [self.path])[self.path]) + @implementer(IGitRef) class GitRef(StormBase, GitRefMixin): @@ -823,6 +830,7 @@ return [] setGrants = _unimplemented +checkPermissions = _unimplemented def __eq__(self, other): return ( === modified file 'lib/lp/code/model/gitrepository.py' --- lib/lp/code/model/gitrepository.py 2018-10-22 16:24:46 + +++ lib/lp/code/model/gitrepository.py 2018-10-25 19:12:27 + @@ -9,9 +9,13 @@ 'parse_git_commits', ] -from collections import OrderedDict +from
[Launchpad-reviewers] [Merge] lp:~cjwatson/launchpad/git-ref-remote-lp-non-production into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/git-ref-remote-lp-non-production into lp:launchpad. Commit message: Allow fetching blobs from repositories hosted on git.launchpad.net from non-production Launchpad instances. Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad/git-ref-remote-lp-non-production/+merge/357827 This makes life easier when doing e.g. snap build QA on dogfood. -- Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/git-ref-remote-lp-non-production into lp:launchpad. === modified file 'lib/lp/code/model/gitref.py' --- lib/lp/code/model/gitref.py 2018-10-22 12:43:55 + +++ lib/lp/code/model/gitref.py 2018-10-25 15:18:55 + @@ -702,6 +702,25 @@ return response.content +def _fetch_blob_from_launchpad(repository_url, ref_path, filename): +repo_path = urlsplit(repository_url).path.strip("/") +try: +response = urlfetch( +"https://git.launchpad.net/%s/plain/%s; % ( +repo_path, quote(filename)), +params={"h": ref_path}) +except requests.RequestException as e: +if (e.response is not None and +e.response.status_code == requests.codes.NOT_FOUND): +raise GitRepositoryBlobNotFound( +repository_url, filename, rev=ref_path) +else: +raise GitRepositoryScanFault( +"Failed to get file from Git repository at %s: %s" % +(repository_url, str(e))) +return response.content + + @implementer(IGitRef) @provider(IGitRefRemoteSet) class GitRefRemote(GitRefMixin): @@ -800,17 +819,29 @@ # dispatch a build job or a code import or something like that to do # so. For now, we just special-case some providers where we know # how to fetch a blob on its own. -repository = getUtility(IGitLookup).getByUrl(self.repository_url) -if repository is not None: -# This is one of our own repositories. Doing this by URL seems -# gratuitously complex, but apparently we already have some -# examples of this on production. -return repository.getBlob(filename, rev=self.path) url = urlsplit(self.repository_url) if (url.hostname == "github.com" and len(url.path.strip("/").split("/")) == 2): return _fetch_blob_from_github( self.repository_url, self.path, filename) +if (url.hostname == "git.launchpad.net" and +config.vhost.mainsite.hostname != "launchpad.net"): +# Even if this isn't launchpad.net, we can still retrieve files +# from git.launchpad.net by URL, as a QA convenience. (We check +# config.vhost.mainsite.hostname rather than +# config.codehosting.git_*_root because the dogfood instance +# points git_*_root to git.launchpad.net but doesn't share the +# production database.) +return _fetch_blob_from_launchpad( +self.repository_url, self.path, filename) +codehosting_host = urlsplit(config.codehosting.git_anon_root).hostname +if url.hostname == codehosting_host: +repository = getUtility(IGitLookup).getByUrl(self.repository_url) +if repository is not None: +# This is one of our own repositories. Doing this by URL +# seems gratuitously complex, but apparently we already have +# some examples of this on production. +return repository.getBlob(filename, rev=self.path) raise GitRepositoryBlobUnsupportedRemote(self.repository_url) @property === modified file 'lib/lp/code/model/tests/test_gitref.py' --- lib/lp/code/model/tests/test_gitref.py 2018-10-16 15:29:37 + +++ lib/lp/code/model/tests/test_gitref.py 2018-10-25 15:18:55 + @@ -344,6 +344,51 @@ self.assertEqual(expected_calls, hosting_fixture.getBlob.calls) @responses.activate +def test_remote_launchpad_production_branch(self): +ref = self.factory.makeGitRefRemote( +repository_url="https://git.launchpad.net/~owner/+git/name;, +path="refs/heads/path") +responses.add( +"GET", +"https://git.launchpad.net/~owner/+git/name/plain/dir/file; +"?h=refs%2Fheads%2Fpath", +body=b"foo") +self.assertEqual(b"foo", ref.getBlob("dir/file")) + +@responses.activate +def test_remote_launchpad_production_HEAD(self): +ref = self.factory.makeGitRefRemote( +repository_url="https://git.launchpad.net/~owner/+git/name;, +path="HEAD") +responses.add( +"GET", +"https://git.launchpad.net/~owner/+git/name/plain/dir/file?h=HEAD;, +body=b"foo") +self.assertEqual(b"foo",
[Launchpad-reviewers] [Merge] lp:~cjwatson/launchpad/relocate-publish-copy-archives into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/relocate-publish-copy-archives into lp:launchpad. Commit message: Remove code tree path dependency from cron.publish-copy-archives. Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad/relocate-publish-copy-archives/+merge/357823 Squashes a cowboyed change on dogfood. -- Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/relocate-publish-copy-archives into lp:launchpad. === modified file 'cronscripts/publishing/cron.publish-copy-archives' --- cronscripts/publishing/cron.publish-copy-archives 2014-08-05 10:37:07 + +++ cronscripts/publishing/cron.publish-copy-archives 2018-10-25 14:52:46 + @@ -33,7 +33,7 @@ fi # Configuration options. -LAUNCHPADROOT=/srv/launchpad.net/production/launchpad +LAUNCHPADROOT="$(dirname "$0")/../.." LOCKFILE=/srv/launchpad.net/rebuild-test/cron.daily.lock DISTRONAME=ubuntu ___ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
Re: [Launchpad-reviewers] [Merge] lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd
@cjwatson, thanks for the clarification on the "resubmit". I'm not sure how you want the re-factoring on `raw_lxc_config` to look like, so maybe it's better if you just mold it to your liking. Thanks. Diff comments: > > === modified file 'lpbuildd/target/lxd.py' > --- lpbuildd/target/lxd.py2018-06-12 23:23:13 + > +++ lpbuildd/target/lxd.py2018-10-25 09:55:10 + > @@ -277,16 +277,30 @@ > old_profile.delete() > > raw_lxc_config = [ > -("lxc.aa_profile", "unconfined"), > ("lxc.cap.drop", ""), > ("lxc.cap.drop", "sys_time sys_module"), > ("lxc.cgroup.devices.deny", ""), > ("lxc.cgroup.devices.allow", ""), > ("lxc.mount.auto", ""), > ("lxc.mount.auto", "proc:rw sys:rw"), > -("lxc.network.0.ipv4", ipv4_address), > -("lxc.network.0.ipv4.gateway", self.ipv4_network.ip), > ] > + > +lxc_version = self._client.host_info["environment"]["driver_version"] > +major, minor = [int(v) for v in lxc_version.split(".")[0:2]] > + > +if major >= 3: > +raw_lxc_config.extend([ > +("lxc.apparmor.profile", "unconfined"), > +("lxc.net.0.ipv4.address", ipv4_address), > +("lxc.net.0.ipv4.gateway", self.ipv4_network.ip), > +]) > +else: > +raw_lxc_config.extend([ > +("lxc.aa_profile", "unconfined"), > +("lxc.network.0.ipv4", ipv4_address), > +("lxc.network.0.ipv4.gateway", self.ipv4_network.ip), > +]) > + I'm not entirely sure, how you want this to look like. Feel free to change it as you see fit. Thanks! > # Linux 4.4 on powerpc doesn't support all the seccomp bits that LXD > # needs. > if self.arch == "powerpc": -- https://code.launchpad.net/~tobijk/launchpad-buildd/launchpad-buildd-bionic/+merge/354331 Your team Launchpad code reviewers is requested to review the proposed merge of lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd. ___ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
Re: [Launchpad-reviewers] [Merge] lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd
(The "Resubmit" vote is something that your reviewer might use to indicate that the MP needs to be entirely withdrawn and resubmitted, perhaps against a different branch. It isn't something you do when you've made revisions in response to a review.) Regarding tests at package build time, oops; I've fixed that in trunk. Thanks for pointing it out. Diff comments: > > === modified file 'lpbuildd/target/lxd.py' > --- lpbuildd/target/lxd.py2018-06-12 23:23:13 + > +++ lpbuildd/target/lxd.py2018-10-25 07:43:01 + > @@ -277,16 +277,31 @@ > old_profile.delete() > > raw_lxc_config = [ > -("lxc.aa_profile", "unconfined"), > ("lxc.cap.drop", ""), > ("lxc.cap.drop", "sys_time sys_module"), > ("lxc.cgroup.devices.deny", ""), > ("lxc.cgroup.devices.allow", ""), > ("lxc.mount.auto", ""), > ("lxc.mount.auto", "proc:rw sys:rw"), > -("lxc.network.0.ipv4", ipv4_address), > -("lxc.network.0.ipv4.gateway", self.ipv4_network.ip), > ] > + > +lxc_version = self._client.host_info.get("environment", {}).get( > +"driver_version") While this was the first part of the train of thought in my previous review, I went on to say that I'd prefer just `lxc_version = self._client.host_info["environment"]["driver_version"]`, since all versions of LXD I can find have those keys. Do you have a reason for using the `.get()` form? > +major, minor = [int(v) for v in lxc_version.split(".")[0:2]] > + > +if major >= 3: > +raw_lxc_config.extend([ > +("lxc.apparmor.profile", "unconfined"), > +("lxc.net.0.ipv4.address", ipv4_address), > +("lxc.net.0.ipv4.gateway", self.ipv4_network.ip), > +]) > +else: > +raw_lxc_config.extend([ > +("lxc.aa_profile", "unconfined"), > +("lxc.network.0.ipv4", ipv4_address), > +("lxc.network.0.ipv4.gateway", self.ipv4_network.ip), > +]) > + Yes, though it's slightly more lines of code to do it this way and IMO less clear. (I would refactor this to use a dict after landing it if you don't do it first.) > # Linux 4.4 on powerpc doesn't support all the seccomp bits that LXD > # needs. > if self.arch == "powerpc": > > === modified file 'lpbuildd/target/tests/test_lxd.py' > --- lpbuildd/target/tests/test_lxd.py 2018-05-08 09:37:16 + > +++ lpbuildd/target/tests/test_lxd.py 2018-10-25 07:43:01 + > @@ -181,22 +203,40 @@ > "type": "nic", > }, > } > +if driver_version == "3.0": > +expected_devices["root"] = { > +"path": "/", > +"pool": "default", > +"type": "disk", > +} > client.profiles.create.assert_called_once_with( > "lpbuildd", expected_config, expected_devices) > > def test_create_profile_amd64(self): > -self.useFixture(MockPatch("pylxd.Client")) > -client = pylxd.Client() > -client.profiles.get.side_effect = FakeLXDAPIException > -LXD("1", "xenial", "amd64").create_profile() > -self.assert_correct_profile() > +for driver_version in ["2.0", "3.0"]: > +with MockPatch("pylxd.Client"): > +client = pylxd.Client() > +client.profiles.get.side_effect = FakeLXDAPIException > +client.host_info = { > +"environment": {"driver_version": driver_version} > +} > +LXD("1", "xenial", "amd64").create_profile() > +self.assert_correct_profile( > +driver_version=driver_version or "3.0") Ah, in that case, please use this approach instead (tested): with MockPatch("pylxd.Client"): for driver_version in ("2.0", "3.0"): client = pylxd.Client() client.reset_mock() ... > > def test_create_profile_powerpc(self): > -self.useFixture(MockPatch("pylxd.Client")) > -client = pylxd.Client() > -client.profiles.get.side_effect = FakeLXDAPIException > -LXD("1", "xenial", "powerpc").create_profile() > -self.assert_correct_profile("lxc.seccomp=\n") > +for driver_version in ["2.0", "3.0"]: > +with MockPatch("pylxd.Client"): > +client = pylxd.Client() > +client.profiles.get.side_effect = FakeLXDAPIException > +client.host_info = { > +"environment": {"driver_version": driver_version} > +} > +LXD("1", "xenial", "powerpc").create_profile() > +self.assert_correct_profile( > +extra_raw_lxc_config=[("lxc.seccomp", ""),], > +
[Launchpad-reviewers] [Merge] lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd
The proposal to merge lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd has been updated. Description changed to: This patch makes the buildd's LXD backend work on Bionic: * Based on LXD version modify raw.lxc settings (some parameter names have changed) * Fix a problem with header checks where uid and gid are set to 0, but need to be strings * Explicitly specify a root device, which seems to be mandatory now For more details, see: https://code.launchpad.net/~tobijk/launchpad-buildd/launchpad-buildd-bionic/+merge/354331 -- Your team Launchpad code reviewers is requested to review the proposed merge of lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd. ___ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
Re: [Launchpad-reviewers] [Merge] lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd
Is there a reason why only tests in lpbuildd/tests are run during package build? -- https://code.launchpad.net/~tobijk/launchpad-buildd/launchpad-buildd-bionic/+merge/354331 Your team Launchpad code reviewers is requested to review the proposed merge of lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd. ___ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
Re: [Launchpad-reviewers] [Merge] lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd
Review: Resubmit @cjwatson, ready for re-review. Diff comments: > > === modified file 'lpbuildd/target/lxd.py' > --- lpbuildd/target/lxd.py2018-06-12 23:23:13 + > +++ lpbuildd/target/lxd.py2018-10-25 07:43:01 + > @@ -277,16 +277,31 @@ > old_profile.delete() > > raw_lxc_config = [ > -("lxc.aa_profile", "unconfined"), > ("lxc.cap.drop", ""), > ("lxc.cap.drop", "sys_time sys_module"), > ("lxc.cgroup.devices.deny", ""), > ("lxc.cgroup.devices.allow", ""), > ("lxc.mount.auto", ""), > ("lxc.mount.auto", "proc:rw sys:rw"), > -("lxc.network.0.ipv4", ipv4_address), > -("lxc.network.0.ipv4.gateway", self.ipv4_network.ip), > ] > + > +lxc_version = self._client.host_info.get("environment", {}).get( > +"driver_version") > +major, minor = [int(v) for v in lxc_version.split(".")[0:2]] > + > +if major >= 3: > +raw_lxc_config.extend([ > +("lxc.apparmor.profile", "unconfined"), > +("lxc.net.0.ipv4.address", ipv4_address), > +("lxc.net.0.ipv4.gateway", self.ipv4_network.ip), > +]) > +else: > +raw_lxc_config.extend([ > +("lxc.aa_profile", "unconfined"), > +("lxc.network.0.ipv4", ipv4_address), > +("lxc.network.0.ipv4.gateway", self.ipv4_network.ip), > +]) > + The suggestion was to use a dict and sort later. I suppose sorting the list is going to do the same. > # Linux 4.4 on powerpc doesn't support all the seccomp bits that LXD > # needs. > if self.arch == "powerpc": > > === modified file 'lpbuildd/target/tests/test_lxd.py' > --- lpbuildd/target/tests/test_lxd.py 2018-05-08 09:37:16 + > +++ lpbuildd/target/tests/test_lxd.py 2018-10-25 07:43:01 + > @@ -181,22 +203,40 @@ > "type": "nic", > }, > } > +if driver_version == "3.0": > +expected_devices["root"] = { > +"path": "/", > +"pool": "default", > +"type": "disk", > +} > client.profiles.create.assert_called_once_with( > "lpbuildd", expected_config, expected_devices) > > def test_create_profile_amd64(self): > -self.useFixture(MockPatch("pylxd.Client")) > -client = pylxd.Client() > -client.profiles.get.side_effect = FakeLXDAPIException > -LXD("1", "xenial", "amd64").create_profile() > -self.assert_correct_profile() > +for driver_version in ["2.0", "3.0"]: > +with MockPatch("pylxd.Client"): > +client = pylxd.Client() > +client.profiles.get.side_effect = FakeLXDAPIException > +client.host_info = { > +"environment": {"driver_version": driver_version} > +} > +LXD("1", "xenial", "amd64").create_profile() > +self.assert_correct_profile( > +driver_version=driver_version or "3.0") The suggestion was to take the fixture out of the loop, but it needs to be reset on each iteration, else checks such as client.profiles.get.assert_called_once_with("lpbuildd") will break. > > def test_create_profile_powerpc(self): > -self.useFixture(MockPatch("pylxd.Client")) > -client = pylxd.Client() > -client.profiles.get.side_effect = FakeLXDAPIException > -LXD("1", "xenial", "powerpc").create_profile() > -self.assert_correct_profile("lxc.seccomp=\n") > +for driver_version in ["2.0", "3.0"]: > +with MockPatch("pylxd.Client"): > +client = pylxd.Client() > +client.profiles.get.side_effect = FakeLXDAPIException > +client.host_info = { > +"environment": {"driver_version": driver_version} > +} > +LXD("1", "xenial", "powerpc").create_profile() > +self.assert_correct_profile( > +extra_raw_lxc_config=[("lxc.seccomp", ""),], > +driver_version=driver_version or "3.0" > +) > > def test_start(self): > fs_fixture = self.useFixture(FakeFilesystem()) -- https://code.launchpad.net/~tobijk/launchpad-buildd/launchpad-buildd-bionic/+merge/354331 Your team Launchpad code reviewers is requested to review the proposed merge of lp:~tobijk/launchpad-buildd/launchpad-buildd-bionic into lp:launchpad-buildd. ___ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp