Colin Watson has proposed merging ~cjwatson/lp-codeimport:charm-codeimport-storage into lp-codeimport:master with ~cjwatson/lp-codeimport:charmcraft as a prerequisite.
Commit message: charm: Add an lp-codeimport-storage charm Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/lp-codeimport/+git/lp-codeimport/+merge/439271 This can be used to provide import data storage instead of the older approach of setting `bazaar_branch_store` and `foreign_tree_store` on `lp-codeimport`. It isn't much more than a vanilla system with SSH key authorization and a couple of pre-created directories, but the new `codeimport-storage` relation makes it a little easier to set things up in a Mojo spec. Data migration from any previous storage systems needs to be handled manually. -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-codeimport:charm-codeimport-storage into lp-codeimport:master.
diff --git a/charm/lp-codeimport-storage/README.md b/charm/lp-codeimport-storage/README.md new file mode 100644 index 0000000..cad1a3b --- /dev/null +++ b/charm/lp-codeimport-storage/README.md @@ -0,0 +1,3 @@ +# Launchpad code import storage + +This charm provides storage for the `lp-codeimport` charm. diff --git a/charm/lp-codeimport-storage/charmcraft.yaml b/charm/lp-codeimport-storage/charmcraft.yaml new file mode 100644 index 0000000..ac76470 --- /dev/null +++ b/charm/lp-codeimport-storage/charmcraft.yaml @@ -0,0 +1,44 @@ +type: charm +bases: + - build-on: + - name: ubuntu + channel: "22.04" + architectures: [amd64] + run-on: + - name: ubuntu + channel: "22.04" + architectures: [amd64] +parts: + charm-wheels: + source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels + source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": charm-wheels/ + prime: + - "-charm-wheels" + ols-layers: + source: https://git.launchpad.net/ols-charm-deps + source-commit: "1ca8acbef7eb49b8a2cc81e5e13479b4f226a48b" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": layers/ + stage: + - layers + prime: + - "-layers" + lp-codeimport-storage: + after: + - charm-wheels + - ols-layers + source: . + plugin: reactive + build-snaps: [charm/2.x/stable] + build-environment: + - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer + - PIP_NO_INDEX: "true" + - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels diff --git a/charm/lp-codeimport-storage/config.yaml b/charm/lp-codeimport-storage/config.yaml new file mode 100644 index 0000000..4f9deac --- /dev/null +++ b/charm/lp-codeimport-storage/config.yaml @@ -0,0 +1,5 @@ +options: + public_ssh_key: + type: string + default: "" + description: Base64-encoded public SSH key of the code import workers. diff --git a/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py diff --git a/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py new file mode 100644 index 0000000..c6b06a0 --- /dev/null +++ b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py @@ -0,0 +1,39 @@ +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from charmhelpers.core import hookenv +from charms.reactive import ( + Endpoint, + remove_state, + set_state, + when, + when_not, + ) + + +class CodeImportStorageProvides(Endpoint): + @when("endpoint.{endpoint_name}.joined") + @when_not("endpoint.{endpoint_name}.available") + def joined(self): + for relation in self.relations: + ingress_address = hookenv.network_get( + relation.endpoint_name, relation_id=relation.relation_id + )["ingress-addresses"][0] + relation.to_publish[ + "bazaar_branch_store" + ] = f"sftp://importd@{ingress_address}/srv/importd/www/" + relation.to_publish[ + "foreign_tree_store" + ] = f"sftp://importd@{ingress_address}/srv/importd/sources/" + set_state(self.expand_name("endpoint.{endpoint_name}.available")) + + @when("endpoint.{endpoint_name}.available") + @when_not("endpoint.{endpoint_name}.joined") + def departed(self): + remove_state(self.expand_name("endpoint.{endpoint_name}.available")) + + def get_codeimport_subnets(self): + subnets = set() + for unit in self.all_joined_units: + subnets.update(unit.received_raw["egress-subnets"].split(",")) + return sorted(subnets) diff --git a/charm/lp-codeimport-storage/layer.yaml b/charm/lp-codeimport-storage/layer.yaml new file mode 100644 index 0000000..590c59f --- /dev/null +++ b/charm/lp-codeimport-storage/layer.yaml @@ -0,0 +1,2 @@ +includes: + - layer:basic diff --git a/charm/lp-codeimport-storage/metadata.yaml b/charm/lp-codeimport-storage/metadata.yaml new file mode 100644 index 0000000..95de7f8 --- /dev/null +++ b/charm/lp-codeimport-storage/metadata.yaml @@ -0,0 +1,14 @@ +name: lp-codeimport-storage +display-name: lp-codeimport-storage +summary: Launchpad code import storage +maintainer: Colin Watson <cjwat...@canonical.com> +description: Storage for use with lp-codeimport workers. +tags: + # https://juju.is/docs/charm-metadata#heading--charm-store-fields + - network +series: + - jammy +subordinate: false +provides: + codeimport-storage: + interface: codeimport-storage diff --git a/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py b/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py new file mode 100644 index 0000000..5be4349 --- /dev/null +++ b/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py @@ -0,0 +1,74 @@ +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +import base64 +import os.path + +from charmhelpers.core import ( + hookenv, + host, + templating, + ) +from charms.reactive import ( + endpoint_from_flag, + hook, + remove_state, + set_state, + when, + when_not, + ) + + +@hook("upgrade-charm") +def upgrade_charm(): + remove_state("service.configured") + + +@hook("config.changed") +def config_changed(): + remove_state("service.configured") + + +@hook( + "{provides:codeimport-storage}-relation-{joined,changed,broken,departed}" +) +def codeimport_storage_changed(*args): + remove_state("service.configured") + + +@when("config.set.public_ssh_key", "endpoint.codeimport-storage.available") +@when_not("service.configured") +def configure(): + codeimport_storage = endpoint_from_flag( + "endpoint.codeimport-storage.available" + ) + hookenv.log("Creating service user") + host.add_group("importd") + host.adduser("importd", primary_group="importd") + for directory in ("/srv/importd/sources", "/srv/importd/www"): + if not os.path.exists(directory): + host.mkdir( + directory, owner="importd", group="importd", perms=0o755 + ) + ssh_dir = "/home/importd/.ssh" + if not os.path.exists(ssh_dir): + host.mkdir(ssh_dir, owner="importd", group="importd", perms=0o700) + config = dict(hookenv.config()) + config["codeimport_subnets"] = codeimport_storage.get_codeimport_subnets() + config["public_ssh_key"] = base64.b64decode( + config["public_ssh_key"].encode("ASCII") + ).decode("ASCII") + templating.render( + "authorized_keys.j2", + os.path.join(ssh_dir, "authorized_keys"), + config, + owner="importd", + group="importd", + perms=0o600, + ) + set_state("service.configured") + + +@when("service.configured") +def check_is_running(): + hookenv.status_set("active", "Ready") diff --git a/charm/lp-codeimport-storage/templates/authorized_keys.j2 b/charm/lp-codeimport-storage/templates/authorized_keys.j2 new file mode 100644 index 0000000..77d398e --- /dev/null +++ b/charm/lp-codeimport-storage/templates/authorized_keys.j2 @@ -0,0 +1,2 @@ +restrict,from="{{ codeimport_subnets|join(",") }}" {{ public_ssh_key }} + diff --git a/charm/lp-codeimport/config.yaml b/charm/lp-codeimport/config.yaml index cac88ce..37f990c 100644 --- a/charm/lp-codeimport/config.yaml +++ b/charm/lp-codeimport/config.yaml @@ -11,13 +11,16 @@ options: default is to use the global CA infrastructure. bazaar_branch_store: type: string - default: sftp://hoo...@code-import-storage.launchpad.test/srv/importd/www/ - description: Where the Bazaar imports are stored. + default: "" + description: > + Where the Bazaar imports are stored. If unset, rely on the + codeimport-storage relation instead. foreign_tree_store: type: string - default: sftp://hoo...@code-import-storage.launchpad.test/srv/importd/sources/ + default: "" description: > - Where the tarballs of foreign branches are uploaded for storage. + Where the tarballs of foreign branches are uploaded for storage. If + unset, rely on the codeimport-storage relation instead. private_ssh_key: type: string default: "" diff --git a/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py b/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py diff --git a/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py b/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py new file mode 100644 index 0000000..16e31d4 --- /dev/null +++ b/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py @@ -0,0 +1,32 @@ +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from charms.reactive import ( + clear_flag, + Endpoint, + toggle_flag, + when, + when_not, + ) + + +class CodeImportStorageRequires(Endpoint): + @when("endpoint.{endpoint_name}.changed") + def joined(self): + toggle_flag( + self.expand_name("endpoint.{endpoint_name}.available"), + bool(self.bazaar_branch_store) and bool(self.foreign_tree_store), + ) + + @when("endpoint.{endpoint_name}.available") + @when_not("endpoint.{endpoint_name}.joined") + def departed(self): + clear_flag(self.expand_name("endpoint.{endpoint_name}.available")) + + @property + def bazaar_branch_store(self): + return self.all_joined_units.received["bazaar_branch_store"] + + @property + def foreign_tree_store(self): + return self.all_joined_units.received["foreign_tree_store"] diff --git a/charm/lp-codeimport/metadata.yaml b/charm/lp-codeimport/metadata.yaml index 6edc5b9..9df966d 100644 --- a/charm/lp-codeimport/metadata.yaml +++ b/charm/lp-codeimport/metadata.yaml @@ -9,6 +9,9 @@ tags: series: - bionic subordinate: false +requires: + codeimport-storage: + interface: codeimport-storage resources: lp-codeimport: type: file diff --git a/charm/lp-codeimport/reactive/lp-codeimport.py b/charm/lp-codeimport/reactive/lp-codeimport.py index 04df2a6..9c7c9e7 100644 --- a/charm/lp-codeimport/reactive/lp-codeimport.py +++ b/charm/lp-codeimport/reactive/lp-codeimport.py @@ -19,9 +19,12 @@ from charmhelpers.core import ( templating, ) from charms.reactive import ( + endpoint_from_flag, remove_state, set_state, when, + when_any, + when_none, when_not, ) from ols import base @@ -166,9 +169,51 @@ def configure_rsync(config): raise RuntimeError('Failed to restart rsync') -@when('ols.configured') +@when_any( + "endpoint.codeimport-storage.available", + "config.set.bazaar_branch_store", +) +def bazaar_branch_store_available(): + set_state("service.bazaar_branch_store.available") + + +@when_none( + "endpoint.codeimport-storage.available", + "config.set.bazaar_branch_store", +) +def bazaar_branch_store_unavailable(): + remove_state("service.bazaar_branch_store.available") + remove_state("service.configured") + + +@when_any( + "endpoint.codeimport-storage.available", + "config.set.foreign_tree_store", +) +def foreign_tree_store_available(): + set_state("service.foreign_tree_store.available") + + +@when_none( + "endpoint.codeimport-storage.available", + "config.set.foreign_tree_store", +) +def foreign_tree_store_unavailable(): + remove_state("service.foreign_tree_store.available") + remove_state("service.configured") + + +@when( + "ols.configured", + "service.bazaar_branch_store.available", + "service.foreign_tree_store.available", +) @when_not('service.configured') def configure(): + codeimport_storage = endpoint_from_flag( + "endpoint.codeimport-storage.available" + ) + ensure_lp_directories() system_packages = os.path.join(base.code_dir(), 'system-packages.txt') @@ -188,6 +233,14 @@ def configure(): config_path = os.path.join( base.code_dir(), 'production-configs', 'charm', 'codeimport-lazr.conf') svc_config = dict(config) + if not svc_config["bazaar_branch_store"]: + svc_config["bazaar_branch_store"] = ( + codeimport_storage.bazaar_branch_store + ) + if not svc_config["foreign_tree_store"]: + svc_config["foreign_tree_store"] = ( + codeimport_storage.foreign_tree_store + ) svc_config.update({ 'base_dir': base.base_dir(), 'code_dir': base.code_dir(), @@ -199,7 +252,7 @@ def configure(): 'home_dir': home_dir(), 'user': base.user(), 'code_import_storage_host': ( - urlparse(config['bazaar_branch_store']).hostname), + urlparse(svc_config['bazaar_branch_store']).hostname), # Chosen to allow distributing dispatch start time over a 30-second # interval. 'dispatch_offset': host.modulo_distribution(modulo=6, wait=5),
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp