Colin Watson has proposed merging ~cjwatson/launchpad:charm-debian-importer into launchpad:master.
Commit message: charm: Add a launchpad-debian-importer charm Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/442049 -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:charm-debian-importer into launchpad:master.
diff --git a/charm/Makefile b/charm/Makefile index 6c4f214..a36139e 100644 --- a/charm/Makefile +++ b/charm/Makefile @@ -16,6 +16,7 @@ ASSET = ../build/$(BUILD_LABEL)/$(TARBALL) CHARMS := \ launchpad-admin \ launchpad-appserver \ + launchpad-debian-importer \ launchpad-librarian all: ## alias to build diff --git a/charm/launchpad-debian-importer/README.md b/charm/launchpad-debian-importer/README.md new file mode 100644 index 0000000..6ae99ca --- /dev/null +++ b/charm/launchpad-debian-importer/README.md @@ -0,0 +1,13 @@ +# Launchpad Debian importer + +This charm runs a job that imports Debian packages into Launchpad. + +You will need the following relations: + + juju relate launchpad-debian-importer:db postgresql:db + juju relate launchpad-debian-importer rabbitmq-server + +You will also need to set the `librarian_upload_host` and +`librarian_upload_port` options to point to a local +[launchpad-librarian](https://charmhub.io/launchpad-librarian) instance; +this is not currently handled using relations. diff --git a/charm/launchpad-debian-importer/charmcraft.yaml b/charm/launchpad-debian-importer/charmcraft.yaml new file mode 100644 index 0000000..1634d67 --- /dev/null +++ b/charm/launchpad-debian-importer/charmcraft.yaml @@ -0,0 +1,60 @@ +type: charm +bases: + - build-on: + - name: ubuntu + channel: "20.04" + architectures: [amd64] + run-on: + - name: ubuntu + channel: "20.04" + architectures: [amd64] +parts: + charm-wheels: + source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels + source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": charm-wheels/ + prime: + - "-charm-wheels" + ols-layers: + source: https://git.launchpad.net/ols-charm-deps + source-commit: "56d219f60a293a6c73759b8439ef5fdb11e19d1f" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": layers/ + stage: + - layers + prime: + - "-layers" + launchpad-layers: + after: + - ols-layers + source: https://git.launchpad.net/launchpad-layers + source-commit: "6a50917f5f6163069ae1661e3320abb5b48173a3" + source-submodules: [] + source-type: git + plugin: dump + organize: + launchpad-base: layers/layer/launchpad-base + stage: + - layers + prime: + - "-layers" + launchpad-debian-importer: + after: + - charm-wheels + - launchpad-layers + source: . + plugin: reactive + build-snaps: [charm/2.x/stable] + build-packages: [libpq-dev] + build-environment: + - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer + - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface + - PIP_NO_INDEX: "true" + - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels diff --git a/charm/launchpad-debian-importer/config.yaml b/charm/launchpad-debian-importer/config.yaml new file mode 100644 index 0000000..6b0c325 --- /dev/null +++ b/charm/launchpad-debian-importer/config.yaml @@ -0,0 +1,14 @@ +options: + active: + type: boolean + description: If true, enable jobs that may change the database. + default: true + debian_mirror_host: + type: string + description: Hostname of the Debian mirror to use. + default: "ftp.debian.org" + debian_suites: + type: string + description: YAML-encoded dict mapping Debian suites to components. + default: | + sid: ["main", "contrib", "non-free"] diff --git a/charm/launchpad-debian-importer/layer.yaml b/charm/launchpad-debian-importer/layer.yaml new file mode 100644 index 0000000..88e2ce1 --- /dev/null +++ b/charm/launchpad-debian-importer/layer.yaml @@ -0,0 +1,13 @@ +includes: + - layer:launchpad-base +repo: https://git.launchpad.net/launchpad +options: + apt: + packages: + - debian-archive-keyring + - debmirror + ols-pg: + databases: + db: + name: launchpad_dev + roles: gina diff --git a/charm/launchpad-debian-importer/metadata.yaml b/charm/launchpad-debian-importer/metadata.yaml new file mode 100644 index 0000000..ff28cae --- /dev/null +++ b/charm/launchpad-debian-importer/metadata.yaml @@ -0,0 +1,15 @@ +name: launchpad-debian-importer +display-name: launchpad-debian-importer +summary: Launchpad Debian importer +maintainer: Colin Watson <cjwat...@canonical.com> +description: | + Launchpad is an open source suite of tools that help people and teams + to work together on software projects. + + This charm runs a job that imports Debian packages into Launchpad. +tags: + # https://juju.is/docs/charm-metadata#heading--charm-store-fields + - network +series: + - focal +subordinate: false diff --git a/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py b/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py new file mode 100644 index 0000000..f87e95b --- /dev/null +++ b/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py @@ -0,0 +1,62 @@ +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +import os.path + +import yaml +from charmhelpers.core import hookenv, host, templating +from charms.launchpad.base import ( + configure_cron, + configure_lazr, + get_service_config, +) +from charms.reactive import set_state, when, when_not +from ols import base + + +def configure_logrotate(config): + hookenv.log("Writing logrotate configuration.") + templating.render( + "logrotate.conf.j2", + "/etc/logrotate.d/launchpad-debian-importer", + config, + perms=0o644, + ) + + +@when("launchpad.base.configured") +@when_not("service.configured") +def configure(): + config = get_service_config() + config["debian_suites"] = yaml.safe_load(config["debian_suites"]) + config["debian_components"] = [] + for suite, components in config["debian_suites"].items(): + for component in components: + if component not in config["debian_components"]: + config["debian_components"].append(component) + config["mirror_dir"] = os.path.join(base.base_dir(), "mirror") + host.mkdir( + config["mirror_dir"], owner=base.user(), group=base.user(), perms=0o755 + ) + config["scripts_dir"] = os.path.join(base.base_dir(), "scripts") + host.mkdir(config["scripts_dir"], perms=0o755) + templating.render( + "mirror-update.sh.j2", + os.path.join(config["scripts_dir"], "mirror-update.sh"), + config, + perms=0o755, + ) + configure_lazr( + config, + "launchpad-debian-importer-lazr.conf", + "launchpad-debian-importer/launchpad-lazr.conf", + ) + configure_logrotate(config) + configure_cron(config, "crontab.j2") + + set_state("service.configured") + + +@when("service.configured") +def check_is_running(): + hookenv.status_set("active", "Ready") diff --git a/charm/launchpad-debian-importer/templates/crontab.j2 b/charm/launchpad-debian-importer/templates/crontab.j2 new file mode 100644 index 0000000..2f9d7e2 --- /dev/null +++ b/charm/launchpad-debian-importer/templates/crontab.j2 @@ -0,0 +1,15 @@ +TZ=UTC +MAILFROM={{ bounce_address }} +MAILTO={{ cron_mailto }} +LPCONFIG=launchpad-debian-importer + +{% if active -%} +# Mirror from Debian and then run the importer. Debian publishes at "52 +# 1,7,13,19", but we also need to allow time for the above mirror to update. +25 4,10,16,22 * * * {{ scripts_dir }}/mirror-update.sh >> {{ logs_dir }}/mirror-update.log 2>&1 && {{ code_dir }}/scripts/gina.py {{ debian_suites.keys()|reverse|join(" ") }} -q --log-file=INFO:{{ logs_dir }}/gina.log + +{% endif -%} +# Catch up with publishing OOPSes that were temporarily spooled to disk due +# to RabbitMQ being unavailable. +*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_username }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_vhost }} --repo {{ oopses_dir }} --key "" + diff --git a/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf b/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf new file mode 100644 index 0000000..2152ea5 --- /dev/null +++ b/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf @@ -0,0 +1,20 @@ +# Public configuration data. The contents of this file may be freely shared +# with developers if needed for debugging. + +# A schema's sections, keys, and values are automatically inherited, except +# for '.optional' sections. Update this config to override key values. +# Values are strings, except for numbers that look like ints. The tokens +# true, false, and none are treated as True, False, and None. + +{% from "macros.j2" import opt -%} + +[meta] +extends: ../launchpad-base-lazr.conf + +{% for suite, components in debian_suites.items() -%} +[gina_target.{{ suite }}] +components: {{ components|join(",") }} +source_only: True +root: {{ mirror_dir }}/ + +{% endfor -%} diff --git a/charm/launchpad-debian-importer/templates/logrotate.conf.j2 b/charm/launchpad-debian-importer/templates/logrotate.conf.j2 new file mode 100644 index 0000000..51d1421 --- /dev/null +++ b/charm/launchpad-debian-importer/templates/logrotate.conf.j2 @@ -0,0 +1,11 @@ +{{ logs_dir }}/*.log +{ + rotate 21 + daily + dateext + delaycompress + compress + notifempty + missingok +} + diff --git a/charm/launchpad-debian-importer/templates/mirror-update.sh.j2 b/charm/launchpad-debian-importer/templates/mirror-update.sh.j2 new file mode 100644 index 0000000..446c3af --- /dev/null +++ b/charm/launchpad-debian-importer/templates/mirror-update.sh.j2 @@ -0,0 +1,24 @@ +#! /bin/sh +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +# Part of the launchpad-debian-importer Juju charm. + +set -e + +DEST="{{ mirror_dir }}/" +HOST="{{ debian_mirror_host }}" +DISTS="{{ debian_suites.keys()|join(',') }}" +SECTIONS="{{ debian_components|join(',') }}" + +{% if http_proxy -%} +http_proxy="{{ http_proxy }}" +ftp_proxy="{{ http_proxy }}" +export http_proxy ftp_proxy + +{% endif -%} +echo "Starting Mirror: $(date)" +rm -rf "${DEST}/.temp" +debmirror --keyring /usr/share/keyrings/debian-archive-keyring.gpg --root=debian --host="$HOST" --method=http --dist="$DISTS" --section="$SECTIONS" --arch=none --diff=none --progress --rsync-extra=none --allow-dist-rename "$DEST" +echo "Finished Mirror: $(date)" +
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp