Colin Watson has proposed merging ~cjwatson/launchpad:charm-librarian into launchpad:master.
Commit message: charm: Add a launchpad-librarian charm Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/439924 This also needs https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/439909 in order for the `librarian-gc` cron job to work, although the merge proposals can land in either order. -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:charm-librarian into launchpad:master.
diff --git a/charm/launchpad-librarian/README.md b/charm/launchpad-librarian/README.md new file mode 100644 index 0000000..4a7f5c0 --- /dev/null +++ b/charm/launchpad-librarian/README.md @@ -0,0 +1,25 @@ +# Launchpad librarian + +This charm runs a Launchpad librarian. + +You will need the following relations: + + juju relate launchpad-librarian:db postgresql:db + juju relate launchpad-librarian:session-db postgresql:db + juju relate launchpad-librarian rabbitmq-server + +The librarian listens on four ports. By default, these are: + +- Public download: 8000 +- Public upload: 9090 +- Restricted download: 8005 +- Restricted upload: 9095 + +The restricted ports allow access to restricted files without +authentication; firewall rules should ensure that they are only accessible +by other parts of Launchpad. + +You will normally want to mount a persistent volume on +`/srv/launchpad/librarian/`. (Even when writing uploads to Swift, this is +currently used as a temporary spool; it is therefore not currently valid to +deploy more than one unit of this charm.) diff --git a/charm/launchpad-librarian/charmcraft.yaml b/charm/launchpad-librarian/charmcraft.yaml new file mode 100644 index 0000000..0556d43 --- /dev/null +++ b/charm/launchpad-librarian/charmcraft.yaml @@ -0,0 +1,60 @@ +type: charm +bases: + - build-on: + - name: ubuntu + channel: "20.04" + architectures: [amd64] + run-on: + - name: ubuntu + channel: "20.04" + architectures: [amd64] +parts: + charm-wheels: + source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels + source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": charm-wheels/ + prime: + - "-charm-wheels" + ols-layers: + source: https://git.launchpad.net/ols-charm-deps + source-commit: "56d219f60a293a6c73759b8439ef5fdb11e19d1f" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": layers/ + stage: + - layers + prime: + - "-layers" + launchpad-layers: + after: + - ols-layers + source: https://git.launchpad.net/launchpad-layers + source-commit: "1920a6f823d8d882a99662bdda55f67f37359850" + source-submodules: [] + source-type: git + plugin: dump + organize: + launchpad-base: layers/layer/launchpad-base + stage: + - layers + prime: + - "-layers" + launchpad-librarian: + after: + - charm-wheels + - launchpad-layers + source: . + plugin: reactive + build-snaps: [charm/2.x/stable] + build-packages: [libpq-dev] + build-environment: + - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer + - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface + - PIP_NO_INDEX: "true" + - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels diff --git a/charm/launchpad-librarian/config.yaml b/charm/launchpad-librarian/config.yaml new file mode 100644 index 0000000..36755aa --- /dev/null +++ b/charm/launchpad-librarian/config.yaml @@ -0,0 +1,106 @@ +options: + active: + type: boolean + default: true + description: If true, enable jobs that may change the database. + nagios_path: + type: string + default: / + description: Path to a file on this librarian to use for Nagios checks. + nagios_expected_regex: + type: string + default: "Launchpad Librarian" + description: > + A regular expression that the response to `nagios_path` is expected to + match. + old_os_auth_url: + type: string + description: > + OpenStack authentication URL for a previous Swift instance that we're + migrating away from, but should still read from if necessary. + default: + old_os_auth_version: + type: string + description: > + OpenStack authentication protocol version for a previous Swift + instance that we're migrating away from, but should still read from if + necessary. + default: "2.0" + old_os_password: + type: string + description: > + OpenStack password for a previous Swift instance that we're migrating + away from, but should still read from if necessary. + default: + old_os_tenant_name: + type: string + description: > + OpenStack tenant name for a previous Swift instance that we're + migrating away from, but should still read from if necessary. + default: + old_os_username: + type: string + description: > + OpenStack username for a previous Swift instance that we're migrating + away from, but should still read from if necessary. + default: + os_auth_url: + type: string + description: OpenStack authentication URL. + default: + os_auth_version: + type: string + description: OpenStack authentication protocol version. + default: "2.0" + os_password: + type: string + description: OpenStack password. + default: + os_tenant_name: + type: string + description: OpenStack tenant name. + default: + os_username: + type: string + description: OpenStack username. + default: + port_download_base: + type: int + description: Base port number for public download workers. + default: 8000 + port_restricted_download_base: + type: int + description: Base port number for restricted download workers. + default: 8005 + port_restricted_upload_base: + type: int + description: Base port number for restricted upload workers. + default: 9095 + port_upload_base: + type: int + description: Base port number for public upload workers. + default: 9090 + swift_feed_workers: + type: int + description: Number of librarian-feed-swift workers to run in parallel. + default: 1 + swift_timeout: + type: int + description: Time in seconds to wait for a response from Swift. + default: 15 + upstream_host: + type: string + description: Host name for the upstream librarian, if any. + default: + upstream_port: + type: int + description: Port for the upstream librarian, if any. + default: 80 + workers: + type: int + description: > + Number of librarian worker processes. If set, each worker will listen + on consecutive ports starting from each of `port_download_base`, + `port_restricted_download_base`, `port_restricted_upload_base`, and + `port_upload_base`, so make sure there is enough space between those. + default: 1 diff --git a/charm/launchpad-librarian/layer.yaml b/charm/launchpad-librarian/layer.yaml new file mode 100644 index 0000000..12f6b73 --- /dev/null +++ b/charm/launchpad-librarian/layer.yaml @@ -0,0 +1,19 @@ +includes: + - layer:launchpad-base +repo: https://git.launchpad.net/launchpad +options: + ols-pg: + apt: + packages: + - run-one + databases: + db: + name: launchpad_dev + roles: + - binaryfile-expire + - librarian + - librarianfeedswift + - librariangc + session-db: + name: session_dev + roles: session diff --git a/charm/launchpad-librarian/metadata.yaml b/charm/launchpad-librarian/metadata.yaml new file mode 100644 index 0000000..da12af8 --- /dev/null +++ b/charm/launchpad-librarian/metadata.yaml @@ -0,0 +1,18 @@ +name: launchpad-librarian +display-name: launchpad-librarian +summary: Launchpad librarian +maintainer: Colin Watson <cjwat...@canonical.com> +description: | + Launchpad is an open source suite of tools that help people and teams + to work together on software projects. + + This charm runs a Launchpad librarian. +tags: + # https://juju.is/docs/charm-metadata#heading--charm-store-fields + - network +series: + - focal +subordinate: false +requires: + session-db: + interface: pgsql diff --git a/charm/launchpad-librarian/reactive/launchpad-librarian.py b/charm/launchpad-librarian/reactive/launchpad-librarian.py new file mode 100644 index 0000000..e8d5adc --- /dev/null +++ b/charm/launchpad-librarian/reactive/launchpad-librarian.py @@ -0,0 +1,194 @@ +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +import os.path +import shlex +import subprocess + +from charmhelpers.core import hookenv, host, templating +from charms.launchpad.base import ( + config_file_path, + configure_cron, + configure_lazr, + get_service_config, + lazr_config_files, + strip_dsn_authentication, + update_pgpass, +) +from charms.reactive import ( + clear_flag, + endpoint_from_flag, + helpers, + hook, + set_flag, + set_state, + when, + when_any, + when_not, +) +from ols import base, postgres +from psycopg2.extensions import parse_dsn + + +def configure_systemd(config): + hookenv.log("Writing systemd service.") + templating.render( + "launchpad-librarian.service.j2", + "/lib/systemd/system/launchpad-librarian.service", + config, + ) + templating.render( + "launchpad-librarian@.service.j2", + "/lib/systemd/system/launchpad-librarian@.service", + config, + ) + subprocess.run(["systemctl", "daemon-reload"]) + + +def configure_logrotate(config): + hookenv.log("Writing logrotate configuration.") + templating.render( + "logrotate.conf.j2", + "/etc/logrotate.d/launchpad-librarian", + config, + perms=0o644, + ) + + +def config_files(): + files = [] + files.extend(lazr_config_files()) + files.append(config_file_path("launchpad-librarian/launchpad-lazr.conf")) + files.append( + config_file_path("launchpad-librarian-secrets-lazr.conf", secret=True) + ) + return files + + +@when( + "config.set.port_download_base", + "config.set.port_restricted_download_base", + "config.set.port_restricted_upload_base", + "config.set.port_upload_base", + "launchpad.base.configured", + "session-db.master.available", +) +@when_not("service.configured") +def configure(): + session_db = endpoint_from_flag("session-db.master.available") + config = get_service_config() + session_db_primary, _ = postgres.get_db_uris(session_db) + # XXX cjwatson 2022-09-23: Mangle the connection string into a form + # Launchpad understands. In the long term it would be better to have + # Launchpad be able to consume unmodified connection strings. + update_pgpass(session_db_primary) + config["db_session"] = strip_dsn_authentication(session_db_primary) + config["db_session_user"] = parse_dsn(session_db_primary)["user"] + config["librarian_dir"] = os.path.join(base.base_dir(), "librarian") + host.mkdir( + config["librarian_dir"], + owner=base.user(), + group=base.user(), + perms=0o700, + ) + for i in range(config["workers"]): + config["logfile"] = os.path.join( + base.logs_dir(), f"librarian{i + 1}.log" + ) + config["worker_download_port"] = config["port_download_base"] + i + config["worker_restricted_download_port"] = ( + config["port_restricted_download_base"] + i + ) + config["worker_restricted_upload_port"] = ( + config["port_restricted_upload_base"] + i + ) + config["worker_upload_port"] = config["port_upload_base"] + i + configure_lazr( + config, + "launchpad-librarian-lazr.conf", + f"launchpad-librarian{i + 1}/launchpad-lazr.conf", + ) + configure_lazr( + config, + "launchpad-librarian-secrets-lazr.conf", + "launchpad-librarian-secrets-lazr.conf", + secret=True, + ) + configure_systemd(config) + configure_logrotate(config) + configure_cron(config, "crontab.j2") + + if helpers.any_file_changed( + [ + base.version_info_path(), + "/lib/systemd/system/launchpad-librarian.service", + "/lib/systemd/system/launchpad-librarian@.service", + ] + + config_files() + ): + hookenv.log("Config files changed; restarting") + # Be careful to restart instances one at a time to minimize downtime. + # XXX cjwatson 2023-03-28: This doesn't deal with stopping instances + # when the worker count is reduced. + for i in range(config["workers"]): + host.service_restart(f"launchpad-librarian@{i + 1}") + else: + hookenv.log("Not restarting, since no config files were changed") + host.service_resume("launchpad-librarian.service") + + set_state("service.configured") + + +@when("service.configured") +def check_is_running(): + hookenv.status_set("active", "Ready") + + +@when("nrpe-external-master.available", "service.configured") +@when_not("launchpad.librarian.nrpe-external-master.published") +def nrpe_available(): + nrpe = endpoint_from_flag("nrpe-external-master.available") + config = hookenv.config() + # XXX cjwatson 2023-03-28: This doesn't deal with removing checks when + # the worker count is reduced. + for i in range(config["workers"]): + nrpe.add_check( + [ + "/usr/lib/nagios/plugins/check_http", + "-H", + "localhost", + "-p", + str(config["port_download_base"] + i), + "-u", + config["nagios_path"], + "-s", + shlex.quote(config["nagios_expected_regex"]), + "-f", + "critical", + ], + name=f"check_librarian{i + 1}", + description=f"Launchpad librarian{i + 1}", + context=config["nagios_context"], + ) + set_flag("launchpad.librarian.nrpe-external-master.published") + + +@when("launchpad.librarian.nrpe-external-master.published") +@when_not("nrpe-external-master.available") +def nrpe_unavailable(): + clear_flag("launchpad.librarian.nrpe-external-master.published") + + +@when_any( + "config.changed.nagios_expected_regex", + "config.changed.nagios_path", + "config.changed.workers", +) +def nagios_options_changed(): + clear_flag("launchpad.librarian.nrpe-external-master.published") + + +@hook("upgrade-charm") +def upgrade_charm(): + # The ols and launchpad-base layer take care of clearing other flags. + clear_flag("launchpad.librarian.nrpe-external-master.published") diff --git a/charm/launchpad-librarian/templates/crontab.j2 b/charm/launchpad-librarian/templates/crontab.j2 new file mode 100644 index 0000000..a545a63 --- /dev/null +++ b/charm/launchpad-librarian/templates/crontab.j2 @@ -0,0 +1,25 @@ +TZ=UTC +MAILTO={{ cron_mailto }} +LPCONFIG=launchpad-librarian1 + +{% if active -%} +45 17 * * * {{ code_dir }}/cronscripts/expire-archive-files.py --expire-after=7 >> {{ logs_dir }}/expire-archive-files.log 2>&1 + +# Garbage collector. Ensure it doesn't run during a backup, or clash with a +# fastdowntime. +15 10 * * * {{ code_dir }}/cronscripts/librarian-gc.py -q --log-file=INFO:{{ logs_dir }}/librarian-gc.log + +{% endif -%} +{% if os_password and not upstream_host -%} +# Feed locally-spooled uploads into Swift. +{% for i in range(swift_feed_workers) -%} +*/10 * * * * run-one {{ code_dir }}/cronscripts/librarian-feed-swift.py --remove -q --log-file=INFO:{{ logs_dir }}/librarian-feed-swift-{{ i }}.log --num-instances={{ swift_feed_workers }} --instance-id={{ i }} +{% endfor %} +{% endif -%} +# Delete old logs +15 0 * * * find {{ logs_dir }} -maxdepth 1 -type f -mtime +90 -name 'librarian.log.*' -delete + +# Catch up with publishing OOPSes that were temporarily spooled to disk due +# to RabbitMQ being unavailable. +*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_username }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_vhost }} --repo {{ oopses_dir }} --key "" + diff --git a/charm/launchpad-librarian/templates/gunicorn.conf.py.j2 b/charm/launchpad-librarian/templates/gunicorn.conf.py.j2 new file mode 100644 index 0000000..efd8a1e --- /dev/null +++ b/charm/launchpad-librarian/templates/gunicorn.conf.py.j2 @@ -0,0 +1,9 @@ +bind = [":{{ port_main }}", ":{{ port_xmlrpc }}"] +workers = {{ wsgi_workers }} +# This is set relatively low to work around memory leaks on Python 3. +max_requests = 2000 +log_level = "DEBUG" +# Must be higher than the highest hard_timeout feature rule. +timeout = 65 +graceful_timeout = 120 + diff --git a/charm/launchpad-librarian/templates/launchpad-librarian-generator.j2 b/charm/launchpad-librarian/templates/launchpad-librarian-generator.j2 new file mode 100755 index 0000000..4929624 --- /dev/null +++ b/charm/launchpad-librarian/templates/launchpad-librarian-generator.j2 @@ -0,0 +1,18 @@ +#! /bin/sh +# Copyright 2022 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +# Part of the launchpad-librarian Juju charm. + +set -e + +wantdir="$1/launchpad-librarian.service.wants" +template=/lib/systemd/system/launchpad-librarian@.service + +# Generate systemd unit dependency symlinks for all configured +# launchpad-librarian instances. +mkdir -p "$wantdir" +for i in $(seq {{ workers }}); do + ln -s "$template" "$wantdir/launchpad-librarian@$i.service" +done + diff --git a/charm/launchpad-librarian/templates/launchpad-librarian-lazr.conf b/charm/launchpad-librarian/templates/launchpad-librarian-lazr.conf new file mode 100644 index 0000000..5cc7c8b --- /dev/null +++ b/charm/launchpad-librarian/templates/launchpad-librarian-lazr.conf @@ -0,0 +1,41 @@ +# Public configuration data. The contents of this file may be freely shared +# with developers if needed for debugging. + +# A schema's sections, keys, and values are automatically inherited, except +# for '.optional' sections. Update this config to override key values. +# Values are strings, except for numbers that look like ints. The tokens +# true, false, and none are treated as True, False, and None. + +{% from "macros.j2" import opt -%} + +[meta] +extends: ../launchpad-base-lazr.conf + +[launchpad_session] +database: {{ db_session }} +dbuser: {{ db_session_user }} + +[librarian] +download_port: {{ worker_download_port }} +restricted_download_port: {{ worker_restricted_download_port }} +restricted_upload_port: {{ worker_restricted_upload_port }} +upload_port: {{ worker_upload_port }} + +[librarian_server] +launch: true +logfile: {{ logfile }} +{{- opt("old_os_auth_url", old_os_auth_url) }} +{{- opt("old_os_auth_version", old_os_auth_version) }} +{{- opt("old_os_tenant_name", old_os_tenant_name) }} +{{- opt("old_os_username", old_os_username) }} +{{- opt("os_auth_url", os_auth_url) }} +{{- opt("os_auth_version", os_auth_version) }} +{{- opt("os_tenant_name", os_tenant_name) }} +{{- opt("os_username", os_username) }} +root: {{ librarian_dir }} +swift_timeout: {{ swift_timeout }} +{%- if upstream_host %} +upstream_host: {{ upstream_host }} +upstream_port: {{ upstream_port }} +{%- endif %} + diff --git a/charm/launchpad-librarian/templates/launchpad-librarian-secrets-lazr.conf b/charm/launchpad-librarian/templates/launchpad-librarian-secrets-lazr.conf new file mode 100644 index 0000000..d030f1d --- /dev/null +++ b/charm/launchpad-librarian/templates/launchpad-librarian-secrets-lazr.conf @@ -0,0 +1,16 @@ +# Secret configuration data. This is stored in an overlay directory, mainly +# to avoid accidental information leaks from the public configuration file. +# Entries in this file should not be shared with developers, although the +# structure of the file is not secret, only configuration values. + +# A schema's sections, keys, and values are automatically inherited, except +# for '.optional' sections. Update this config to override key values. +# Values are strings, except for numbers that look like ints. The tokens +# true, false, and none are treated as True, False, and None. + +{% from "macros.j2" import opt -%} + +[librarian_server] +{{- opt("old_os_password", old_os_password) }} +{{- opt("os_password", os_password) }} + diff --git a/charm/launchpad-librarian/templates/launchpad-librarian.service.j2 b/charm/launchpad-librarian/templates/launchpad-librarian.service.j2 new file mode 100644 index 0000000..fc7469d --- /dev/null +++ b/charm/launchpad-librarian/templates/launchpad-librarian.service.j2 @@ -0,0 +1,16 @@ +# This service is really a systemd target, but we use a service since +# targets cannot be reloaded. See launchpad-librarian@.service for instance +# configuration. + +[Unit] +Description=Launchpad librarian + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true +ExecReload=/bin/true + +[Install] +WantedBy=multi-user.target + diff --git a/charm/launchpad-librarian/templates/launchpad-librarian@.service.j2 b/charm/launchpad-librarian/templates/launchpad-librarian@.service.j2 new file mode 100644 index 0000000..846016e --- /dev/null +++ b/charm/launchpad-librarian/templates/launchpad-librarian@.service.j2 @@ -0,0 +1,25 @@ +[Unit] +Description=Launchpad librarian (%i) +PartOf=launchpad-librarian.service +Before=launchpad-librarian.service +ReloadPropagatedFrom=launchpad-librarian.service +After=network.target +ConditionPathExists=!{{ code_dir }}/maintenance.txt + +[Service] +User=launchpad +Group=launchpad +WorkingDirectory={{ code_dir }} +# https://portal.admin.canonical.com/C44221 +MemoryMax=4G +Environment=LPCONFIG=launchpad-librarian%i +SyslogIdentifier=librarian +ExecStart={{ code_dir }}/bin/twistd --python daemons/librarian.tac --pidfile {{ var_dir }}/librarian%i.pid --prefix librarian --logfile {{ logfile }} --nodaemon +ExecReload=/bin/kill -USR1 $MAINPID +KillMode=mixed +Restart=on-failure +PrivateTmp=true + +[Install] +WantedBy=multi-user.target + diff --git a/charm/launchpad-librarian/templates/logrotate.conf.j2 b/charm/launchpad-librarian/templates/logrotate.conf.j2 new file mode 100644 index 0000000..66e2b12 --- /dev/null +++ b/charm/launchpad-librarian/templates/logrotate.conf.j2 @@ -0,0 +1,28 @@ +{{ logs_dir }}/librarian[0-9]*.log +{ + rotate 21 + daily + dateext + delaycompress + compress + notifempty + missingok + create 0644 syslog adm + sharedscripts + postrotate + systemctl reload launchpad-librarian.service + endscript +} + +{{ logs_dir }}/expire-archive-files.log {{ logs_dir }}/librarian-gc.log {{ logs_dir }}/librarian-feed-swift-*.log +{ + rotate 21 + daily + dateext + delaycompress + compress + notifempty + missingok + create 0644 syslog adm +} +
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp