Guruprasad has proposed merging
~lgp171188/launchpad:charm-launchpad-scripts-bzrsyncd into launchpad:master.
Commit message:
charm: Implement the launchpad-scripts-bzrsyncd charm
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~lgp171188/launchpad/+git/launchpad/+merge/446073
--
Your team Launchpad code reviewers is requested to review the proposed merge of
~lgp171188/launchpad:charm-launchpad-scripts-bzrsyncd into launchpad:master.
diff --git a/charm/launchpad-scripts-bzrsyncd/README.md b/charm/launchpad-scripts-bzrsyncd/README.md
new file mode 100644
index 0000000..6e5fced
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/README.md
@@ -0,0 +1,19 @@
+# Launchpad scripts - bzrsyncd
+
+This charm sets up and runs the Launchpad bzrsyncd scripts and Celery workers.
+
+You will need the following relations:
+
+ juju relate launchpad-scripts-bzrsyncd:db postgresql:db
+ juju relate launchpad-scripts-bzrsyncd memcached
+ juju relate launchpad-scripts-bzrsyncd rabbitmq-server
+
+## Maintenance actions
+
+To stop Celery workers (perhaps in preparation for a schema upgrade), run:
+
+ juju run-action --wait launchpad-scripts-bzrsyncd/leader stop-services
+
+To start them again once maintenance is complete:
+
+ juju run-action --wait launchpad-scripts-bzrsyncd/leader start-services
diff --git a/charm/launchpad-scripts-bzrsyncd/actions.yaml b/charm/launchpad-scripts-bzrsyncd/actions.yaml
new file mode 100644
index 0000000..e99d06e
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/actions.yaml
@@ -0,0 +1,8 @@
+start-services:
+ description: Start celery workers. Usually run after maintenance.
+stop-services:
+ description: |
+ Stop celery workers. Usually run in preparation for maintenance. (Note that
+ this does not stop services in a way that will persist across a reboot. It
+ also doesn't disable cron jobs, since those are handled by the cron-control
+ mechanism instead. see lp.services.scripts.base.cronscript_enabled.)
diff --git a/charm/launchpad-scripts-bzrsyncd/actions/actions.py b/charm/launchpad-scripts-bzrsyncd/actions/actions.py
new file mode 100644
index 0000000..02617ec
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/actions/actions.py
@@ -0,0 +1,59 @@
+#! /usr/bin/python3
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import subprocess
+import sys
+import traceback
+from pathlib import Path
+
+sys.path.append("lib")
+
+from charms.layer import basic # noqa: E402
+
+basic.bootstrap_charm_deps()
+basic.init_config_states()
+
+from charmhelpers.core import hookenv # noqa: E402
+
+services = (
+ "celerybeat_bzrsyncd.service",
+ "celeryd_bzrsyncd_job.service",
+ "celeryd_bzrsyncd_job_slow.service",
+)
+
+
+def start_services():
+ for service in services:
+ hookenv.log(f"Starting {service}.")
+ subprocess.run(["systemctl", "start", service], check=True)
+ hookenv.action_set({"result": "Services started"})
+
+
+def stop_services():
+ for service in services:
+ hookenv.log(f"Stopping {service}.")
+ subprocess.run(["systemctl", "stop", service], check=True)
+ hookenv.action_set({"result": "Services stopped"})
+
+
+def main(argv):
+ action = Path(argv[0]).name
+ try:
+ if action == "start-services":
+ start_services()
+ elif action == "stop-services":
+ stop_services()
+ else:
+ hookenv.action_fail(f"Action {action} not implemented.")
+ except Exception:
+ hookenv.action_fail("Unhandled exception")
+ tb = traceback.format_exc()
+ hookenv.action_set(dict(traceback=tb))
+ hookenv.log(f"Unhandled exception in action {action}:")
+ for line in tb.splitlines():
+ hookenv.log(line)
+
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/charm/launchpad-scripts-bzrsyncd/actions/start-services b/charm/launchpad-scripts-bzrsyncd/actions/start-services
new file mode 100644
index 0000000..405a394
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/actions/start-services
@@ -0,0 +1 @@
+actions.py
\ No newline at end of file
diff --git a/charm/launchpad-scripts-bzrsyncd/actions/stop-services b/charm/launchpad-scripts-bzrsyncd/actions/stop-services
new file mode 100644
index 0000000..405a394
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/actions/stop-services
@@ -0,0 +1 @@
+actions.py
\ No newline at end of file
diff --git a/charm/launchpad-scripts-bzrsyncd/charmcraft.yaml b/charm/launchpad-scripts-bzrsyncd/charmcraft.yaml
new file mode 100644
index 0000000..9b28632
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/charmcraft.yaml
@@ -0,0 +1,63 @@
+type: charm
+bases:
+ - build-on:
+ - name: ubuntu
+ channel: "20.04"
+ architectures: [amd64]
+ run-on:
+ - name: ubuntu
+ channel: "20.04"
+ architectures: [amd64]
+parts:
+ charm-wheels:
+ source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels
+ source-commit: "42c89d9c66dbe137139b047fd54aed49b66d1a5e"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": charm-wheels/
+ prime:
+ - "-charm-wheels"
+ ols-layers:
+ source: https://git.launchpad.net/ols-charm-deps
+ source-commit: "f63ae0386275bf9089b30c8abae252a0ea523633"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ "*": layers/
+ stage:
+ - layers
+ prime:
+ - "-layers"
+ launchpad-layers:
+ after:
+ - ols-layers
+ source: https://git.launchpad.net/launchpad-layers
+ source-commit: "42a4b4c4f62936b1d050c775e84f7364dfb5efc0"
+ source-submodules: []
+ source-type: git
+ plugin: dump
+ organize:
+ launchpad-base: layers/layer/launchpad-base
+ launchpad-db: layers/layer/launchpad-db
+ launchpad-payload: layers/layer/launchpad-payload
+ stage:
+ - layers
+ prime:
+ - "-layers"
+ launchpad-scripts-bzrsyncd:
+ after:
+ - charm-wheels
+ - launchpad-layers
+ source: .
+ plugin: reactive
+ build-snaps: [charm]
+ build-packages: [libpq-dev, python3-dev]
+ build-environment:
+ - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer
+ - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface
+ - PIP_NO_INDEX: "true"
+ - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels
+ reactive-charm-build-arguments: [--binary-wheels-from-source]
diff --git a/charm/launchpad-scripts-bzrsyncd/config.yaml b/charm/launchpad-scripts-bzrsyncd/config.yaml
new file mode 100644
index 0000000..6702f25
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/config.yaml
@@ -0,0 +1,5 @@
+options:
+ active:
+ type: boolean
+ description: Should the cron jobs and the celery services be active?
+ default: true
diff --git a/charm/launchpad-scripts-bzrsyncd/layer.yaml b/charm/launchpad-scripts-bzrsyncd/layer.yaml
new file mode 100644
index 0000000..e7b38bd
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/layer.yaml
@@ -0,0 +1,14 @@
+includes:
+ - layer:launchpad-db
+ - interface:memcache
+options:
+ ols-pg:
+ databases:
+ db:
+ name: launchpad_dev
+ roles:
+ - branchscanner
+ - merge-proposal-jobs
+ - request-daily-builds
+ - send-branch-mail
+ - translationsbranchscanner
diff --git a/charm/launchpad-scripts-bzrsyncd/metadata.yaml b/charm/launchpad-scripts-bzrsyncd/metadata.yaml
new file mode 100644
index 0000000..632b902
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/metadata.yaml
@@ -0,0 +1,20 @@
+name: launchpad-scripts-bzrsyncd
+display-name: launchpad-scripts-bzrsyncd
+summary: Launchpad scripts - bzrsyncd
+maintainer: Launchpad Developers <launchpad-...@lists.launchpad.net>
+description: |
+ Launchpad is an open source suite of tools that help people and teams
+ to work together on software projects.
+
+ This charm sets up the cron jobs to run the scripts needed for 'bzrsyncd'.
+tags:
+ # https://juju.is/docs/charm-metadata#heading--charm-store-fields
+ - network
+series:
+ - focal
+subordinate: false
+requires:
+ session-db:
+ interface: pgsql
+ memcache:
+ interface: memcache
diff --git a/charm/launchpad-scripts-bzrsyncd/reactive/launchpad-scripts-bzrsyncd.py b/charm/launchpad-scripts-bzrsyncd/reactive/launchpad-scripts-bzrsyncd.py
new file mode 100644
index 0000000..6e85f10
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/reactive/launchpad-scripts-bzrsyncd.py
@@ -0,0 +1,104 @@
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import subprocess
+
+from charmhelpers.core import hookenv, host, templating
+from charms.launchpad.base import configure_email, get_service_config
+from charms.launchpad.payload import configure_cron, configure_lazr
+from charms.reactive import (
+ endpoint_from_flag,
+ remove_state,
+ set_state,
+ when,
+ when_not,
+ when_not_all,
+)
+
+
+def configure_logrotate(config):
+ hookenv.log("Writing logrotate configuration.")
+ templating.render(
+ "logrotate.conf.j2",
+ "/etc/logrotate.d/launchpad",
+ config,
+ perms=0o644,
+ )
+
+
+@host.restart_on_change(
+ {
+ "/lib/systemd/system/celerybeat-bzrsyncd.service": [
+ "celerybeat-bzrsyncd",
+ ],
+ "/lib/systemd/system/celeryd_bzrsyncd_job.service": [
+ "celeryd_bzrsyncd_job",
+ ],
+ "/lib/systemd/system/celeryd_bzrsyncd_job_slow.service": [
+ "celeryd_bzrsyncd_job_slow",
+ ]
+ }
+)
+def configure_celery(config):
+ hookenv.log("Writing celery systemd service files.")
+ destination_dir = "/lib/systemd/system"
+ service_files = (
+ "celerybeat_bzrsyncd.service",
+ "celeryd_bzrsyncd_job.service",
+ "celeryd_bzrsyncd_job_slow.service",
+ )
+ for service_file in service_files:
+ templating.render(
+ f"{service_file}.j2",
+ f"{destination_dir}/{service_file}",
+ config,
+ )
+ subprocess.check_call(["systemctl", "daemon-reload"])
+ for service_file in service_files:
+ subprocess.check_call(["systemctl", "enable", service_file])
+
+
+@when(
+ "launchpad.db.configured",
+ "memcache.available",
+)
+@when_not("service.configured")
+def configure():
+ config = get_service_config()
+ memcache = endpoint_from_flag("memcache.available")
+ config["memcache_servers"] = ",".join(
+ sorted(
+ f"({host}:{port},1)"
+ for host, port in memcache.memcache_hosts_ports()
+ )
+ )
+ configure_lazr(
+ config,
+ "launchpad-scripts-bzrsyncd-lazr.conf.j2",
+ "launchpad-scripts-bzrsyncd/launchpad-lazr.conf",
+ )
+ configure_lazr(
+ config,
+ "launchpad-scripts-bzrsyncd-secrets-lazr.conf.j2",
+ "launchpad-scripts-bzrsyncd-secrets-lazr.conf",
+ secret=True,
+ )
+ configure_email(config, "launchpad-scripts")
+ configure_logrotate(config)
+ set_state("service.configured")
+ configure_cron(config, "crontab.j2")
+ configure_celery(config)
+
+
+@when("service.configured")
+def check_is_running():
+ hookenv.status_set("active", "Ready")
+
+
+@when("service.configured")
+@when_not_all(
+ "launchpad.db.configured",
+ "memcache.available",
+)
+def deconfigure():
+ remove_state("service.configured")
diff --git a/charm/launchpad-scripts-bzrsyncd/templates/celerybeat_bzrsyncd.service.j2 b/charm/launchpad-scripts-bzrsyncd/templates/celerybeat_bzrsyncd.service.j2
new file mode 100644
index 0000000..598aecc
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/templates/celerybeat_bzrsyncd.service.j2
@@ -0,0 +1,15 @@
+[Unit]
+Description=celerybeat scheduler for bzrsyncd
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Environment=LPCONFIG=launchpad-scripts-bzrsyncd
+User=launchpad
+WorkingDirectory={{ code_dir }}
+ExecStart={{ code_dir }}/bin/celery beat --config=lp.services.job.celeryconfig
+Restart=on-failure
+LimitNOFILE=10000
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/charm/launchpad-scripts-bzrsyncd/templates/celeryd_bzrsyncd_job.service.j2 b/charm/launchpad-scripts-bzrsyncd/templates/celeryd_bzrsyncd_job.service.j2
new file mode 100644
index 0000000..47d6eda
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/templates/celeryd_bzrsyncd_job.service.j2
@@ -0,0 +1,24 @@
+[Unit]
+Description=Celery service for Launchpad bzrsyncd jobs
+After=network.target
+
+[Service]
+Type=exec
+User=launchpad
+Group=launchpad
+Environment=LPCONFIG=launchpad-scripts-bzrsyncd
+WorkingDirectory={{ code_dir }}
+ExecStart={{ code_dir }}/bin/celery worker \
+ --queue bzrsyncd_job \
+ --config=lp.services.job.celeryconfig \
+ --hostname=launchpad_bzrsyncd_job \
+ --loglevel=DEBUG \
+ --logfile={{ logs_dir }}/celeryd_launchpad_bzrsyncd_job.log
+ExecStop=/bin/kill -TERM $MAINPID
+ExecReload=/bin/kill -HUP $MAINPID
+PrivateTmp=true
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/charm/launchpad-scripts-bzrsyncd/templates/celeryd_bzrsyncd_job_slow.service.j2 b/charm/launchpad-scripts-bzrsyncd/templates/celeryd_bzrsyncd_job_slow.service.j2
new file mode 100644
index 0000000..34923be
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/templates/celeryd_bzrsyncd_job_slow.service.j2
@@ -0,0 +1,24 @@
+[Unit]
+Description=Celery service for Launchpad bzrsyncd slow jobs
+After=network.target
+
+[Service]
+Type=exec
+User=launchpad
+Group=launchpad
+Environment=LPCONFIG=launchpad-scripts-bzrsyncd
+WorkingDirectory={{ code_dir }}
+ExecStart={{ code_dir }}/bin/celery worker \
+ --queue bzrsyncd_job_slow \
+ --config=lp.services.job.celeryconfig \
+ --hostname=launchpad_bzrsyncd_job_slow \
+ --loglevel=DEBUG \
+ --logfile={{ logs_dir }}/celeryd_launchpad_bzrsyncd_job_slow.log
+ExecStop=/bin/kill -TERM $MAINPID
+ExecReload=/bin/kill -HUP $MAINPID
+PrivateTmp=true
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/charm/launchpad-scripts-bzrsyncd/templates/crontab.j2 b/charm/launchpad-scripts-bzrsyncd/templates/crontab.j2
new file mode 100644
index 0000000..923857d
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/templates/crontab.j2
@@ -0,0 +1,32 @@
+TZ=UTC
+MAILTO={{ cron_mailto }}
+LPCONFIG=launchpad-scripts-bzrsyncd
+
+{%- if active %}
+
+# cleanup /tmp/bzr-limbo-* leftovers
+0 1 * * * find /tmp -maxdepth 1 -name 'bzr-limbo-??????' -type d -mtime +1|xargs -r -I@ rm -rf '@'
+
+# branch scanner
+* * * * * {{ code_dir }}/cronscripts/process-job-source.py IBranchScanJobSource -q --log-file=DEBUG:{{ logs_dir }}/process-job-source.IBranchScanJobSource.log
+
+# Branch Updates - emailed out
+# set hard memory limit as per https://bugs.launchpad.net/launchpad/+bug/585126
+* * * * * ulimit -v 1843200; {{ code_dir }}/cronscripts/process-job-source.py IRevisionMailJobSource -q --log-file=DEBUG:{{ logs_dir }}/process-job-source.IRevisionMailJobSource.log
+* * * * * ulimit -v 1843200; {{ code_dir}}/cronscripts/process-job-source.py IRevisionsAddedJobSource -q --log-file=DEBUG:{{ logs_dir }}/process-job-source.IRevisionsAddedJobSource.log
+
+# processes code branches that have new translations imports pending
+*/10 * * * * {{ code_dir }}/cronscripts/process-job-source.py IRosettaUploadJobSource -q --log-file=DEBUG:{{ logs_dir }}/process-job-source.IRosettaUploadJobSource.log
+
+# Update or create previews diffs for branch merge proposals &
+# Send email about new merge proposals, generating a diff first if needed
+* * * * * {{ code_dir }}/cronscripts/process-job-source.py IBranchMergeProposalJobSource -q --log-file=DEBUG:{{ logs_dir }}/process-job-source.IBranchMergeProposalJobSource.log
+
+# Script requests builds for source package recipes that have "build daily" enabled
+*/15 * * * * {{ code_dir }}/cronscripts/request_daily_builds.py -v >> {{ logs_dir }}/request_daily_builds.log 2>&1
+
+{%- endif %}
+
+# OOPS amqp
+*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_username }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_vhost }} --repo {{ oopses_dir }} --key ""
+
diff --git a/charm/launchpad-scripts-bzrsyncd/templates/launchpad-scripts-bzrsyncd-lazr.conf.j2 b/charm/launchpad-scripts-bzrsyncd/templates/launchpad-scripts-bzrsyncd-lazr.conf.j2
new file mode 100644
index 0000000..955c9df
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/templates/launchpad-scripts-bzrsyncd-lazr.conf.j2
@@ -0,0 +1,19 @@
+# Public configuration data. The contents of this file may be freely shared
+# with developers if needed for debugging.
+
+# A schema's sections, keys, and values are automatically inherited, except
+# for '.optional' sections. Update this config to override key values.
+# Values are strings, except for numbers that look like ints. The tokens
+# true, false, and none are treated as True, False, and None.
+
+{% from "macros.j2" import opt -%}
+
+[meta]
+extends: ../launchpad-db-lazr.conf
+
+[codehosting]
+internal_branch_by_id_root: http://bazaar.lp.internal/
+
+[memcache]
+servers: {{ memcache_servers }}
+
diff --git a/charm/launchpad-scripts-bzrsyncd/templates/launchpad-scripts-bzrsyncd-secrets-lazr.conf.j2 b/charm/launchpad-scripts-bzrsyncd/templates/launchpad-scripts-bzrsyncd-secrets-lazr.conf.j2
new file mode 100644
index 0000000..e7d11ab
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/templates/launchpad-scripts-bzrsyncd-secrets-lazr.conf.j2
@@ -0,0 +1,10 @@
+# Secret configuration data. This is stored in an overlay directory, mainly
+# to avoid accidental information leaks from the public configuration file.
+# Entries in this file should not be shared with developers, although the
+# structure of the file is not secret, only configuration values.
+
+# A schema's sections, keys, and values are automatically inherited, except
+# for '.optional' sections. Update this config to override key values.
+# Values are strings, except for numbers that look like ints. The tokens
+# true, false, and none are treated as True, False, and None.
+
diff --git a/charm/launchpad-scripts-bzrsyncd/templates/logrotate.conf.j2 b/charm/launchpad-scripts-bzrsyncd/templates/logrotate.conf.j2
new file mode 100644
index 0000000..fc28d9d
--- /dev/null
+++ b/charm/launchpad-scripts-bzrsyncd/templates/logrotate.conf.j2
@@ -0,0 +1,13 @@
+{{ logs_dir }}/*.log {
+ rotate 21
+ dateext
+ daily
+ compress
+ delaycompress
+ sharedscripts
+ postrotate
+ systemctl restart celeryd_bzrsyncd_job
+ systemctl restart celeryd_bzrsyncd_job_slow
+ endscript
+}
+
_______________________________________________
Mailing list: https://launchpad.net/~launchpad-reviewers
Post to : launchpad-reviewers@lists.launchpad.net
Unsubscribe : https://launchpad.net/~launchpad-reviewers
More help : https://help.launchpad.net/ListHelp
