Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:charm-fixes
into autopkgtest-cloud:master.
Requested reviews:
Skia (hyask)
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/472678
charm bugfixes
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:charm-fixes into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py b/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
index 68c7768..e01a689 100644
--- a/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
+++ b/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
@@ -235,8 +235,10 @@ def stop():
@when_all("autopkgtest.target-restart-needed", "autopkgtest.target_running")
def restart_target():
- status.maintenance("Restarting autopkgtest systemd target")
- subprocess.check_call(["systemctl", "restart", "autopkgtest.target"])
+ status.maintenance("Restarting autopkgtest systemd target with --no-block")
+ subprocess.check_call(
+ ["systemctl", "restart", "autopkgtest.target", "--no-block"]
+ )
status.maintenance("Done restarting autopkgtest systemd target")
clear_flag("autopkgtest.target-restart-needed")
@@ -332,11 +334,13 @@ def clear_rabbitmq():
status.maintenance("Done clearing rabbitmq configuration")
-@when("config.changed.nova-rcs")
+@when_any(
+ "config.set.nova-rcs",
+ "autopkgtest.no-nova-rcs",
+)
def update_nova_rcs():
status.maintenance("Updating nova rc files")
# pylint: disable=import-outside-toplevel
- import base64
from io import BytesIO
from tarfile import TarFile
@@ -350,13 +354,14 @@ def update_nova_rcs():
clear_old_rcs()
- bytes_file = BytesIO(base64.b64decode(rctar))
+ bytes_file = BytesIO(bytes(rctar, encoding="utf-8"))
tar = TarFile(fileobj=bytes_file)
log("...got {}".format(", ".join(tar.getnames())), "INFO")
tar.extractall(os.path.expanduser("~ubuntu/cloudrcs/"))
status.maintenance("Done updating nova rc files")
+ clear_flag("autopkgtest.no-nova-rcs")
@when("config.default.nova-rcs")
@@ -365,6 +370,8 @@ def clear_old_rcs():
rcfiles = glob.glob(os.path.expanduser("~ubuntu/cloudrcs/*.rc"))
if not rcfiles:
+ set_flag("autopkgtest.no-nova-rcs")
+ status.maintenance("No old nova rc files to clear")
return
log("Deleting old cloud .rc files...", "INFO")
@@ -374,7 +381,7 @@ def clear_old_rcs():
os.unlink(rcfile)
log("...done", "INFO")
- status.maintenance("Done cleaning old nova rc files")
+ status.active("Done cleaning old nova rc files")
@when_all(
@@ -413,9 +420,11 @@ def remove_old_lxd_units():
"config.set.releases",
)
@when_any(
+ "config.set.n-workers",
"config.changed.n-workers",
"config.set.lxd-remotes",
"config.changed.lxd-remotes",
+ "config.set.releases",
"config.changed.releases",
)
def enable_disable_units():
@@ -547,8 +556,20 @@ def write_swift_config():
"config.changed.worker-net-names",
"config.changed.worker-upstream-percentage",
"config.changed.stable-release-percentage",
+ "config.set.worker-flavor-config",
+ "config.set.worker-args",
+ "config.set.worker-setup-command",
+ "config.set.worker-setup-command2",
+ "config.set.releases",
+ "config.set.n-workers",
+ "config.set.lxd-remotes",
+ "config.set.mirror",
+ "config.set.worker-net-names",
+ "config.set.worker-upstream-percentage",
+ "config.set.stable-release-percentage",
+ "config.set.nova-rcs",
+ "config.set.lxd-remotes",
)
-@when_any("config.set.nova-rcs", "config.set.lxd-remotes")
def write_worker_config():
status.maintenance("Writing worker configuration")
@@ -794,5 +815,5 @@ def unset_influx_creds():
os.unlink(os.path.expanduser("~ubuntu/influx.cred"))
except FileNotFoundError:
pass
- status.maintenance("Done deleting influxdb credentials")
+ status.active("Done deleting influxdb credentials")
clear_flag("autopkgtest.influx-creds-written")
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index 1e0f84d..618eac1 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -40,9 +40,11 @@ SWIFT_WEB_CREDENTIALS_PATH = os.path.expanduser(
)
API_KEYS_PATH = "/home/ubuntu/external-web-requests-api-keys.json"
CONFIG_DIR = pathlib.Path("/home/ubuntu/.config/autopkgtest-web/")
+if not CONFIG_DIR.exists():
+ set_flag("autopkgtest-web.config-needs-writing")
for parent in reversed(CONFIG_DIR.parents):
- parent.mkdir(mode=0o770, exist_ok=True)
-CONFIG_DIR.mkdir(mode=0o770, exist_ok=True)
+ parent.mkdir(mode=0o777, exist_ok=True)
+CONFIG_DIR.mkdir(mode=0o777, exist_ok=True)
ALLOWED_REQUESTOR_TEAMS_PATH = CONFIG_DIR / "allowed-requestor-teams"
PUBLIC_SWIFT_CREDS_PATH = os.path.expanduser("~ubuntu/public-swift-creds")
@@ -187,6 +189,7 @@ def clone_autopkgtest_cloud():
def set_up_systemd_units():
status.maintenance("Setting up systemd units")
any_changed = False
+ new_units = False
for unit in glob.glob(
os.path.join(
AUTOPKGTEST_CLOUD_GIT_LOCATION,
@@ -203,6 +206,7 @@ def set_up_systemd_units():
unit,
os.path.join(os.path.sep, "etc", "systemd", "system", base),
)
+ new_units = True
except FileExistsError:
pass
p = subprocess.run(
@@ -219,7 +223,7 @@ def set_up_systemd_units():
subprocess.check_call(["systemctl", "enable", base])
status.active("systemd units installed")
- if any_changed:
+ if any_changed or new_units:
set_flag("autopkgtest-web.autopkgtest-web-target-needs-restart")
@@ -337,9 +341,10 @@ def set_up_web_config(apache):
apache.send_enabled()
-@when_all(
+@when_any(
"config.changed.allowed-requestor-teams",
"config.set.allowed-requestor-teams",
+ "autopkgtest-web.config-needs-writing",
)
def write_allowed_teams():
allowed_requestor_teams = config().get("allowed-requestor-teams")
@@ -347,7 +352,11 @@ def write_allowed_teams():
allowed_teams_path.write_text(allowed_requestor_teams, encoding="utf-8")
-@when_all("config.changed.github-secrets", "config.set.github-secrets")
+@when_any(
+ "config.changed.github-secrets",
+ "config.set.github-secrets",
+ "autopkgtest-web.config-needs-writing",
+)
def write_github_secrets():
status.maintenance("Writing github secrets")
github_secrets = config().get("github-secrets")
@@ -368,6 +377,7 @@ def write_github_secrets():
@when_all(
"config.changed.external-web-requests-api-keys",
"config.set.external-web-requests-api-keys",
+ "autopkgtest-web.config-needs-writing",
)
def write_api_keys():
status.maintenance("Writing api keys")
@@ -582,9 +592,12 @@ def symlink_public_db():
),
)
set_flag(symlink_flag)
- status.maintenance(f"Done creating symlink for {symlink_file}")
+ status.active(f"Done creating symlink for {symlink_file}")
except FileExistsError:
- pass
+ clear_flag(symlink_flag)
+ status.active(
+ "symlinking public db and sha256 checksum already done"
+ )
@when("leadership.is_leader")
diff --git a/charms/focal/autopkgtest-web/units/download-all-results.service b/charms/focal/autopkgtest-web/units/download-all-results.service
deleted file mode 100644
index 464d2db..0000000
--- a/charms/focal/autopkgtest-web/units/download-all-results.service
+++ /dev/null
@@ -1,10 +0,0 @@
-[Unit]
-Description=Download all results
-
-[Service]
-User=ubuntu
-Type=oneshot
-ExecStart=/home/ubuntu/webcontrol/download-all-results
-
-[Install]
-WantedBy=autopkgtest-web.target
diff --git a/charms/focal/autopkgtest-web/units/sqlite-writer.service b/charms/focal/autopkgtest-web/units/sqlite-writer.service
index 3a47c08..cf3b48e 100644
--- a/charms/focal/autopkgtest-web/units/sqlite-writer.service
+++ b/charms/focal/autopkgtest-web/units/sqlite-writer.service
@@ -5,6 +5,7 @@ StartLimitBurst=60
[Service]
User=ubuntu
+EnvironmentFile=/home/ubuntu/public-swift-creds
ExecStart=/home/ubuntu/webcontrol/sqlite-writer
Restart=on-failure
RestartSec=1s
diff --git a/charms/focal/autopkgtest-web/webcontrol/cache-amqp b/charms/focal/autopkgtest-web/webcontrol/cache-amqp
index 124d1b4..e953c9d 100755
--- a/charms/focal/autopkgtest-web/webcontrol/cache-amqp
+++ b/charms/focal/autopkgtest-web/webcontrol/cache-amqp
@@ -12,7 +12,7 @@ import urllib.parse
import amqplib.client_0_8 as amqp
from amqplib.client_0_8.exceptions import AMQPChannelException
-from helpers.utils import get_autopkgtest_cloud_conf
+from helpers.utils import get_autopkgtest_cloud_conf, is_db_empty
AMQP_CONTEXTS = ["ubuntu", "huge", "ppa", "upstream"]
@@ -85,6 +85,11 @@ class AutopkgtestQueueContents:
"""
db_con = sqlite3.connect("file:%s?mode=ro" % self.database, uri=True)
+ if is_db_empty(db_con=db_con):
+ logging.warning(
+ "Database is currently empty - waiting for it to be populated, exiting cache-amqp"
+ )
+ sys.exit(0)
release_arches = {}
releases = []
diff --git a/charms/focal/autopkgtest-web/webcontrol/db-backup b/charms/focal/autopkgtest-web/webcontrol/db-backup
index b03100b..7f2d83f 100755
--- a/charms/focal/autopkgtest-web/webcontrol/db-backup
+++ b/charms/focal/autopkgtest-web/webcontrol/db-backup
@@ -6,17 +6,16 @@ and clears up old backups
import atexit
import datetime
-import gzip
import hashlib
+import io
import logging
import os
-import shutil
import sqlite3
import sys
from pathlib import Path
import swiftclient
-from helpers.utils import get_autopkgtest_cloud_conf, init_db
+from helpers.utils import get_autopkgtest_cloud_conf, init_db, init_swift_con
DB_PATH = ""
DB_NAME = ""
@@ -47,40 +46,9 @@ def db_connect() -> sqlite3.Connection:
def backup_db(db_con: sqlite3.Connection):
- db_backup_con = sqlite3.connect(DB_BACKUP_PATH)
- with db_backup_con:
- db_con.backup(db_backup_con, pages=1)
- db_backup_con.close()
-
-
-def compress_db():
- """
- use gzip to compress database
- """
- with open(DB_BACKUP_PATH, "rb") as f_in, gzip.open(
- "%s.gz" % DB_BACKUP_PATH, "wb"
- ) as f_out:
- shutil.copyfileobj(f_in, f_out)
-
-
-def init_swift_con() -> swiftclient.Connection:
- """
- Establish connection to swift storage
- """
- swift_creds = {
- "authurl": os.environ["OS_AUTH_URL"],
- "user": os.environ["OS_USERNAME"],
- "key": os.environ["OS_PASSWORD"],
- "os_options": {
- "region_name": os.environ["OS_REGION_NAME"],
- "project_domain_name": os.environ["OS_PROJECT_DOMAIN_NAME"],
- "project_name": os.environ["OS_PROJECT_NAME"],
- "user_domain_name": os.environ["OS_USER_DOMAIN_NAME"],
- },
- "auth_version": 3,
- }
- swift_conn = swiftclient.Connection(**swift_creds)
- return swift_conn
+ with io.open(DB_BACKUP_PATH, "w") as bkp:
+ for line in db_con.iterdump():
+ bkp.write(f"{line}\n")
def create_container_if_it_doesnt_exist(swift_conn: swiftclient.Connection):
@@ -96,12 +64,12 @@ def create_container_if_it_doesnt_exist(swift_conn: swiftclient.Connection):
def get_db_backup_checksum():
- with open("%s.gz" % DB_BACKUP_PATH, "rb") as bkp_f:
+ with open(DB_BACKUP_PATH, "rb") as bkp_f:
md5 = hashlib.md5(bkp_f.read()).hexdigest()
return md5
-def upload_backup_to_db(
+def upload_backup_to_swift(
swift_conn: swiftclient.Connection,
) -> swiftclient.Connection:
"""
@@ -111,18 +79,17 @@ def upload_backup_to_db(
checksum = get_db_backup_checksum()
object_path = "%s/%s-%s.%s" % (
now,
- DB_PATH.name.split(".")[0],
+ DB_BACKUP_NAME.split(".")[0],
checksum,
- "db.gz",
+ "db.bak",
)
+ db_backup_contents = Path(DB_BACKUP_PATH).read_bytes()
for retry in range(SWIFT_RETRIES):
try:
swift_conn.put_object(
- CONTAINER_NAME,
- object_path,
- "%s.gz" % DB_BACKUP_PATH,
- content_type="text/plain; charset=UTF-8",
- headers={"Content-Encoding": "gzip"},
+ container=CONTAINER_NAME,
+ obj=object_path,
+ contents=db_backup_contents,
)
break
except swiftclient.exceptions.ClientException as e:
@@ -183,15 +150,13 @@ if __name__ == "__main__":
db_con = db_connect()
logging.info("Creating a backup of the db...")
backup_db(db_con)
- logging.info("Compressing db")
- compress_db()
logging.info("Registering cleanup function")
atexit.register(cleanup)
logging.info("Setting up swift connection")
swift_conn = init_swift_con()
create_container_if_it_doesnt_exist(swift_conn)
logging.info("Uploading db to swift!")
- swift_conn = upload_backup_to_db(swift_conn)
+ swift_conn = upload_backup_to_swift(swift_conn)
logging.info("Pruning old database backups")
swift_conn = delete_old_backups(swift_conn)
cleanup()
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index 7b22012..9524473 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -17,6 +17,7 @@ import typing
from dataclasses import dataclass
import distro_info
+import swiftclient
sqlite3.paramstyle = "named"
@@ -241,4 +242,44 @@ def get_test_id(db_con, release, arch, src):
return test_id
+def init_swift_con() -> swiftclient.Connection:
+ """
+ Establish connection to swift storage
+ """
+ swift_creds = {
+ "authurl": os.environ["OS_AUTH_URL"],
+ "user": os.environ["OS_USERNAME"],
+ "key": os.environ["OS_PASSWORD"],
+ "os_options": {
+ "region_name": os.environ["OS_REGION_NAME"],
+ "project_domain_name": os.environ["OS_PROJECT_DOMAIN_NAME"],
+ "project_name": os.environ["OS_PROJECT_NAME"],
+ "user_domain_name": os.environ["OS_USER_DOMAIN_NAME"],
+ },
+ "auth_version": 3,
+ }
+ swift_conn = swiftclient.Connection(**swift_creds)
+ return swift_conn
+
+
+def is_db_empty(db_con):
+ # maybe we need to check the db path exists also?
+ # for id_, name, filename in db_con.execute("PRAGMA database_list"):
+ # if name == "main" and filename is not None:
+ # path = filename
+ # elif name == "main" and filename is None:
+ # if pathlib.Path()
+ cursor = db_con.cursor()
+ cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+ tables = cursor.fetchall()
+ if len(tables) == 0:
+ return True
+ for table in tables:
+ cursor.execute(f"SELECT * FROM {table[0]};")
+ entries = cursor.fetchall()
+ if len(entries) > 0:
+ return False
+ return True
+
+
get_test_id._cache = {}
diff --git a/charms/focal/autopkgtest-web/webcontrol/publish-db b/charms/focal/autopkgtest-web/webcontrol/publish-db
index a621f8a..6c4ffd6 100755
--- a/charms/focal/autopkgtest-web/webcontrol/publish-db
+++ b/charms/focal/autopkgtest-web/webcontrol/publish-db
@@ -11,11 +11,12 @@ import hashlib
import logging
import os
import sqlite3
+import sys
import tempfile
import urllib.request
import apt_pkg
-from helpers.utils import get_autopkgtest_cloud_conf
+from helpers.utils import get_autopkgtest_cloud_conf, is_db_empty
sqlite3.paramstyle = "named"
@@ -28,11 +29,29 @@ components = ["main", "restricted", "universe", "multiverse"]
def init_db(path, path_current, path_rw):
"""Create DB if it does not exist, and connect to it"""
-
db = sqlite3.connect(path)
db_rw = sqlite3.connect("file:%s?mode=ro" % path_rw, uri=True)
- # Copy r/w database over
+ # checks if db is empty
+ if is_db_empty(db_rw):
+ logging.warning(
+ (
+ "Looks like this unit has been recently deployed - publish-db will"
+ " exit and wait for the sqlite-writer to restore the db from a backup"
+ )
+ )
+ sys.exit(0)
+
+ # if no db, we need to copy /home/ubuntu/autopkgtest.db to /home/ubuntu/public/autopkgtest.db?
+ if not os.path.exists(path_current):
+ logging.warning(
+ f"Looks like there's no pre-existing db at {path_current}, copying..."
+ )
+ public_db_con = sqlite3.connect(path_current)
+ db_rw.backup(public_db_con)
+ public_db_con.close()
+
+ logging.info(f"backing up {path_rw} to {path}")
with db:
db_rw.backup(db)
db_rw.close()
diff --git a/charms/focal/autopkgtest-web/webcontrol/sqlite-writer b/charms/focal/autopkgtest-web/webcontrol/sqlite-writer
index d0ec23a..bdd66e2 100755
--- a/charms/focal/autopkgtest-web/webcontrol/sqlite-writer
+++ b/charms/focal/autopkgtest-web/webcontrol/sqlite-writer
@@ -13,7 +13,13 @@ sqlite3.paramstyle = "named"
import urllib.parse
import amqplib.client_0_8 as amqp
-from helpers.utils import SqliteWriterConfig, get_test_id, init_db
+from helpers.utils import (
+ SqliteWriterConfig,
+ get_test_id,
+ init_db,
+ init_swift_con,
+ is_db_empty,
+)
LAST_CHECKPOINT = datetime.datetime.now()
@@ -38,14 +44,15 @@ def amqp_connect():
return amqp_con
-def db_connect():
- """Connect to SQLite DB"""
+def get_db_path():
cp = configparser.ConfigParser()
cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+ return cp["web"]["database"]
- db_con = init_db(cp["web"]["database"])
- return db_con
+def db_connect():
+ """Connect to SQLite DB"""
+ return init_db(get_db_path())
def check_msg(queue_msg):
@@ -109,9 +116,59 @@ def msg_callback(msg, db_con):
checkpoint_db_if_necessary(db_con)
+def restore_db_from_backup(db_con: sqlite3.Connection):
+ backups_container = "db-backups"
+ new_db_path = f"{get_db_path()}.new"
+ logging.info(f"Creating new db: {new_db_path}")
+ if os.path.isfile(new_db_path):
+ os.remove(new_db_path)
+ os.mknod(new_db_path)
+ logging.info(f"Connecting to new db {new_db_path}")
+ new_db = sqlite3.connect(new_db_path)
+ logging.info("Connecting to swift")
+ swift_conn = init_swift_con()
+ logging.info(
+ f"Connected to swift! Getting backups from container: {backups_container}"
+ )
+ _, objects = swift_conn.get_container(container=backups_container)
+ latest = objects[-1]
+ _, db_dump = swift_conn.get_object(
+ container=backups_container, obj=latest["name"]
+ )
+ logging.info(
+ (
+ f"Restoring db {new_db_path} from swift - "
+ f"container: {backups_container} - object: {latest['name']}"
+ )
+ )
+ for line in db_dump.splitlines():
+ try:
+ new_db.execute(line.decode("utf-8"))
+ except sqlite3.OperationalError as e:
+ logging.warning(
+ f"Running sql command: `{line.decode('utf-8')}` failed with {e}"
+ )
+ # checkpoint old db
+ db_con.execute("PRAGMA wal_checkpoint(TRUNCATE);")
+ # remove it
+ os.remove(get_db_path())
+ # move new db to old db location
+ os.rename(new_db_path, get_db_path())
+ # reinit db_con - with the new db after it's been moved, and return it
+ db_con.close()
+ logging.info("db restored from backup!")
+ return db_connect()
+
+
def main():
logging.basicConfig(level=logging.INFO)
db_con = db_connect()
+ if is_db_empty(db_con):
+ logging.info(
+ "DB is empty, indicating this unit has been recently deployed."
+ )
+ logging.info("Restoring database from a swift backup")
+ db_con = restore_db_from_backup(db_con)
amqp_con = amqp_connect()
status_ch = amqp_con.channel()
status_ch.access_request("/complete", active=True, read=True, write=False)
diff --git a/mojo/service-bundle b/mojo/service-bundle
index 4170fe9..143df3a 100644
--- a/mojo/service-bundle
+++ b/mojo/service-bundle
@@ -294,8 +294,8 @@ applications:
- service_name: results_reverse_proxy
service_options:
- server swift {{ storage_host_internal }} ssl verify required ca-file /etc/ssl/certs/ca-certificates.crt
- - reqirep ^(GET|POST|HEAD)\ /results/(.*) \1\ {{ storage_path_internal }}/\2
- - rspirep ^Location:\ (http|https)://{{ storage_host_internal }}{{ storage_path_internal }}\/(.*) Location:\ \1://{{ hostname }}/results/\2
+ - http-request replace-pathq ^(GET|POST|HEAD)\ /results/(.*) \1\ {{ storage_path_internal }}/\2
+ - http-response replace-header ^Location:\ (http|https)://{{ storage_host_internal }}{{ storage_path_internal }}\/(.*) Location:\ \1://{{ hostname }}/results/\2
ssl_cert: DEFAULT
ssl_key:
{%- elif stage_name == "devel" %}
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
