The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/3906
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From 0dd52f7aeb357b81b5e2a3fc6f0397198968c892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 17:05:01 -0400 Subject: [PATCH 01/10] tests: Fix shell return value masking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- test/suites/basic.sh | 12 ++++++------ test/suites/concurrent.sh | 2 +- test/suites/config.sh | 6 +++--- test/suites/deps.sh | 2 +- test/suites/image.sh | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/suites/basic.sh b/test/suites/basic.sh index f51bc378b..cb7d147ae 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -370,11 +370,11 @@ test_basic_usage() { aa_namespace="lxd-lxd-apparmor-test_<$(echo "${LXD_DIR}" | sed -e 's/\//-/g' -e 's/^.//')>" aa-status | grep -q ":${aa_namespace}:unconfined" || aa-status | grep -q ":${aa_namespace}://unconfined" lxc stop lxd-apparmor-test --force - ! aa-status | grep -q ":${aa_namespace}:" + ! aa-status | grep -q ":${aa_namespace}:" || false else aa-status | grep "lxd-lxd-apparmor-test_<${LXD_DIR}>" lxc stop lxd-apparmor-test --force - ! aa-status | grep -q "lxd-lxd-apparmor-test_<${LXD_DIR}>" + ! aa-status | grep -q "lxd-lxd-apparmor-test_<${LXD_DIR}>" || false fi lxc delete lxd-apparmor-test [ ! -f "${LXD_DIR}/security/apparmor/profiles/lxd-lxd-apparmor-test" ] @@ -420,9 +420,9 @@ test_basic_usage() { lxc publish --force c3 --alias=image3 # Delete multiple images with lxc delete and confirm they're deleted lxc image delete local:image1 local:image2 local:image3 - ! lxc image list | grep -q image1 - ! lxc image list | grep -q image2 - ! lxc image list | grep -q image3 + ! lxc image list | grep -q image1 || false + ! lxc image list | grep -q image2 || false + ! lxc image list | grep -q image3 || false # Cleanup the containers lxc delete --force c1 c2 c3 @@ -453,5 +453,5 @@ test_basic_usage() { sleep 2 lxc stop foo --force || true - ! lxc list | grep -q foo + ! lxc list | grep -q foo || false } diff --git a/test/suites/concurrent.sh b/test/suites/concurrent.sh index 107f4f08c..f0ec2884b 100644 --- a/test/suites/concurrent.sh +++ b/test/suites/concurrent.sh @@ -29,5 +29,5 @@ test_concurrent() { wait "${pid}" done - ! lxc list | grep -q concurrent + ! lxc list | grep -q concurrent || false } diff --git a/test/suites/config.sh b/test/suites/config.sh index 3231bd431..526378dfc 100644 --- a/test/suites/config.sh +++ b/test/suites/config.sh @@ -161,9 +161,9 @@ test_config_profiles() { # test live-adding a nic lxc start foo lxc exec foo -- cat /proc/self/mountinfo | grep -q "/mnt1.*ro," - ! lxc config show foo | grep -q "raw.lxc" + ! lxc config show foo | grep -q "raw.lxc" || false lxc config show foo --expanded | grep -q "raw.lxc" - ! lxc config show foo | grep -v "volatile.eth0" | grep -q "eth0" + ! lxc config show foo | grep -v "volatile.eth0" | grep -q "eth0" || false lxc config show foo --expanded | grep -v "volatile.eth0" | grep -q "eth0" lxc config device add foo eth2 nic nictype=p2p name=eth10 lxc exec foo -- /sbin/ifconfig -a | grep eth0 @@ -288,7 +288,7 @@ test_container_metadata() { # templates can be removed lxc config template delete c my.tpl - ! (lxc config template list c | grep -q my.tpl) + ! lxc config template list c | grep -q my.tpl || false lxc delete c } diff --git a/test/suites/deps.sh b/test/suites/deps.sh index 2ec30a983..69a1c879b 100644 --- a/test/suites/deps.sh +++ b/test/suites/deps.sh @@ -1,3 +1,3 @@ test_check_deps() { - ! ldd "$(which lxc)" | grep -q liblxc + ! ldd "$(which lxc)" | grep -q liblxc || false } diff --git a/test/suites/image.sh b/test/suites/image.sh index 62fa5019a..804cc84ad 100644 --- a/test/suites/image.sh +++ b/test/suites/image.sh @@ -26,7 +26,7 @@ test_image_expiry() { lxc_remote config set images.remote_cache_expiry 0 lxc_remote remote set-default local - ! lxc_remote image list l2: | grep -q "${fpbrief}" + ! lxc_remote image list l2: | grep -q "${fpbrief}" || false lxc_remote delete l2:c1 From d18836e133ef0b634e7659602756f2c2fcd9756e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 17:21:22 -0400 Subject: [PATCH 02/10] images: Respect disabled cache expiry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/images.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lxd/images.go b/lxd/images.go index eb3a3dabf..58b185d04 100644 --- a/lxd/images.go +++ b/lxd/images.go @@ -1010,6 +1010,10 @@ func pruneExpiredImages(d *Daemon) { // Get the list of expired images. expiry := daemonConfig["images.remote_cache_expiry"].GetInt64() + if expiry <= 0 { + return + } + images, err := db.ImagesGetExpired(d.db, expiry) if err != nil { logger.Error("Unable to retrieve the list of expired images", log.Ctx{"err": err}) From 45f86043b4614bf0d9eb7e53191e5b3595c0ae69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 17:38:29 -0400 Subject: [PATCH 03/10] images: Store UploadedAt as RFC3399 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/db/images.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lxd/db/images.go b/lxd/db/images.go index b2d259925..9f6d00a62 100644 --- a/lxd/db/images.go +++ b/lxd/db/images.go @@ -462,14 +462,14 @@ func ImageInsert(db *sql.DB, fp string, fname string, sz int64, public bool, aut autoUpdateInt = 1 } - stmt, err := tx.Prepare(`INSERT INTO images (fingerprint, filename, size, public, auto_update, architecture, creation_date, expiry_date, upload_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, strftime("%s"))`) + stmt, err := tx.Prepare(`INSERT INTO images (fingerprint, filename, size, public, auto_update, architecture, creation_date, expiry_date, upload_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`) if err != nil { tx.Rollback() return err } defer stmt.Close() - result, err := stmt.Exec(fp, fname, sz, publicInt, autoUpdateInt, arch, createdAt, expiresAt) + result, err := stmt.Exec(fp, fname, sz, publicInt, autoUpdateInt, arch, createdAt, expiresAt, time.Now().UTC()) if err != nil { tx.Rollback() return err From f29baf9f1cfed4dff19bb3b0614574b78fcdb4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 17:48:24 -0400 Subject: [PATCH 04/10] patches: Convert UploadedAt to RFC3399 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/patches.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lxd/patches.go b/lxd/patches.go index 2c630b10f..6f2c89777 100644 --- a/lxd/patches.go +++ b/lxd/patches.go @@ -49,6 +49,7 @@ var patches = []patch{ {name: "storage_zfs_volume_size", run: patchStorageZFSVolumeSize}, {name: "network_dnsmasq_hosts", run: patchNetworkDnsmasqHosts}, {name: "storage_api_dir_bind_mount", run: patchStorageApiDirBindMount}, + {name: "fix_uploaded_at", run: patchFixUploadedAt}, } type patch struct { @@ -2475,6 +2476,27 @@ func patchStorageApiDirBindMount(name string, d *Daemon) error { return nil } +func patchFixUploadedAt(name string, d *Daemon) error { + images, err := db.ImagesGet(d.db, false) + if err != nil { + return err + } + + for _, fingerprint := range images { + id, image, err := db.ImageGet(d.db, fingerprint, false, true) + if err != nil { + return err + } + + _, err = db.Exec(d.db, "UPDATE images SET upload_date=? WHERE id=?", image.UploadedAt, id) + if err != nil { + return err + } + } + + return nil +} + // Patches end here // Here are a couple of legacy patches that were originally in From eeafcab4bb19f06d1f674a16026ee9d10d8807e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 18:05:51 -0400 Subject: [PATCH 05/10] tests: Fix image expiry test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- test/suites/image.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/suites/image.sh b/test/suites/image.sh index 804cc84ad..1b05cfbc5 100644 --- a/test/suites/image.sh +++ b/test/suites/image.sh @@ -12,29 +12,30 @@ test_image_expiry() { # shellcheck disable=2153 lxc_remote remote add l1 "${LXD_ADDR}" --accept-certificate --password foo fi + if ! lxc_remote remote list | grep -q l2; then lxc_remote remote add l2 "${LXD2_ADDR}" --accept-certificate --password foo fi + + # Create a container from a remote image lxc_remote init l1:testimage l2:c1 fp=$(lxc_remote image info testimage | awk -F: '/^Fingerprint/ { print $2 }' | awk '{ print $1 }') + + # Confirm the image is cached [ ! -z "${fp}" ] fpbrief=$(echo "${fp}" | cut -c 1-10) - lxc_remote image list l2: | grep -q "${fpbrief}" - lxc_remote remote set-default l2 - lxc_remote config set images.remote_cache_expiry 0 - lxc_remote remote set-default local + # Override the upload date + sqlite3 "${LXD2_DIR}/lxd.db" "UPDATE images SET last_use_date='$(date --rfc-3339=seconds -u -d "2 days ago")' WHERE fingerprint='${fp}'" + # Trigger the expiry + lxc_remote config set l2: images.remote_cache_expiry 1 ! lxc_remote image list l2: | grep -q "${fpbrief}" || false + # Cleanup and reset lxc_remote delete l2:c1 - - # reset the default expiry - lxc_remote remote set-default l2 - lxc_remote config set images.remote_cache_expiry 10 - lxc_remote remote set-default local - + lxc_remote config set l2: images.remote_cache_expiry 10 lxc_remote remote remove l2 kill_lxd "$LXD2_DIR" } From e76c6dbc88ed0fd88f109c2507e079b0397612ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 18:23:19 -0400 Subject: [PATCH 06/10] daemon: Simplify time channels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/daemon.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lxd/daemon.go b/lxd/daemon.go index 0962fec52..2843ff3b6 100644 --- a/lxd/daemon.go +++ b/lxd/daemon.go @@ -806,9 +806,8 @@ func (d *Daemon) Ready() error { pruneExpiredImages(d) for { timer := time.NewTimer(24 * time.Hour) - timeChan := timer.C select { - case <-timeChan: + case <-timer.C: /* run once per day */ pruneExpiredImages(d) case <-d.pruneChan: @@ -833,10 +832,9 @@ func (d *Daemon) Ready() error { interval := daemonConfig["images.auto_update_interval"].GetInt64() if interval > 0 { timer := time.NewTimer(time.Duration(interval) * time.Hour) - timeChan := timer.C select { - case <-timeChan: + case <-timer.C: autoUpdateImages(d) case <-d.resetAutoUpdateChan: timer.Stop() From 8e91ef392551cae59948c0376b45b661a62b8b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 18:23:43 -0400 Subject: [PATCH 07/10] daemon: Fix handling of config triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/daemon_config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lxd/daemon_config.go b/lxd/daemon_config.go index 35b63220f..6b5ad7381 100644 --- a/lxd/daemon_config.go +++ b/lxd/daemon_config.go @@ -133,6 +133,11 @@ func (k *daemonConfigKey) Set(d *Daemon, value string) error { return err } + // Run the trigger (if any) + if k.trigger != nil { + k.trigger(d, k.name(), value) + } + return nil } From 941e0b60ccb4dbb45370902ac7eba20f9b1ecc43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 18:43:50 -0400 Subject: [PATCH 08/10] daemon: Don't update images while pruning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/daemon.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lxd/daemon.go b/lxd/daemon.go index 2843ff3b6..aefa007b2 100644 --- a/lxd/daemon.go +++ b/lxd/daemon.go @@ -803,7 +803,6 @@ func (d *Daemon) Ready() error { /* Prune images */ d.pruneChan = make(chan bool) go func() { - pruneExpiredImages(d) for { timer := time.NewTimer(24 * time.Hour) select { @@ -818,6 +817,9 @@ func (d *Daemon) Ready() error { } }() + // Do an initial pruning run before we start updating images + pruneExpiredImages(d) + /* Auto-update images */ d.resetAutoUpdateChan = make(chan bool) go func() { From ac493dcb118eb1cb8ece958bcedbccee26d27d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 18:44:18 -0400 Subject: [PATCH 09/10] images: Actually get the list of images to remove MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/db/images.go | 2 +- lxd/images.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lxd/db/images.go b/lxd/db/images.go index 9f6d00a62..c1b1fa470 100644 --- a/lxd/db/images.go +++ b/lxd/db/images.go @@ -76,7 +76,7 @@ func ImagesGetExpired(db *sql.DB, expiry int64) ([]string, error) { results = append(results, r[0].(string)) } - return []string{}, nil + return results, nil } func ImageSourceInsert(db *sql.DB, imageId int, server string, protocol string, certificate string, alias string) error { diff --git a/lxd/images.go b/lxd/images.go index 58b185d04..f915fd12f 100644 --- a/lxd/images.go +++ b/lxd/images.go @@ -1006,14 +1006,15 @@ func autoUpdateImage(d *Daemon, op *operation, id int, info *api.Image) error { } func pruneExpiredImages(d *Daemon) { - logger.Infof("Pruning expired images") - // Get the list of expired images. expiry := daemonConfig["images.remote_cache_expiry"].GetInt64() + + // Check if we're supposed to prune something if expiry <= 0 { return } + logger.Infof("Pruning expired images") images, err := db.ImagesGetExpired(d.db, expiry) if err != nil { logger.Error("Unable to retrieve the list of expired images", log.Ctx{"err": err}) From abdc002cc572f8ad36df2036d986bb332aa7ab58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Thu, 5 Oct 2017 18:56:17 -0400 Subject: [PATCH 10/10] test: Setup basic channel handler for triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/main_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lxd/main_test.go b/lxd/main_test.go index dfb2ef2b7..c8a46e6c5 100644 --- a/lxd/main_test.go +++ b/lxd/main_test.go @@ -121,6 +121,18 @@ func (suite *lxdTestSuite) SetupTest() { os.Exit(1) } suite.Req = require.New(suite.T()) + + suite.d.pruneChan = make(chan bool) + suite.d.resetAutoUpdateChan = make(chan bool) + go func() { + for { + select { + case <-suite.d.pruneChan: + case <-suite.d.resetAutoUpdateChan: + continue + } + } + }() } func (suite *lxdTestSuite) TearDownTest() {
_______________________________________________ lxc-devel mailing list [email protected] http://lists.linuxcontainers.org/listinfo/lxc-devel
