The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/3675
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From df098ccda7be31b3ce2124ab756c98acce81b14c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Fri, 18 Aug 2017 00:56:07 -0400 Subject: [PATCH 1/4] client: Cleanup code duplication in image download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- client/simplestreams_images.go | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/client/simplestreams_images.go b/client/simplestreams_images.go index fd82d9912..1ae9c6acf 100644 --- a/client/simplestreams_images.go +++ b/client/simplestreams_images.go @@ -2,6 +2,7 @@ package lxd import ( "fmt" + "io" "strings" "github.com/lxc/lxd/shared/api" @@ -57,22 +58,32 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe // Prepare the response resp := ImageFileResponse{} - // Download the LXD image file - meta, ok := files["meta"] - if ok && req.MetaFile != nil { + // Download function + download := func(path string, filename string, sha256 string, target io.WriteSeeker) (int64, error) { // Try over http - url := fmt.Sprintf("http://%s/%s";, strings.TrimPrefix(r.httpHost, "https://";), meta.Path) + url := fmt.Sprintf("http://%s/%s";, strings.TrimPrefix(r.httpHost, "https://";), path) - size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "metadata", url, meta.Sha256, req.MetaFile) + size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target) if err != nil { // Try over https - url = fmt.Sprintf("%s/%s", r.httpHost, meta.Path) - size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "metadata", url, meta.Sha256, req.MetaFile) + url = fmt.Sprintf("%s/%s", r.httpHost, path) + size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target) if err != nil { - return nil, err + return -1, err } } + return size, nil + } + + // Download the LXD image file + meta, ok := files["meta"] + if ok && req.MetaFile != nil { + size, err := download(meta.Path, "metadata", meta.Sha256, req.MetaFile) + if err != nil { + return nil, err + } + parts := strings.Split(meta.Path, "/") resp.MetaName = parts[len(parts)-1] resp.MetaSize = size @@ -81,17 +92,9 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe // Download the rootfs rootfs, ok := files["root"] if ok && req.RootfsFile != nil { - // Try over http - url := fmt.Sprintf("http://%s/%s";, strings.TrimPrefix(r.httpHost, "https://";), rootfs.Path) - - size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "rootfs", url, rootfs.Sha256, req.RootfsFile) + size, err := download(rootfs.Path, "rootfs", rootfs.Sha256, req.RootfsFile) if err != nil { - // Try over https - url = fmt.Sprintf("%s/%s", r.httpHost, rootfs.Path) - size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "rootfs", url, rootfs.Sha256, req.RootfsFile) - if err != nil { - return nil, err - } + return nil, err } parts := strings.Split(rootfs.Path, "/") From 3011ed610aca6ba4e71ee8236de58b2f62834233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Fri, 18 Aug 2017 01:40:37 -0400 Subject: [PATCH 2/4] shared/simplestreams: Add delta support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- shared/simplestreams/simplestreams.go | 39 ++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index 0615fe011..322f4fffe 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -115,8 +115,14 @@ func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) { var meta SimpleStreamsManifestProductVersionItem var rootTar SimpleStreamsManifestProductVersionItem var rootSquash SimpleStreamsManifestProductVersionItem + deltas := []SimpleStreamsManifestProductVersionItem{} for _, item := range version.Items { + // Identify deltas + if item.FileType == "squashfs.vcdiff" { + deltas = append(deltas, item) + } + // Skip the files we don't care about if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "squashfs"}) { continue @@ -223,9 +229,39 @@ func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) { } } - downloads[fingerprint] = [][]string{ + imgDownloads := [][]string{ {metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}, {rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}} + + // Add the deltas + for _, delta := range deltas { + srcImage, ok := product.Versions[delta.DeltaBase] + if !ok { + continue + } + + var srcFingerprint string + for _, item := range srcImage.Items { + if item.FileType != "lxd.tar.xz" { + continue + } + + srcFingerprint = item.LXDHashSha256SquashFs + break + } + + if srcFingerprint == "" { + continue + } + + imgDownloads = append(imgDownloads, []string{ + delta.Path, + delta.HashSha256, + fmt.Sprintf("root.delta-%s", srcFingerprint), + fmt.Sprintf("%d", delta.Size)}) + } + + downloads[fingerprint] = imgDownloads images = append(images, image) } } @@ -261,6 +297,7 @@ type SimpleStreamsManifestProductVersionItem struct { LXDHashSha256RootXz string `json:"combined_rootxz_sha256"` LXDHashSha256SquashFs string `json:"combined_squashfs_sha256"` Size int64 `json:"size"` + DeltaBase string `json:"delta_base"` } type SimpleStreamsIndex struct { From 217b609723b4a9a33b379d47d7ee3fb20ff7312f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Fri, 18 Aug 2017 02:08:52 -0400 Subject: [PATCH 3/4] client: Add support for delta image transfer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- client/interfaces.go | 4 +++ client/simplestreams_images.go | 77 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/client/interfaces.go b/client/interfaces.go index ed9b001a3..f6cb6006c 100644 --- a/client/interfaces.go +++ b/client/interfaces.go @@ -209,6 +209,10 @@ type ImageFileRequest struct { // A canceler that can be used to interrupt some part of the image download request Canceler *cancel.Canceler + + // Path retriever for image delta downloads + // If set, it must return the path to the image file or an empty string if not available + DeltaSourceRetriever func(fingerprint string, file string) string } // The ImageFileResponse struct is used as the response for image downloads diff --git a/client/simplestreams_images.go b/client/simplestreams_images.go index 1ae9c6acf..21c266d8f 100644 --- a/client/simplestreams_images.go +++ b/client/simplestreams_images.go @@ -3,8 +3,12 @@ package lxd import ( "fmt" "io" + "io/ioutil" + "os" + "os/exec" "strings" + "github.com/lxc/lxd/shared" "github.com/lxc/lxd/shared/api" ) @@ -92,14 +96,75 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe // Download the rootfs rootfs, ok := files["root"] if ok && req.RootfsFile != nil { - size, err := download(rootfs.Path, "rootfs", rootfs.Sha256, req.RootfsFile) - if err != nil { - return nil, err + // Look for deltas (requires xdelta3) + downloaded := false + _, err := exec.LookPath("xdelta3") + if err == nil && req.DeltaSourceRetriever != nil { + for filename, file := range files { + if !strings.HasPrefix(filename, "root.delta-") { + continue + } + + // Check if we have the source file for the delta + srcFingerprint := strings.Split(filename, "root.delta-")[1] + srcPath := req.DeltaSourceRetriever(srcFingerprint, "rootfs") + if srcPath == "" { + continue + } + + // Create temporary file for the delta + deltaFile, err := ioutil.TempFile("", "lxc_image_") + if err != nil { + return nil, err + } + defer deltaFile.Close() + defer os.Remove(deltaFile.Name()) + + // Download the delta + _, err = download(file.Path, "rootfs delta", file.Sha256, deltaFile) + if err != nil { + return nil, err + } + deltaFile.Close() + + // Create temporary file for the delta + patchedFile, err := ioutil.TempFile("", "lxc_image_") + if err != nil { + return nil, err + } + defer patchedFile.Close() + defer os.Remove(patchedFile.Name()) + + // Apply it + _, err = shared.RunCommand("xdelta3", "-f", "-d", "-s", srcPath, deltaFile.Name(), patchedFile.Name()) + if err != nil { + return nil, err + } + + // Copy to the target + size, err := io.Copy(req.RootfsFile, patchedFile) + if err != nil { + return nil, err + } + + parts := strings.Split(rootfs.Path, "/") + resp.RootfsName = parts[len(parts)-1] + resp.RootfsSize = size + downloaded = true + } } - parts := strings.Split(rootfs.Path, "/") - resp.RootfsName = parts[len(parts)-1] - resp.RootfsSize = size + // Download the whole file + if !downloaded { + size, err := download(rootfs.Path, "rootfs", rootfs.Sha256, req.RootfsFile) + if err != nil { + return nil, err + } + + parts := strings.Split(rootfs.Path, "/") + resp.RootfsName = parts[len(parts)-1] + resp.RootfsSize = size + } } return &resp, nil From 81a19a6707c912edd03cb5522c67ea76e327cc81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <[email protected]> Date: Fri, 18 Aug 2017 02:30:27 -0400 Subject: [PATCH 4/4] lxd: Add support for delta images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <[email protected]> --- lxd/daemon_images.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go index 659bb0d1a..4a56ca861 100644 --- a/lxd/daemon_images.go +++ b/lxd/daemon_images.go @@ -403,6 +403,14 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce RootfsFile: io.WriteSeeker(destRootfs), ProgressHandler: progress, Canceler: canceler, + DeltaSourceRetriever: func(fingerprint string, file string) string { + path := shared.VarPath("images", fmt.Sprintf("%s.%s", fingerprint, file)) + if shared.PathExists(path) { + return path + } + + return "" + }, } if secret != "" {
_______________________________________________ lxc-devel mailing list [email protected] http://lists.linuxcontainers.org/listinfo/lxc-devel
