The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6158
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From 67e611c4e4b8d65694c5697025bc67ef8682cf6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Thu, 5 Sep 2019 14:52:12 -0400 Subject: [PATCH 1/2] lxd/images: Use native tar parser for metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- lxd/images.go | 82 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/lxd/images.go b/lxd/images.go index efb35eee83..3a258d9fa9 100644 --- a/lxd/images.go +++ b/lxd/images.go @@ -1,6 +1,7 @@ package main import ( + "archive/tar" "bytes" "context" "crypto/sha256" @@ -810,46 +811,85 @@ func imagesPost(d *Daemon, r *http.Request) Response { } func getImageMetadata(fname string) (*api.ImageMetadata, error) { - metadataName := "metadata.yaml" + var tr *tar.Reader + var result api.ImageMetadata - compressionArgs, _, _, err := shared.DetectCompression(fname) + // Open the file + r, err := os.Open(fname) + if err != nil { + return nil, err + } + defer r.Close() + // Decompress if needed + _, _, unpacker, err := shared.DetectCompressionFile(r) if err != nil { - return nil, fmt.Errorf( - "detectCompression failed, err='%v', tarfile='%s'", - err, - fname) + return nil, err } + r.Seek(0, 0) - args := []string{"-O"} - args = append(args, compressionArgs...) - args = append(args, fname, metadataName) + if unpacker == nil { + return nil, fmt.Errorf("Unsupported backup compression") + } - // read the metadata.yaml - output, err := shared.RunCommand("tar", args...) + // Open the tarball + if len(unpacker) > 0 { + cmd := exec.Command(unpacker[0], unpacker[1:]...) + cmd.Stdin = r - if err != nil { - outputLines := strings.Split(output, "\n") - return nil, fmt.Errorf("Could not extract image %s from tar: %v (%s)", metadataName, err, outputLines[0]) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + defer stdout.Close() + + err = cmd.Start() + if err != nil { + return nil, err + } + defer cmd.Wait() + + tr = tar.NewReader(stdout) + } else { + tr = tar.NewReader(r) } - metadata := api.ImageMetadata{} - err = yaml.Unmarshal([]byte(output), &metadata) + // Parse the content + hasMeta := false + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + return nil, err + } - if err != nil { - return nil, fmt.Errorf("Could not parse %s: %v", metadataName, err) + if hdr.Name == "metadata.yaml" || hdr.Name == "./metadata.yaml" { + err = yaml.NewDecoder(tr).Decode(&result) + if err != nil { + return nil, err + } + + hasMeta = true + break + } + } + + if !hasMeta { + return nil, fmt.Errorf("Metadata tarball is missing metadata.yaml") } - _, err = osarch.ArchitectureId(metadata.Architecture) + _, err = osarch.ArchitectureId(result.Architecture) if err != nil { return nil, err } - if metadata.CreationDate == 0 { + if result.CreationDate == 0 { return nil, fmt.Errorf("Missing creation date") } - return &metadata, nil + return &result, nil } func doImagesGet(d *Daemon, recursion bool, project string, public bool) (interface{}, error) { From 07e74d30479253fbadb09900de3cfb02f49d618a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Thu, 5 Sep 2019 15:10:23 -0400 Subject: [PATCH 2/2] lxd/images: Tweak wrapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- lxd/images.go | 71 +++++++++++++++++---------------------------------- 1 file changed, 24 insertions(+), 47 deletions(-) diff --git a/lxd/images.go b/lxd/images.go index 3a258d9fa9..eaaf6caf01 100644 --- a/lxd/images.go +++ b/lxd/images.go @@ -463,24 +463,19 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string, imageTarf.Close() if err != nil { - logger.Error( - "Failed to copy the image tarfile", - log.Ctx{"err": err}) + logger.Error("Failed to copy the image tarfile", log.Ctx{"err": err}) return nil, err } // Get the rootfs tarball part, err = mr.NextPart() if err != nil { - logger.Error( - "Failed to get the next part", - log.Ctx{"err": err}) + logger.Error("Failed to get the next part", log.Ctx{"err": err}) return nil, err } if part.FormName() != "rootfs" { - logger.Error( - "Invalid multipart image") + logger.Error("Invalid multipart image") return nil, fmt.Errorf("Invalid multipart image") } @@ -497,9 +492,7 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string, rootfsTarf.Close() if err != nil { - logger.Error( - "Failed to copy the rootfs tarfile", - log.Ctx{"err": err}) + logger.Error("Failed to copy the rootfs tarfile", log.Ctx{"err": err}) return nil, err } @@ -514,33 +507,27 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string, imageMeta, err = getImageMetadata(imageTarf.Name()) if err != nil { - logger.Error( - "Failed to get image metadata", - log.Ctx{"err": err}) + logger.Error("Failed to get image metadata", log.Ctx{"err": err}) return nil, err } imgfname := shared.VarPath("images", info.Fingerprint) err = shared.FileMove(imageTarf.Name(), imgfname) if err != nil { - logger.Error( - "Failed to move the image tarfile", - log.Ctx{ - "err": err, - "source": imageTarf.Name(), - "dest": imgfname}) + logger.Error("Failed to move the image tarfile", log.Ctx{ + "err": err, + "source": imageTarf.Name(), + "dest": imgfname}) return nil, err } rootfsfname := shared.VarPath("images", info.Fingerprint+".rootfs") err = shared.FileMove(rootfsTarf.Name(), rootfsfname) if err != nil { - logger.Error( - "Failed to move the rootfs tarfile", - log.Ctx{ - "err": err, - "source": rootfsTarf.Name(), - "dest": imgfname}) + logger.Error("Failed to move the rootfs tarfile", log.Ctx{ + "err": err, + "source": rootfsTarf.Name(), + "dest": imgfname}) return nil, err } } else { @@ -549,9 +536,7 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string, info.Size = size logger.Debug("Tar size", log.Ctx{"size": size}) if err != nil { - logger.Error( - "Failed to copy the tarfile", - log.Ctx{"err": err}) + logger.Error("Failed to copy the tarfile", log.Ctx{"err": err}) return nil, err } @@ -560,35 +545,26 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string, expectedFingerprint := r.Header.Get("X-LXD-fingerprint") if expectedFingerprint != "" && info.Fingerprint != expectedFingerprint { - logger.Error( - "Fingerprints don't match", - log.Ctx{ - "got": info.Fingerprint, - "expected": expectedFingerprint}) - err = fmt.Errorf( - "fingerprints don't match, got %s expected %s", - info.Fingerprint, - expectedFingerprint) + logger.Error("Fingerprints don't match", log.Ctx{ + "got": info.Fingerprint, + "expected": expectedFingerprint}) + err = fmt.Errorf("fingerprints don't match, got %s expected %s", info.Fingerprint, expectedFingerprint) return nil, err } imageMeta, err = getImageMetadata(post.Name()) if err != nil { - logger.Error( - "Failed to get image metadata", - log.Ctx{"err": err}) + logger.Error("Failed to get image metadata", log.Ctx{"err": err}) return nil, err } imgfname := shared.VarPath("images", info.Fingerprint) err = shared.FileMove(post.Name(), imgfname) if err != nil { - logger.Error( - "Failed to move the tarfile", - log.Ctx{ - "err": err, - "source": post.Name(), - "dest": imgfname}) + logger.Error("Failed to move the tarfile", log.Ctx{ + "err": err, + "source": post.Name(), + "dest": imgfname}) return nil, err } } @@ -612,6 +588,7 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string, if err != nil { return nil, err } + if exists { // Do not create a database entry if the request is coming from the internal // cluster communications for image synchronization
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel