The following pull request was submitted through Github.
It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/5452
This e-mail was sent by the LXC bot, direct replies will not reach the author
unless they happen to be subscribed to this list.
=== Description (from pull-request) ===
From 20a55a17baf3933d10ca7aee009d54b8365cec7e Mon Sep 17 00:00:00 2001
From: Joel Hockey <joelhoc...@chromium.org>
Date: Thu, 31 Jan 2019 18:52:24 -0800
Subject: [PATCH 1/3] lxd/images: change compressFile to take io.Reader and
io.Writer
This is part 1 of a series of patches to add better progress
tracking support for export and import.
By using Reader and Writer rather than filename for compressing
the caller can provide a tracking reader/writer for progress.
Signed-off-by: Joel Hockey <joelhoc...@chromium.org>
---
lxd/backup.go | 16 ++++++++++++++--
lxd/images.go | 35 ++++++++++++++++++-----------------
lxd/patches.go | 16 ++++++++++++++--
3 files changed, 46 insertions(+), 21 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index 46a53a3014..e469a19429 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -353,7 +353,19 @@ func backupCreateTarball(s *state.State, path string,
backup backup) error {
}
if compress != "none" {
- compressedPath, err := compressFile(backupPath, compress)
+ infile, err := os.Open(backupPath)
+ if err != nil {
+ return err
+ }
+ defer infile.Close()
+
+ compressed, err := os.Create(backupPath + ".compressed")
+ if err != nil {
+ return err
+ }
+ defer compressed.Close()
+
+ err = compressFile(compress, infile, compressed)
if err != nil {
return err
}
@@ -363,7 +375,7 @@ func backupCreateTarball(s *state.State, path string,
backup backup) error {
return err
}
- err = os.Rename(compressedPath, backupPath)
+ err = os.Rename(compressed.Name(), backupPath)
if err != nil {
return err
}
diff --git a/lxd/images.go b/lxd/images.go
index a27f4f5620..ea8c2cc69f 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -129,7 +129,7 @@ func unpackImage(imagefname string, destpath string, sType
storageType, runningI
return nil
}
-func compressFile(path string, compress string) (string, error) {
+func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
reproducible := []string{"gzip"}
args := []string{"-c"}
@@ -137,24 +137,11 @@ func compressFile(path string, compress string) (string,
error) {
args = append(args, "-n")
}
- args = append(args, path)
cmd := exec.Command(compress, args...)
-
- outfile, err := os.Create(path + ".compressed")
- if err != nil {
- return "", err
- }
-
- defer outfile.Close()
+ cmd.Stdin = infile
cmd.Stdout = outfile
- err = cmd.Run()
- if err != nil {
- os.Remove(outfile.Name())
- return "", err
- }
-
- return outfile.Name(), nil
+ return cmd.Run()
}
/*
@@ -223,7 +210,21 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
}
if compress != "none" {
- compressedPath, err = compressFile(tarfile.Name(), compress)
+ tarfile, err = os.Open(tarfile.Name())
+ if err != nil {
+ return nil, err
+ }
+ defer tarfile.Close()
+
+ compressedPath = tarfile.Name() + ".compressed"
+
+ compressed, err := os.Create(compressedPath)
+ if err != nil {
+ return nil, err
+ }
+ defer compressed.Close()
+
+ err = compressFile(compress, tarfile, compressed)
if err != nil {
return nil, err
}
diff --git a/lxd/patches.go b/lxd/patches.go
index 235410e66c..93b2ab8e5f 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -3154,7 +3154,19 @@ func patchMoveBackups(name string, d *Daemon) error {
}
// Compress it
- compressedPath, err := compressFile(backupPath,
"xz")
+ infile, err := os.Open(backupPath)
+ if err != nil {
+ return err
+ }
+ defer infile.Close()
+
+ compressed, err := os.Create(backupPath +
".compressed")
+ if err != nil {
+ return err
+ }
+ defer compressed.Close()
+
+ err = compressFile("xz", infile, compressed)
if err != nil {
return err
}
@@ -3164,7 +3176,7 @@ func patchMoveBackups(name string, d *Daemon) error {
return err
}
- err = os.Rename(compressedPath, backupPath)
+ err = os.Rename(compressed.Name(), backupPath)
if err != nil {
return err
}
From a7723ab209169960b503f3ee789bb2f36050d21b Mon Sep 17 00:00:00 2001
From: Joel Hockey <joelhoc...@chromium.org>
Date: Thu, 31 Jan 2019 19:03:24 -0800
Subject: [PATCH 2/3] lxd/images: calculate sha256 as image is written
This is part 2 of a series of patches to add better progress
tracking support for export and import.
Calculate sha256 as either tarfile is written (if no compression)
or as compression is done to improve performance.
Signed-off-by: Joel Hockey <joelhoc...@chromium.org>
---
lxd/images.go | 46 ++++++++++++++++++++++++++--------------------
1 file changed, 26 insertions(+), 20 deletions(-)
diff --git a/lxd/images.go b/lxd/images.go
index ea8c2cc69f..e32488f5b9 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -191,14 +191,10 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
}
defer os.Remove(tarfile.Name())
- if err := c.Export(tarfile, req.Properties); err != nil {
- tarfile.Close()
- return nil, err
- }
- tarfile.Close()
-
+ sha256 := sha256.New()
var compressedPath string
var compress string
+ var writer io.Writer
if req.CompressionAlgorithm != "" {
compress = req.CompressionAlgorithm
@@ -208,8 +204,23 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
return nil, err
}
}
+ usingCompression := compress != "none"
- if compress != "none" {
+ // If there is no compression, then calculate sha256 on tarfile
+ if usingCompression {
+ writer = tarfile
+ } else {
+ writer = io.MultiWriter(tarfile, sha256)
+ compressedPath = tarfile.Name()
+ }
+
+ if err := c.Export(writer, req.Properties); err != nil {
+ tarfile.Close()
+ return nil, err
+ }
+ tarfile.Close()
+
+ if usingCompression {
tarfile, err = os.Open(tarfile.Name())
if err != nil {
return nil, err
@@ -222,29 +233,24 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
if err != nil {
return nil, err
}
+
defer compressed.Close()
+ defer os.Remove(compressed.Name())
- err = compressFile(compress, tarfile, compressed)
+ // Calculate sha256 as we compress
+ writer := io.MultiWriter(compressed, sha256)
+
+ err = compressFile(compress, tarfile, writer)
if err != nil {
return nil, err
}
- } else {
- compressedPath = tarfile.Name()
}
- defer os.Remove(compressedPath)
- sha256 := sha256.New()
- tarf, err := os.Open(compressedPath)
- if err != nil {
- return nil, err
- }
-
- info.Size, err = io.Copy(sha256, tarf)
- tarf.Close()
+ fi, err := os.Stat(compressedPath)
if err != nil {
return nil, err
}
-
+ info.Size = fi.Size()
info.Fingerprint = fmt.Sprintf("%x", sha256.Sum(nil))
_, _, err = d.cluster.ImageGet(project, info.Fingerprint, false, true)
From 09329e150b98f80f407b77b1716cc0c7e96d1f35 Mon Sep 17 00:00:00 2001
From: Joel Hockey <joelhoc...@chromium.org>
Date: Thu, 31 Jan 2019 19:15:25 -0800
Subject: [PATCH 3/3] shared.Unpack: Add support for a ProgressTracker during
unpack
This is part 3 of a series of patches to add better progress
tracking support for export and import.
Signed-off-by: Joel Hockey <joelhoc...@chromium.org>
---
lxd/images.go | 6 +++---
lxd/storage_btrfs.go | 2 +-
lxd/storage_ceph.go | 2 +-
lxd/storage_dir.go | 2 +-
lxd/storage_lvm.go | 2 +-
lxd/storage_lvm_utils.go | 2 +-
lxd/storage_zfs.go | 2 +-
shared/archive_linux.go | 41 ++++++++++++++++++++++++++++++----------
8 files changed, 40 insertions(+), 19 deletions(-)
diff --git a/lxd/images.go b/lxd/images.go
index e32488f5b9..c8c33b29ee 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -97,14 +97,14 @@ var aliasCmd = Command{
end for whichever finishes last. */
var imagePublishLock sync.Mutex
-func unpackImage(imagefname string, destpath string, sType storageType,
runningInUserns bool) error {
+func unpackImage(imagefname string, destpath string, sType storageType,
runningInUserns bool, tracker *ioprogress.ProgressTracker) error {
blockBackend := false
if sType == storageTypeLvm || sType == storageTypeCeph {
blockBackend = true
}
- err := shared.Unpack(imagefname, destpath, blockBackend,
runningInUserns)
+ err := shared.Unpack(imagefname, destpath, blockBackend,
runningInUserns, tracker)
if err != nil {
return err
}
@@ -116,7 +116,7 @@ func unpackImage(imagefname string, destpath string, sType
storageType, runningI
return fmt.Errorf("Error creating rootfs directory")
}
- err = shared.Unpack(imagefname+".rootfs", rootfsPath,
blockBackend, runningInUserns)
+ err = shared.Unpack(imagefname+".rootfs", rootfsPath,
blockBackend, runningInUserns, tracker)
if err != nil {
return err
}
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 7c423976c3..26f6a95b26 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -2057,7 +2057,7 @@ func (s *storageBtrfs) ImageCreate(fingerprint string)
error {
// Unpack the image in imageMntPoint.
imagePath := shared.VarPath("images", fingerprint)
- err = unpackImage(imagePath, tmpImageSubvolumeName, storageTypeBtrfs,
s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, tmpImageSubvolumeName, storageTypeBtrfs,
s.s.OS.RunningInUserNS, nil)
if err != nil {
return err
}
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index c141c26974..8dc18784c0 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -2189,7 +2189,7 @@ func (s *storageCeph) ImageCreate(fingerprint string)
error {
// rsync contents into image
imagePath := shared.VarPath("images", fingerprint)
- err = unpackImage(imagePath, imageMntPoint, storageTypeCeph,
s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, imageMntPoint, storageTypeCeph,
s.s.OS.RunningInUserNS, nil)
if err != nil {
logger.Errorf(`Failed to unpack image for RBD storage
volume for image "%s" on storage pool "%s": %s`, fingerprint, s.pool.Name, err)
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index dd7f7a3396..fc32a9d939 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -542,7 +542,7 @@ func (s *storageDir) ContainerCreateFromImage(container
container, imageFingerpr
}()
imagePath := shared.VarPath("images", imageFingerprint)
- err = unpackImage(imagePath, containerMntPoint, storageTypeDir,
s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, containerMntPoint, storageTypeDir,
s.s.OS.RunningInUserNS, nil)
if err != nil {
return errors.Wrap(err, "Unpack image")
}
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index d8022512f4..b2df2b938d 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -1968,7 +1968,7 @@ func (s *storageLvm) ImageCreate(fingerprint string)
error {
}
imagePath := shared.VarPath("images", fingerprint)
- err = unpackImage(imagePath, imageMntPoint, storageTypeLvm,
s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, imageMntPoint, storageTypeLvm,
s.s.OS.RunningInUserNS, nil)
if err != nil {
return err
}
diff --git a/lxd/storage_lvm_utils.go b/lxd/storage_lvm_utils.go
index d16175dbbf..77e7ad8372 100644
--- a/lxd/storage_lvm_utils.go
+++ b/lxd/storage_lvm_utils.go
@@ -502,7 +502,7 @@ func (s *storageLvm) containerCreateFromImageLv(c
container, fp string) error {
imagePath := shared.VarPath("images", fp)
containerMntPoint := getContainerMountPoint(c.Project(), s.pool.Name,
containerName)
- err = unpackImage(imagePath, containerMntPoint, storageTypeLvm,
s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, containerMntPoint, storageTypeLvm,
s.s.OS.RunningInUserNS, nil)
if err != nil {
logger.Errorf(`Failed to unpack image "%s" into non-thinpool
LVM storage volume "%s" for container "%s" on storage pool "%s": %s`,
imagePath, containerMntPoint, containerName, s.pool.Name, err)
return err
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index 76b347a9d4..6eff09ff19 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -2443,7 +2443,7 @@ func (s *storageZfs) ImageCreate(fingerprint string)
error {
}
// Unpack the image into the temporary mountpoint.
- err = unpackImage(imagePath, tmpImageDir, storageTypeZfs,
s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, tmpImageDir, storageTypeZfs,
s.s.OS.RunningInUserNS, nil)
if err != nil {
return err
}
diff --git a/shared/archive_linux.go b/shared/archive_linux.go
index ca359763cf..c141936387 100644
--- a/shared/archive_linux.go
+++ b/shared/archive_linux.go
@@ -8,6 +8,7 @@ import (
"strings"
"syscall"
+ "github.com/lxc/lxd/shared/ioprogress"
"github.com/lxc/lxd/shared/logger"
)
@@ -54,7 +55,7 @@ func DetectCompressionFile(f io.ReadSeeker) ([]string,
string, []string, error)
}
}
-func Unpack(file string, path string, blockBackend bool, runningInUserns bool)
error {
+func Unpack(file string, path string, blockBackend bool, runningInUserns bool,
tracker *ioprogress.ProgressTracker) error {
extractArgs, extension, _, err := DetectCompression(file)
if err != nil {
return err
@@ -62,6 +63,7 @@ func Unpack(file string, path string, blockBackend bool,
runningInUserns bool) e
command := ""
args := []string{}
+ var reader io.Reader
if strings.HasPrefix(extension, ".tar") {
command = "tar"
if runningInUserns {
@@ -73,8 +75,32 @@ func Unpack(file string, path string, blockBackend bool,
runningInUserns bool) e
}
args = append(args, "-C", path, "--numeric-owner",
"--xattrs-include=*")
args = append(args, extractArgs...)
- args = append(args, file)
+ args = append(args, "-")
+
+ f, err := os.Open(file)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ reader = f
+
+ // Attach the ProgressTracker if supplied.
+ if tracker != nil {
+ fsinfo, err := f.Stat()
+ if err != nil {
+ return err
+ }
+
+ tracker.Length = fsinfo.Size()
+ reader = &ioprogress.ProgressReader{
+ ReadCloser: f,
+ Tracker: tracker,
+ }
+ }
} else if strings.HasPrefix(extension, ".squashfs") {
+ // unsquashfs does not support reading from stdin,
+ // so ProgressTracker is not possible.
command = "unsquashfs"
args = append(args, "-f", "-d", path, "-n")
@@ -91,7 +117,7 @@ func Unpack(file string, path string, blockBackend bool,
runningInUserns bool) e
return fmt.Errorf("Unsupported image format: %s", extension)
}
- output, err := RunCommand(command, args...)
+ err = RunCommandWithFds(reader, nil, command, args...)
if err != nil {
// Check if we ran out of space
fs := syscall.Statfs_t{}
@@ -110,14 +136,9 @@ func Unpack(file string, path string, blockBackend bool,
runningInUserns bool) e
}
}
- co := output
logger.Debugf("Unpacking failed")
- logger.Debugf(co)
-
- // Truncate the output to a single line for inclusion in the
error
- // message. The first line isn't guaranteed to pinpoint the
issue,
- // but it's better than nothing and better than a multi-line
message.
- return fmt.Errorf("Unpack failed, %s. %s", err,
strings.SplitN(co, "\n", 2)[0])
+ logger.Debugf(err.Error())
+ return fmt.Errorf("Unpack failed, %s.", err)
}
return nil
_______________________________________________
lxc-devel mailing list
lxc-devel@lists.linuxcontainers.org
http://lists.linuxcontainers.org/listinfo/lxc-devel
