The following pull request was submitted through Github.
It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/5451
This e-mail was sent by the LXC bot, direct replies will not reach the author
unless they happen to be subscribed to this list.
=== Description (from pull-request) ===
From 20a55a17baf3933d10ca7aee009d54b8365cec7e Mon Sep 17 00:00:00 2001
From: Joel Hockey <joelhoc...@chromium.org>
Date: Thu, 31 Jan 2019 18:52:24 -0800
Subject: [PATCH 1/2] lxd/images: change compressFile to take io.Reader and
io.Writer
This is part 1 of a series of patches to add better progress
tracking support for export and import.
By using Reader and Writer rather than filename for compressing
the caller can provide a tracking reader/writer for progress.
Signed-off-by: Joel Hockey <joelhoc...@chromium.org>
---
lxd/backup.go | 16 ++++++++++++++--
lxd/images.go | 35 ++++++++++++++++++-----------------
lxd/patches.go | 16 ++++++++++++++--
3 files changed, 46 insertions(+), 21 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index 46a53a3014..e469a19429 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -353,7 +353,19 @@ func backupCreateTarball(s *state.State, path string,
backup backup) error {
}
if compress != "none" {
- compressedPath, err := compressFile(backupPath, compress)
+ infile, err := os.Open(backupPath)
+ if err != nil {
+ return err
+ }
+ defer infile.Close()
+
+ compressed, err := os.Create(backupPath + ".compressed")
+ if err != nil {
+ return err
+ }
+ defer compressed.Close()
+
+ err = compressFile(compress, infile, compressed)
if err != nil {
return err
}
@@ -363,7 +375,7 @@ func backupCreateTarball(s *state.State, path string,
backup backup) error {
return err
}
- err = os.Rename(compressedPath, backupPath)
+ err = os.Rename(compressed.Name(), backupPath)
if err != nil {
return err
}
diff --git a/lxd/images.go b/lxd/images.go
index a27f4f5620..ea8c2cc69f 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -129,7 +129,7 @@ func unpackImage(imagefname string, destpath string, sType
storageType, runningI
return nil
}
-func compressFile(path string, compress string) (string, error) {
+func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
reproducible := []string{"gzip"}
args := []string{"-c"}
@@ -137,24 +137,11 @@ func compressFile(path string, compress string) (string,
error) {
args = append(args, "-n")
}
- args = append(args, path)
cmd := exec.Command(compress, args...)
-
- outfile, err := os.Create(path + ".compressed")
- if err != nil {
- return "", err
- }
-
- defer outfile.Close()
+ cmd.Stdin = infile
cmd.Stdout = outfile
- err = cmd.Run()
- if err != nil {
- os.Remove(outfile.Name())
- return "", err
- }
-
- return outfile.Name(), nil
+ return cmd.Run()
}
/*
@@ -223,7 +210,21 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
}
if compress != "none" {
- compressedPath, err = compressFile(tarfile.Name(), compress)
+ tarfile, err = os.Open(tarfile.Name())
+ if err != nil {
+ return nil, err
+ }
+ defer tarfile.Close()
+
+ compressedPath = tarfile.Name() + ".compressed"
+
+ compressed, err := os.Create(compressedPath)
+ if err != nil {
+ return nil, err
+ }
+ defer compressed.Close()
+
+ err = compressFile(compress, tarfile, compressed)
if err != nil {
return nil, err
}
diff --git a/lxd/patches.go b/lxd/patches.go
index 235410e66c..93b2ab8e5f 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -3154,7 +3154,19 @@ func patchMoveBackups(name string, d *Daemon) error {
}
// Compress it
- compressedPath, err := compressFile(backupPath,
"xz")
+ infile, err := os.Open(backupPath)
+ if err != nil {
+ return err
+ }
+ defer infile.Close()
+
+ compressed, err := os.Create(backupPath +
".compressed")
+ if err != nil {
+ return err
+ }
+ defer compressed.Close()
+
+ err = compressFile("xz", infile, compressed)
if err != nil {
return err
}
@@ -3164,7 +3176,7 @@ func patchMoveBackups(name string, d *Daemon) error {
return err
}
- err = os.Rename(compressedPath, backupPath)
+ err = os.Rename(compressed.Name(), backupPath)
if err != nil {
return err
}
From a7723ab209169960b503f3ee789bb2f36050d21b Mon Sep 17 00:00:00 2001
From: Joel Hockey <joelhoc...@chromium.org>
Date: Thu, 31 Jan 2019 19:03:24 -0800
Subject: [PATCH 2/2] lxd/images: calculate sha256 as image is written
This is part 2 of a series of patches to add better progress
tracking support for export and import.
Calculate sha256 as either tarfile is written (if no compression)
or as compression is done to improve performance.
Signed-off-by: Joel Hockey <joelhoc...@chromium.org>
---
lxd/images.go | 46 ++++++++++++++++++++++++++--------------------
1 file changed, 26 insertions(+), 20 deletions(-)
diff --git a/lxd/images.go b/lxd/images.go
index ea8c2cc69f..e32488f5b9 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -191,14 +191,10 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
}
defer os.Remove(tarfile.Name())
- if err := c.Export(tarfile, req.Properties); err != nil {
- tarfile.Close()
- return nil, err
- }
- tarfile.Close()
-
+ sha256 := sha256.New()
var compressedPath string
var compress string
+ var writer io.Writer
if req.CompressionAlgorithm != "" {
compress = req.CompressionAlgorithm
@@ -208,8 +204,23 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
return nil, err
}
}
+ usingCompression := compress != "none"
- if compress != "none" {
+ // If there is no compression, then calculate sha256 on tarfile
+ if usingCompression {
+ writer = tarfile
+ } else {
+ writer = io.MultiWriter(tarfile, sha256)
+ compressedPath = tarfile.Name()
+ }
+
+ if err := c.Export(writer, req.Properties); err != nil {
+ tarfile.Close()
+ return nil, err
+ }
+ tarfile.Close()
+
+ if usingCompression {
tarfile, err = os.Open(tarfile.Name())
if err != nil {
return nil, err
@@ -222,29 +233,24 @@ func imgPostContInfo(d *Daemon, r *http.Request, req
api.ImagesPost, builddir st
if err != nil {
return nil, err
}
+
defer compressed.Close()
+ defer os.Remove(compressed.Name())
- err = compressFile(compress, tarfile, compressed)
+ // Calculate sha256 as we compress
+ writer := io.MultiWriter(compressed, sha256)
+
+ err = compressFile(compress, tarfile, writer)
if err != nil {
return nil, err
}
- } else {
- compressedPath = tarfile.Name()
}
- defer os.Remove(compressedPath)
- sha256 := sha256.New()
- tarf, err := os.Open(compressedPath)
- if err != nil {
- return nil, err
- }
-
- info.Size, err = io.Copy(sha256, tarf)
- tarf.Close()
+ fi, err := os.Stat(compressedPath)
if err != nil {
return nil, err
}
-
+ info.Size = fi.Size()
info.Fingerprint = fmt.Sprintf("%x", sha256.Sum(nil))
_, _, err = d.cluster.ImageGet(project, info.Fingerprint, false, true)
_______________________________________________
lxc-devel mailing list
lxc-devel@lists.linuxcontainers.org
http://lists.linuxcontainers.org/listinfo/lxc-devel
