The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/4168
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === Move the functions `Unpack` and `DownloadFileSha256`, along with some dependencies, to shared.
From 37aeeaadad4599402ba68eb66aee9214453e8aa5 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <[email protected]> Date: Fri, 12 Jan 2018 12:27:57 +0100 Subject: [PATCH 1/2] lxd,shared: move archive functions to shared This moves the Unpack() function, as well as some of its dependent code to shared. Signed-off-by: Thomas Hipp <[email protected]> --- lxd/container_lxc.go | 6 +-- lxd/devices.go | 34 ------------- lxd/images.go | 122 +++-------------------------------------------- lxd/main_test.go | 2 +- lxd/patches.go | 16 +++---- lxd/storage.go | 66 +++++++++++-------------- lxd/storage_btrfs.go | 4 +- lxd/storage_ceph.go | 4 +- lxd/storage_dir.go | 4 +- lxd/storage_lvm.go | 4 +- lxd/storage_lvm_utils.go | 2 +- lxd/storage_migration.go | 2 +- lxd/storage_mock.go | 3 +- lxd/storage_shared.go | 4 +- lxd/storage_zfs.go | 4 +- shared/archive.go | 120 ++++++++++++++++++++++++++++++++++++++++++++++ shared/devices.go | 42 ++++++++++++++++ shared/storage.go | 13 +++++ 18 files changed, 236 insertions(+), 216 deletions(-) create mode 100644 shared/archive.go create mode 100644 shared/devices.go create mode 100644 shared/storage.go diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go index f72a1aedc..97ce25f66 100644 --- a/lxd/container_lxc.go +++ b/lxd/container_lxc.go @@ -1168,7 +1168,7 @@ func (c *containerLXC) initLXC(config bool) error { return err } - memoryTotal, err := deviceTotalMemory() + memoryTotal, err := shared.DeviceTotalMemory() if err != nil { return err } @@ -3846,7 +3846,7 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error { return err } - memoryTotal, err := deviceTotalMemory() + memoryTotal, err := shared.DeviceTotalMemory() if err != nil { return err } @@ -5807,7 +5807,7 @@ func (c *containerLXC) StorageStart() (bool, error) { isOurOperation, err := c.StorageStartSensitive() // Remove this as soon as zfs is fixed - if c.storage.GetStorageType() == storageTypeZfs && err == syscall.EBUSY { + if c.storage.GetStorageType() == shared.StorageTypeZfs && err == syscall.EBUSY { return isOurOperation, nil } diff --git a/lxd/devices.go b/lxd/devices.go index f29b2df24..ec11a7de9 100644 --- a/lxd/devices.go +++ b/lxd/devices.go @@ -1067,40 +1067,6 @@ func deviceParseCPU(cpuAllowance string, cpuPriority string) (string, string, st return fmt.Sprintf("%d", cpuShares), cpuCfsQuota, cpuCfsPeriod, nil } -func deviceTotalMemory() (int64, error) { - // Open /proc/meminfo - f, err := os.Open("/proc/meminfo") - if err != nil { - return -1, err - } - defer f.Close() - - // Read it line by line - scan := bufio.NewScanner(f) - for scan.Scan() { - line := scan.Text() - - // We only care about MemTotal - if !strings.HasPrefix(line, "MemTotal:") { - continue - } - - // Extract the before last (value) and last (unit) fields - fields := strings.Split(line, " ") - value := fields[len(fields)-2] + fields[len(fields)-1] - - // Feed the result to shared.ParseByteSizeString to get an int value - valueBytes, err := shared.ParseByteSizeString(value) - if err != nil { - return -1, err - } - - return valueBytes, nil - } - - return -1, fmt.Errorf("Couldn't find MemTotal") -} - func deviceGetParentBlocks(path string) ([]string, error) { var devices []string var device []string diff --git a/lxd/images.go b/lxd/images.go index 68fd51f6d..72d4a530b 100644 --- a/lxd/images.go +++ b/lxd/images.go @@ -17,7 +17,6 @@ import ( "strconv" "strings" "sync" - "syscall" "time" "github.com/gorilla/mux" @@ -47,117 +46,8 @@ import ( end for whichever finishes last. */ var imagePublishLock sync.Mutex -func detectCompression(fname string) ([]string, string, error) { - f, err := os.Open(fname) - if err != nil { - return []string{""}, "", err - } - defer f.Close() - - // read header parts to detect compression method - // bz2 - 2 bytes, 'BZ' signature/magic number - // gz - 2 bytes, 0x1f 0x8b - // lzma - 6 bytes, { [0x000, 0xE0], '7', 'z', 'X', 'Z', 0x00 } - - // xy - 6 bytes, header format { 0xFD, '7', 'z', 'X', 'Z', 0x00 } - // tar - 263 bytes, trying to get ustar from 257 - 262 - header := make([]byte, 263) - _, err = f.Read(header) - if err != nil { - return []string{""}, "", err - } - - switch { - case bytes.Equal(header[0:2], []byte{'B', 'Z'}): - return []string{"-jxf"}, ".tar.bz2", nil - case bytes.Equal(header[0:2], []byte{0x1f, 0x8b}): - return []string{"-zxf"}, ".tar.gz", nil - case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] == 0xFD): - return []string{"-Jxf"}, ".tar.xz", nil - case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] != 0xFD): - return []string{"--lzma", "-xf"}, ".tar.lzma", nil - case bytes.Equal(header[0:3], []byte{0x5d, 0x00, 0x00}): - return []string{"--lzma", "-xf"}, ".tar.lzma", nil - case bytes.Equal(header[257:262], []byte{'u', 's', 't', 'a', 'r'}): - return []string{"-xf"}, ".tar", nil - case bytes.Equal(header[0:4], []byte{'h', 's', 'q', 's'}): - return []string{""}, ".squashfs", nil - default: - return []string{""}, "", fmt.Errorf("Unsupported compression") - } - -} - -func unpack(file string, path string, sType storageType, runningInUserns bool) error { - extractArgs, extension, err := detectCompression(file) - if err != nil { - return err - } - - command := "" - args := []string{} - if strings.HasPrefix(extension, ".tar") { - command = "tar" - if runningInUserns { - args = append(args, "--wildcards") - args = append(args, "--exclude=dev/*") - args = append(args, "--exclude=./dev/*") - args = append(args, "--exclude=rootfs/dev/*") - args = append(args, "--exclude=rootfs/./dev/*") - } - args = append(args, "-C", path, "--numeric-owner") - args = append(args, extractArgs...) - args = append(args, file) - } else if strings.HasPrefix(extension, ".squashfs") { - command = "unsquashfs" - args = append(args, "-f", "-d", path, "-n") - - // Limit unsquashfs chunk size to 10% of memory and up to 256MB (default) - // When running on a low memory system, also disable multi-processing - mem, err := deviceTotalMemory() - mem = mem / 1024 / 1024 / 10 - if err == nil && mem < 256 { - args = append(args, "-da", fmt.Sprintf("%d", mem), "-fr", fmt.Sprintf("%d", mem), "-p", "1") - } - - args = append(args, file) - } else { - return fmt.Errorf("Unsupported image format: %s", extension) - } - - output, err := shared.RunCommand(command, args...) - if err != nil { - // Check if we ran out of space - fs := syscall.Statfs_t{} - - err1 := syscall.Statfs(path, &fs) - if err1 != nil { - return err1 - } - - // Check if we're running out of space - if int64(fs.Bfree) < int64(2*fs.Bsize) { - if sType == storageTypeLvm { - return fmt.Errorf("Unable to unpack image, run out of disk space (consider increasing your pool's volume.size).") - } else { - return fmt.Errorf("Unable to unpack image, run out of disk space.") - } - } - - co := output - logger.Debugf("Unpacking failed") - logger.Debugf(co) - - // Truncate the output to a single line for inclusion in the error - // message. The first line isn't guaranteed to pinpoint the issue, - // but it's better than nothing and better than a multi-line message. - return fmt.Errorf("Unpack failed, %s. %s", err, strings.SplitN(co, "\n", 2)[0]) - } - - return nil -} - -func unpackImage(imagefname string, destpath string, sType storageType, runningInUserns bool) error { - err := unpack(imagefname, destpath, sType, runningInUserns) +func unpackImage(imagefname string, destpath string, sType shared.StorageType, runningInUserns bool) error { + err := shared.Unpack(imagefname, destpath, sType, runningInUserns) if err != nil { return err } @@ -169,7 +59,7 @@ func unpackImage(imagefname string, destpath string, sType storageType, runningI return fmt.Errorf("Error creating rootfs directory") } - err = unpack(imagefname+".rootfs", rootfsPath, sType, runningInUserns) + err = shared.Unpack(imagefname+".rootfs", rootfsPath, sType, runningInUserns) if err != nil { return err } @@ -771,7 +661,7 @@ func imagesPost(d *Daemon, r *http.Request) Response { func getImageMetadata(fname string) (*api.ImageMetadata, error) { metadataName := "metadata.yaml" - compressionArgs, _, err := detectCompression(fname) + compressionArgs, _, err := shared.DetectCompression(fname) if err != nil { return nil, fmt.Errorf( @@ -1589,7 +1479,7 @@ func imageExport(d *Daemon, r *http.Request) Response { imagePath := shared.VarPath("images", imgInfo.Fingerprint) rootfsPath := imagePath + ".rootfs" - _, ext, err := detectCompression(imagePath) + _, ext, err := shared.DetectCompression(imagePath) if err != nil { ext = "" } @@ -1604,7 +1494,7 @@ func imageExport(d *Daemon, r *http.Request) Response { // Recompute the extension for the root filesystem, it may use a different // compression algorithm than the metadata. - _, ext, err = detectCompression(rootfsPath) + _, ext, err = shared.DetectCompression(rootfsPath) if err != nil { ext = "" } diff --git a/lxd/main_test.go b/lxd/main_test.go index 0f05ac700..4a76412a2 100644 --- a/lxd/main_test.go +++ b/lxd/main_test.go @@ -68,7 +68,7 @@ func (suite *lxdTestSuite) SetupTest() { // the next function. poolConfig := map[string]string{} - mockStorage, _ := storageTypeToString(storageTypeMock) + mockStorage, _ := storageTypeToString(shared.StorageTypeMock) // Create the database entry for the storage pool. poolDescription := fmt.Sprintf("%s storage pool", lxdTestSuiteDefaultStoragePool) _, err = dbStoragePoolCreateAndUpdateCache(suite.d.db, lxdTestSuiteDefaultStoragePool, poolDescription, mockStorage, poolConfig) diff --git a/lxd/patches.go b/lxd/patches.go index 79367d5a3..60ad7773f 100644 --- a/lxd/patches.go +++ b/lxd/patches.go @@ -166,16 +166,16 @@ func patchStorageApi(name string, d *Daemon) error { lvmVgName := daemonConfig["storage.lvm_vg_name"].Get() zfsPoolName := daemonConfig["storage.zfs_pool_name"].Get() defaultPoolName := "default" - preStorageApiStorageType := storageTypeDir + preStorageApiStorageType := shared.StorageTypeDir if lvmVgName != "" { - preStorageApiStorageType = storageTypeLvm + preStorageApiStorageType = shared.StorageTypeLvm defaultPoolName = lvmVgName } else if zfsPoolName != "" { - preStorageApiStorageType = storageTypeZfs + preStorageApiStorageType = shared.StorageTypeZfs defaultPoolName = zfsPoolName } else if d.os.BackingFS == "btrfs" { - preStorageApiStorageType = storageTypeBtrfs + preStorageApiStorageType = shared.StorageTypeBtrfs } else { // Dir storage pool. } @@ -229,13 +229,13 @@ func patchStorageApi(name string, d *Daemon) error { // If any of these are actually called, there's no way back. poolName := defaultPoolName switch preStorageApiStorageType { - case storageTypeBtrfs: + case shared.StorageTypeBtrfs: err = upgradeFromStorageTypeBtrfs(name, d, defaultPoolName, defaultStorageTypeName, cRegular, cSnapshots, imgPublic, imgPrivate) - case storageTypeDir: + case shared.StorageTypeDir: err = upgradeFromStorageTypeDir(name, d, defaultPoolName, defaultStorageTypeName, cRegular, cSnapshots, imgPublic, imgPrivate) - case storageTypeLvm: + case shared.StorageTypeLvm: err = upgradeFromStorageTypeLvm(name, d, defaultPoolName, defaultStorageTypeName, cRegular, cSnapshots, imgPublic, imgPrivate) - case storageTypeZfs: + case shared.StorageTypeZfs: // The user is using a zfs dataset. This case needs to be // handled with care: diff --git a/lxd/storage.go b/lxd/storage.go index 4a651d70e..99dacae79 100644 --- a/lxd/storage.go +++ b/lxd/storage.go @@ -75,53 +75,41 @@ func readStoragePoolDriversCache() map[string]string { return drivers.(map[string]string) } -// storageType defines the type of a storage -type storageType int - -const ( - storageTypeBtrfs storageType = iota - storageTypeCeph - storageTypeDir - storageTypeLvm - storageTypeMock - storageTypeZfs -) - var supportedStoragePoolDrivers = []string{"btrfs", "ceph", "dir", "lvm", "zfs"} -func storageTypeToString(sType storageType) (string, error) { +func storageTypeToString(sType shared.StorageType) (string, error) { switch sType { - case storageTypeBtrfs: + case shared.StorageTypeBtrfs: return "btrfs", nil - case storageTypeCeph: + case shared.StorageTypeCeph: return "ceph", nil - case storageTypeDir: + case shared.StorageTypeDir: return "dir", nil - case storageTypeLvm: + case shared.StorageTypeLvm: return "lvm", nil - case storageTypeMock: + case shared.StorageTypeMock: return "mock", nil - case storageTypeZfs: + case shared.StorageTypeZfs: return "zfs", nil } return "", fmt.Errorf("invalid storage type") } -func storageStringToType(sName string) (storageType, error) { +func storageStringToType(sName string) (shared.StorageType, error) { switch sName { case "btrfs": - return storageTypeBtrfs, nil + return shared.StorageTypeBtrfs, nil case "ceph": - return storageTypeCeph, nil + return shared.StorageTypeCeph, nil case "dir": - return storageTypeDir, nil + return shared.StorageTypeDir, nil case "lvm": - return storageTypeLvm, nil + return shared.StorageTypeLvm, nil case "mock": - return storageTypeMock, nil + return shared.StorageTypeMock, nil case "zfs": - return storageTypeZfs, nil + return shared.StorageTypeZfs, nil } return -1, fmt.Errorf("invalid storage type name") @@ -132,7 +120,7 @@ func storageStringToType(sName string) (storageType, error) { type storage interface { // Functions dealing with basic driver properties only. StorageCoreInit() error - GetStorageType() storageType + GetStorageType() shared.StorageType GetStorageTypeName() string GetStorageTypeVersion() string @@ -234,42 +222,42 @@ func storageCoreInit(driver string) (storage, error) { } switch sType { - case storageTypeBtrfs: + case shared.StorageTypeBtrfs: btrfs := storageBtrfs{} err = btrfs.StorageCoreInit() if err != nil { return nil, err } return &btrfs, nil - case storageTypeDir: + case shared.StorageTypeDir: dir := storageDir{} err = dir.StorageCoreInit() if err != nil { return nil, err } return &dir, nil - case storageTypeCeph: + case shared.StorageTypeCeph: ceph := storageCeph{} err = ceph.StorageCoreInit() if err != nil { return nil, err } return &ceph, nil - case storageTypeLvm: + case shared.StorageTypeLvm: lvm := storageLvm{} err = lvm.StorageCoreInit() if err != nil { return nil, err } return &lvm, nil - case storageTypeMock: + case shared.StorageTypeMock: mock := storageMock{} err = mock.StorageCoreInit() if err != nil { return nil, err } return &mock, nil - case storageTypeZfs: + case shared.StorageTypeZfs: zfs := storageZfs{} err = zfs.StorageCoreInit() if err != nil { @@ -310,7 +298,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType } switch sType { - case storageTypeBtrfs: + case shared.StorageTypeBtrfs: btrfs := storageBtrfs{} btrfs.poolID = poolID btrfs.pool = pool @@ -322,7 +310,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType return nil, err } return &btrfs, nil - case storageTypeDir: + case shared.StorageTypeDir: dir := storageDir{} dir.poolID = poolID dir.pool = pool @@ -334,7 +322,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType return nil, err } return &dir, nil - case storageTypeCeph: + case shared.StorageTypeCeph: ceph := storageCeph{} ceph.poolID = poolID ceph.pool = pool @@ -346,7 +334,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType return nil, err } return &ceph, nil - case storageTypeLvm: + case shared.StorageTypeLvm: lvm := storageLvm{} lvm.poolID = poolID lvm.pool = pool @@ -358,7 +346,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType return nil, err } return &lvm, nil - case storageTypeMock: + case shared.StorageTypeMock: mock := storageMock{} mock.poolID = poolID mock.pool = pool @@ -370,7 +358,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType return nil, err } return &mock, nil - case storageTypeZfs: + case shared.StorageTypeZfs: zfs := storageZfs{} zfs.poolID = poolID zfs.pool = pool diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go index 1218ee271..6cbe83532 100644 --- a/lxd/storage_btrfs.go +++ b/lxd/storage_btrfs.go @@ -61,7 +61,7 @@ func (s *storageBtrfs) getCustomSubvolumePath(poolName string) string { } func (s *storageBtrfs) StorageCoreInit() error { - s.sType = storageTypeBtrfs + s.sType = shared.StorageTypeBtrfs typeName, err := storageTypeToString(s.sType) if err != nil { return err @@ -1433,7 +1433,7 @@ func (s *storageBtrfs) ImageCreate(fingerprint string) error { // Unpack the image in imageMntPoint. imagePath := shared.VarPath("images", fingerprint) - err = unpackImage(imagePath, tmpImageSubvolumeName, storageTypeBtrfs, s.s.OS.RunningInUserNS) + err = unpackImage(imagePath, tmpImageSubvolumeName, shared.StorageTypeBtrfs, s.s.OS.RunningInUserNS) if err != nil { return err } diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go index 1af3cc410..5bc742f85 100644 --- a/lxd/storage_ceph.go +++ b/lxd/storage_ceph.go @@ -21,7 +21,7 @@ type storageCeph struct { } func (s *storageCeph) StorageCoreInit() error { - s.sType = storageTypeCeph + s.sType = shared.StorageTypeCeph typeName, err := storageTypeToString(s.sType) if err != nil { return err @@ -2368,7 +2368,7 @@ func (s *storageCeph) ImageCreate(fingerprint string) error { // rsync contents into image imagePath := shared.VarPath("images", fingerprint) - err = unpackImage(imagePath, imageMntPoint, storageTypeCeph, s.s.OS.RunningInUserNS) + err = unpackImage(imagePath, imageMntPoint, shared.StorageTypeCeph, s.s.OS.RunningInUserNS) if err != nil { logger.Errorf(`Failed to unpack image for RBD storage `+ `volume for image "%s" on storage pool "%s": %s`, diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go index 78e562fa7..afbb324a4 100644 --- a/lxd/storage_dir.go +++ b/lxd/storage_dir.go @@ -21,7 +21,7 @@ type storageDir struct { // Only initialize the minimal information we need about a given storage type. func (s *storageDir) StorageCoreInit() error { - s.sType = storageTypeDir + s.sType = shared.StorageTypeDir typeName, err := storageTypeToString(s.sType) if err != nil { return err @@ -509,7 +509,7 @@ func (s *storageDir) ContainerCreateFromImage(container container, imageFingerpr }() imagePath := shared.VarPath("images", imageFingerprint) - err = unpackImage(imagePath, containerMntPoint, storageTypeDir, s.s.OS.RunningInUserNS) + err = unpackImage(imagePath, containerMntPoint, shared.StorageTypeDir, s.s.OS.RunningInUserNS) if err != nil { return err } diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go index 95c596b1c..8866b0d17 100644 --- a/lxd/storage_lvm.go +++ b/lxd/storage_lvm.go @@ -25,7 +25,7 @@ type storageLvm struct { // Only initialize the minimal information we need about a given storage type. func (s *storageLvm) StorageCoreInit() error { - s.sType = storageTypeLvm + s.sType = shared.StorageTypeLvm typeName, err := storageTypeToString(s.sType) if err != nil { return err @@ -1627,7 +1627,7 @@ func (s *storageLvm) ImageCreate(fingerprint string) error { } imagePath := shared.VarPath("images", fingerprint) - err = unpackImage(imagePath, imageMntPoint, storageTypeLvm, s.s.OS.RunningInUserNS) + err = unpackImage(imagePath, imageMntPoint, shared.StorageTypeLvm, s.s.OS.RunningInUserNS) if err != nil { return err } diff --git a/lxd/storage_lvm_utils.go b/lxd/storage_lvm_utils.go index f9f3f1340..b71dcb1e7 100644 --- a/lxd/storage_lvm_utils.go +++ b/lxd/storage_lvm_utils.go @@ -461,7 +461,7 @@ func (s *storageLvm) containerCreateFromImageLv(c container, fp string) error { imagePath := shared.VarPath("images", fp) containerMntPoint := getContainerMountPoint(s.pool.Name, containerName) - err = unpackImage(imagePath, containerMntPoint, storageTypeLvm, s.s.OS.RunningInUserNS) + err = unpackImage(imagePath, containerMntPoint, shared.StorageTypeLvm, s.s.OS.RunningInUserNS) if err != nil { logger.Errorf(`Failed to unpack image "%s" into non-thinpool `+ `LVM storage volume "%s" for container "%s" on `+ diff --git a/lxd/storage_migration.go b/lxd/storage_migration.go index 76308e96d..57f122bb7 100644 --- a/lxd/storage_migration.go +++ b/lxd/storage_migration.go @@ -149,7 +149,7 @@ func rsyncMigrationSink(live bool, container container, snapshots []*Snapshot, c return fmt.Errorf("the container's root device is missing the pool property") } - isDirBackend := container.Storage().GetStorageType() == storageTypeDir + isDirBackend := container.Storage().GetStorageType() == shared.StorageTypeDir if isDirBackend { if !containerOnly { for _, snap := range snapshots { diff --git a/lxd/storage_mock.go b/lxd/storage_mock.go index aacf7cdc0..bef1f3dac 100644 --- a/lxd/storage_mock.go +++ b/lxd/storage_mock.go @@ -5,6 +5,7 @@ import ( "github.com/gorilla/websocket" + "github.com/lxc/lxd/shared" "github.com/lxc/lxd/shared/api" "github.com/lxc/lxd/shared/idmap" "github.com/lxc/lxd/shared/logger" @@ -15,7 +16,7 @@ type storageMock struct { } func (s *storageMock) StorageCoreInit() error { - s.sType = storageTypeMock + s.sType = shared.StorageTypeMock typeName, err := storageTypeToString(s.sType) if err != nil { return err diff --git a/lxd/storage_shared.go b/lxd/storage_shared.go index f2e1b692d..e4ab8c288 100644 --- a/lxd/storage_shared.go +++ b/lxd/storage_shared.go @@ -11,7 +11,7 @@ import ( ) type storageShared struct { - sType storageType + sType shared.StorageType sTypeName string sTypeVersion string @@ -24,7 +24,7 @@ type storageShared struct { volume *api.StorageVolume } -func (s *storageShared) GetStorageType() storageType { +func (s *storageShared) GetStorageType() shared.StorageType { return s.sType } diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go index 30a616b93..137621ad6 100644 --- a/lxd/storage_zfs.go +++ b/lxd/storage_zfs.go @@ -40,7 +40,7 @@ func (s *storageZfs) getOnDiskPoolName() string { // Only initialize the minimal information we need about a given storage type. func (s *storageZfs) StorageCoreInit() error { - s.sType = storageTypeZfs + s.sType = shared.StorageTypeZfs typeName, err := storageTypeToString(s.sType) if err != nil { return err @@ -1907,7 +1907,7 @@ func (s *storageZfs) ImageCreate(fingerprint string) error { } // Unpack the image into the temporary mountpoint. - err = unpackImage(imagePath, tmpImageDir, storageTypeZfs, s.s.OS.RunningInUserNS) + err = unpackImage(imagePath, tmpImageDir, shared.StorageTypeZfs, s.s.OS.RunningInUserNS) if err != nil { return err } diff --git a/shared/archive.go b/shared/archive.go new file mode 100644 index 000000000..cc576a727 --- /dev/null +++ b/shared/archive.go @@ -0,0 +1,120 @@ +package shared + +import ( + "bytes" + "fmt" + "os" + "strings" + "syscall" + + "github.com/lxc/lxd/shared/logger" +) + +func DetectCompression(fname string) ([]string, string, error) { + f, err := os.Open(fname) + if err != nil { + return []string{""}, "", err + } + defer f.Close() + + // read header parts to detect compression method + // bz2 - 2 bytes, 'BZ' signature/magic number + // gz - 2 bytes, 0x1f 0x8b + // lzma - 6 bytes, { [0x000, 0xE0], '7', 'z', 'X', 'Z', 0x00 } - + // xy - 6 bytes, header format { 0xFD, '7', 'z', 'X', 'Z', 0x00 } + // tar - 263 bytes, trying to get ustar from 257 - 262 + header := make([]byte, 263) + _, err = f.Read(header) + if err != nil { + return []string{""}, "", err + } + + switch { + case bytes.Equal(header[0:2], []byte{'B', 'Z'}): + return []string{"-jxf"}, ".tar.bz2", nil + case bytes.Equal(header[0:2], []byte{0x1f, 0x8b}): + return []string{"-zxf"}, ".tar.gz", nil + case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] == 0xFD): + return []string{"-Jxf"}, ".tar.xz", nil + case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] != 0xFD): + return []string{"--lzma", "-xf"}, ".tar.lzma", nil + case bytes.Equal(header[0:3], []byte{0x5d, 0x00, 0x00}): + return []string{"--lzma", "-xf"}, ".tar.lzma", nil + case bytes.Equal(header[257:262], []byte{'u', 's', 't', 'a', 'r'}): + return []string{"-xf"}, ".tar", nil + case bytes.Equal(header[0:4], []byte{'h', 's', 'q', 's'}): + return []string{""}, ".squashfs", nil + default: + return []string{""}, "", fmt.Errorf("Unsupported compression") + } + +} + +func Unpack(file string, path string, sType StorageType, runningInUserns bool) error { + extractArgs, extension, err := DetectCompression(file) + if err != nil { + return err + } + + command := "" + args := []string{} + if strings.HasPrefix(extension, ".tar") { + command = "tar" + if runningInUserns { + args = append(args, "--wildcards") + args = append(args, "--exclude=dev/*") + args = append(args, "--exclude=./dev/*") + args = append(args, "--exclude=rootfs/dev/*") + args = append(args, "--exclude=rootfs/./dev/*") + } + args = append(args, "-C", path, "--numeric-owner") + args = append(args, extractArgs...) + args = append(args, file) + } else if strings.HasPrefix(extension, ".squashfs") { + command = "unsquashfs" + args = append(args, "-f", "-d", path, "-n") + + // Limit unsquashfs chunk size to 10% of memory and up to 256MB (default) + // When running on a low memory system, also disable multi-processing + mem, err := DeviceTotalMemory() + mem = mem / 1024 / 1024 / 10 + if err == nil && mem < 256 { + args = append(args, "-da", fmt.Sprintf("%d", mem), "-fr", fmt.Sprintf("%d", mem), "-p", "1") + } + + args = append(args, file) + } else { + return fmt.Errorf("Unsupported image format: %s", extension) + } + + output, err := RunCommand(command, args...) + if err != nil { + // Check if we ran out of space + fs := syscall.Statfs_t{} + + err1 := syscall.Statfs(path, &fs) + if err1 != nil { + return err1 + } + + // Check if we're running out of space + if int64(fs.Bfree) < int64(2*fs.Bsize) { + if sType == StorageTypeLvm { + return fmt.Errorf("Unable to unpack image, run out of disk space (consider increasing your pool's volume.size).") + } else { + return fmt.Errorf("Unable to unpack image, run out of disk space.") + } + } + + co := output + logger.Debugf("Unpacking failed") + logger.Debugf(co) + + // Truncate the output to a single line for inclusion in the error + // message. The first line isn't guaranteed to pinpoint the issue, + // but it's better than nothing and better than a multi-line message. + return fmt.Errorf("Unpack failed, %s. %s", err, strings.SplitN(co, "\n", 2)[0]) + } + + return nil +} diff --git a/shared/devices.go b/shared/devices.go new file mode 100644 index 000000000..df281970a --- /dev/null +++ b/shared/devices.go @@ -0,0 +1,42 @@ +package shared + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +func DeviceTotalMemory() (int64, error) { + // Open /proc/meminfo + f, err := os.Open("/proc/meminfo") + if err != nil { + return -1, err + } + defer f.Close() + + // Read it line by line + scan := bufio.NewScanner(f) + for scan.Scan() { + line := scan.Text() + + // We only care about MemTotal + if !strings.HasPrefix(line, "MemTotal:") { + continue + } + + // Extract the before last (value) and last (unit) fields + fields := strings.Split(line, " ") + value := fields[len(fields)-2] + fields[len(fields)-1] + + // Feed the result to shared.ParseByteSizeString to get an int value + valueBytes, err := ParseByteSizeString(value) + if err != nil { + return -1, err + } + + return valueBytes, nil + } + + return -1, fmt.Errorf("Couldn't find MemTotal") +} diff --git a/shared/storage.go b/shared/storage.go new file mode 100644 index 000000000..d13209fd5 --- /dev/null +++ b/shared/storage.go @@ -0,0 +1,13 @@ +package shared + +// StorageType defines the type of a storage +type StorageType int + +const ( + StorageTypeBtrfs StorageType = iota + StorageTypeCeph + StorageTypeDir + StorageTypeLvm + StorageTypeMock + StorageTypeZfs +) From 1c8d4cc81b18c8d37dd14393851716c6a163e0c7 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <[email protected]> Date: Fri, 12 Jan 2018 14:37:51 +0100 Subject: [PATCH 2/2] *: move download function to shared This moves the DownloadFileSha256 function and its dependent code to shared. Signed-off-by: Thomas Hipp <[email protected]> --- client/interfaces.go | 20 ++------------ client/lxd_images.go | 4 +-- client/simplestreams_images.go | 4 +-- client/util.go | 62 ------------------------------------------ lxc/utils.go | 3 +- lxd/daemon_images.go | 4 +-- shared/util.go | 62 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 73 insertions(+), 86 deletions(-) diff --git a/client/interfaces.go b/client/interfaces.go index b88fbef2d..6bfc245f0 100644 --- a/client/interfaces.go +++ b/client/interfaces.go @@ -6,6 +6,7 @@ import ( "github.com/gorilla/websocket" + "github.com/lxc/lxd/shared" "github.com/lxc/lxd/shared/api" "github.com/lxc/lxd/shared/cancel" ) @@ -172,21 +173,6 @@ type ConnectionInfo struct { Protocol string } -// The ProgressData struct represents new progress information on an operation -type ProgressData struct { - // Preferred string repreentation of progress (always set) - Text string - - // Progress in percent - Percentage int - - // Number of bytes transferred (for files) - TransferredBytes int64 - - // Total number of bytes (for files) - TotalBytes int64 -} - // The ImageCreateArgs struct is used for direct image upload type ImageCreateArgs struct { // Reader for the meta file @@ -202,7 +188,7 @@ type ImageCreateArgs struct { RootfsName string // Progress handler (called with upload progress) - ProgressHandler func(progress ProgressData) + ProgressHandler func(progress shared.ProgressData) } // The ImageFileRequest struct is used for an image download request @@ -214,7 +200,7 @@ type ImageFileRequest struct { RootfsFile io.WriteSeeker // Progress handler (called whenever some progress is made) - ProgressHandler func(progress ProgressData) + ProgressHandler func(progress shared.ProgressData) // A canceler that can be used to interrupt some part of the image download request Canceler *cancel.Canceler diff --git a/client/lxd_images.go b/client/lxd_images.go index 8d0889640..385ad04de 100644 --- a/client/lxd_images.go +++ b/client/lxd_images.go @@ -145,7 +145,7 @@ func (r *ProtocolLXD) GetPrivateImageFile(fingerprint string, secret string, req Tracker: &ioprogress.ProgressTracker{ Length: response.ContentLength, Handler: func(percent int64, speed int64) { - req.ProgressHandler(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))}) + req.ProgressHandler(shared.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))}) }, }, } @@ -363,7 +363,7 @@ func (r *ProtocolLXD) CreateImage(image api.ImagesPost, args *ImageCreateArgs) ( Tracker: &ioprogress.ProgressTracker{ Length: size, Handler: func(percent int64, speed int64) { - args.ProgressHandler(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))}) + args.ProgressHandler(shared.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))}) }, }, } diff --git a/client/simplestreams_images.go b/client/simplestreams_images.go index 39a1f5e87..065bedabe 100644 --- a/client/simplestreams_images.go +++ b/client/simplestreams_images.go @@ -67,11 +67,11 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe // Try over http url := fmt.Sprintf("http://%s/%s";, strings.TrimPrefix(r.httpHost, "https://";), path) - size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target) + size, err := shared.DownloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target) if err != nil { // Try over https url = fmt.Sprintf("%s/%s", r.httpHost, path) - size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target) + size, err = shared.DownloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target) if err != nil { return -1, err } diff --git a/client/util.go b/client/util.go index e041fd979..256a184f8 100644 --- a/client/util.go +++ b/client/util.go @@ -1,16 +1,12 @@ package lxd import ( - "crypto/sha256" - "fmt" "io" "net" "net/http" "net/url" "github.com/lxc/lxd/shared" - "github.com/lxc/lxd/shared/cancel" - "github.com/lxc/lxd/shared/ioprogress" ) func tlsHTTPClient(client *http.Client, tlsClientCert string, tlsClientKey string, tlsCA string, tlsServerCert string, insecureSkipVerify bool, proxy func(req *http.Request) (*url.URL, error)) (*http.Client, error) { @@ -84,64 +80,6 @@ func unixHTTPClient(client *http.Client, path string) (*http.Client, error) { return client, nil } -func downloadFileSha256(httpClient *http.Client, useragent string, progress func(progress ProgressData), canceler *cancel.Canceler, filename string, url string, hash string, target io.WriteSeeker) (int64, error) { - // Always seek to the beginning - target.Seek(0, 0) - - // Prepare the download request - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return -1, err - } - - if useragent != "" { - req.Header.Set("User-Agent", useragent) - } - - // Perform the request - r, doneCh, err := cancel.CancelableDownload(canceler, httpClient, req) - if err != nil { - return -1, err - } - defer r.Body.Close() - defer close(doneCh) - - if r.StatusCode != http.StatusOK { - return -1, fmt.Errorf("Unable to fetch %s: %s", url, r.Status) - } - - // Handle the data - body := r.Body - if progress != nil { - body = &ioprogress.ProgressReader{ - ReadCloser: r.Body, - Tracker: &ioprogress.ProgressTracker{ - Length: r.ContentLength, - Handler: func(percent int64, speed int64) { - if filename != "" { - progress(ProgressData{Text: fmt.Sprintf("%s: %d%% (%s/s)", filename, percent, shared.GetByteSizeString(speed, 2))}) - } else { - progress(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))}) - } - }, - }, - } - } - - sha256 := sha256.New() - size, err := io.Copy(io.MultiWriter(target, sha256), body) - if err != nil { - return -1, err - } - - result := fmt.Sprintf("%x", sha256.Sum(nil)) - if result != hash { - return -1, fmt.Errorf("Hash mismatch for %s: %s != %s", url, result, hash) - } - - return size, nil -} - type nullReadWriteCloser int func (nullReadWriteCloser) Close() error { return nil } diff --git a/lxc/utils.go b/lxc/utils.go index 0215bd394..358269841 100644 --- a/lxc/utils.go +++ b/lxc/utils.go @@ -13,6 +13,7 @@ import ( "time" "github.com/lxc/lxd/client" + "github.com/lxc/lxd/shared" "github.com/lxc/lxd/shared/api" "github.com/lxc/lxd/shared/i18n" ) @@ -119,7 +120,7 @@ func (p *ProgressRenderer) Warn(status string, timeout time.Duration) { fmt.Print(msg) } -func (p *ProgressRenderer) UpdateProgress(progress lxd.ProgressData) { +func (p *ProgressRenderer) UpdateProgress(progress shared.ProgressData) { p.Update(progress.Text) } diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go index abc81a965..ca5d49f16 100644 --- a/lxd/daemon_images.go +++ b/lxd/daemon_images.go @@ -343,7 +343,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce defer cleanup() // Setup a progress handler - progress := func(progress lxd.ProgressData) { + progress := func(progress shared.ProgressData) { if op == nil { return } @@ -462,7 +462,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce Tracker: &ioprogress.ProgressTracker{ Length: raw.ContentLength, Handler: func(percent int64, speed int64) { - progress(lxd.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))}) + progress(shared.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))}) }, }, } diff --git a/shared/util.go b/shared/util.go index e039905f0..f262c3aca 100644 --- a/shared/util.go +++ b/shared/util.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "crypto/rand" + "crypto/sha256" "encoding/gob" "encoding/hex" "encoding/json" @@ -24,6 +25,9 @@ import ( "strings" "time" "unicode" + + "github.com/lxc/lxd/shared/cancel" + "github.com/lxc/lxd/shared/ioprogress" ) const SnapshotDelimiter = "/" @@ -902,3 +906,61 @@ func EscapePathFstab(path string) string { "\\", "\\\\") return r.Replace(path) } + +func DownloadFileSha256(httpClient *http.Client, useragent string, progress func(progress ProgressData), canceler *cancel.Canceler, filename string, url string, hash string, target io.WriteSeeker) (int64, error) { + // Always seek to the beginning + target.Seek(0, 0) + + // Prepare the download request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return -1, err + } + + if useragent != "" { + req.Header.Set("User-Agent", useragent) + } + + // Perform the request + r, doneCh, err := cancel.CancelableDownload(canceler, httpClient, req) + if err != nil { + return -1, err + } + defer r.Body.Close() + defer close(doneCh) + + if r.StatusCode != http.StatusOK { + return -1, fmt.Errorf("Unable to fetch %s: %s", url, r.Status) + } + + // Handle the data + body := r.Body + if progress != nil { + body = &ioprogress.ProgressReader{ + ReadCloser: r.Body, + Tracker: &ioprogress.ProgressTracker{ + Length: r.ContentLength, + Handler: func(percent int64, speed int64) { + if filename != "" { + progress(ProgressData{Text: fmt.Sprintf("%s: %d%% (%s/s)", filename, percent, GetByteSizeString(speed, 2))}) + } else { + progress(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, GetByteSizeString(speed, 2))}) + } + }, + }, + } + } + + sha256 := sha256.New() + size, err := io.Copy(io.MultiWriter(target, sha256), body) + if err != nil { + return -1, err + } + + result := fmt.Sprintf("%x", sha256.Sum(nil)) + if result != hash { + return -1, fmt.Errorf("Hash mismatch for %s: %s != %s", url, result, hash) + } + + return size, nil +}
_______________________________________________ lxc-devel mailing list [email protected] http://lists.linuxcontainers.org/listinfo/lxc-devel
