The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/3613
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === Signed-off-by: Christian Brauner <[email protected]>
From d854abfb089537d1bbd6bfd890a6fd84c812eac8 Mon Sep 17 00:00:00 2001 From: Christian Brauner <[email protected]> Date: Mon, 31 Jul 2017 21:08:04 +0200 Subject: [PATCH] ceph: use "/dev/rbd<idx" via sysfs Signed-off-by: Christian Brauner <[email protected]> --- lxd/storage_ceph.go | 75 ++++++++++++--------------- lxd/storage_ceph_utils.go | 126 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 128 insertions(+), 73 deletions(-) diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go index b721cdca9..bee2775e6 100644 --- a/lxd/storage_ceph.go +++ b/lxd/storage_ceph.go @@ -371,8 +371,8 @@ func (s *storageCeph) StoragePoolVolumeCreate() error { } }() - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, s.volume.Name, - storagePoolVolumeTypeNameCustom, s.UserName) + RBDDevPath, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, + s.volume.Name, storagePoolVolumeTypeNameCustom, s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for "%s" on `+ `storage pool "%s": %s`, s.volume.Name, s.pool.Name, err) @@ -401,14 +401,6 @@ func (s *storageCeph) StoragePoolVolumeCreate() error { `"%s" on storage pool "%s"`, RBDFilesystem, s.volume.Name, s.pool.Name) - // get rbd device path - RBDDevPath := getRBDDevPath( - s.OSDPoolName, - storagePoolVolumeTypeNameCustom, - s.volume.Name) - logger.Debugf(`Retrieved device path "%s" of RBD storage volume "%s" `+ - `on storage pool "%s"`, RBDDevPath, s.volume.Name, s.pool.Name) - msg, err := makeFSType(RBDDevPath, RBDFilesystem) if err != nil { logger.Errorf(`Failed to create filesystem type "%s" on `+ @@ -526,10 +518,6 @@ func (s *storageCeph) StoragePoolVolumeMount() (bool, error) { s.volume.Name, s.pool.Name) RBDFilesystem := s.getRBDFilesystem() - RBDDevPath := getRBDDevPath( - s.OSDPoolName, - storagePoolVolumeTypeNameCustom, - s.volume.Name) volumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name) customMountLockID := getCustomMountLockID(s.pool.Name, s.volume.Name) @@ -551,9 +539,14 @@ func (s *storageCeph) StoragePoolVolumeMount() (bool, error) { lxdStorageOngoingOperationMap[customMountLockID] = make(chan bool) lxdStorageMapLock.Unlock() + var ret int var customerr error ourMount := false + RBDDevPath := "" if !shared.IsMountPoint(volumeMntPoint) { + RBDDevPath, ret = getRBDMappedDevPath(s.ClusterName, s.OSDPoolName, + storagePoolVolumeTypeNameCustom, s.volume.Name, true, + s.UserName) mountFlags, mountOptions := lxdResolveMountoptions(s.getRBDMountOptions()) customerr = tryMount( RBDDevPath, @@ -571,7 +564,7 @@ func (s *storageCeph) StoragePoolVolumeMount() (bool, error) { } lxdStorageMapLock.Unlock() - if customerr != nil { + if customerr != nil || ret < 0 { logger.Errorf(`Failed to mount RBD storage volume "%s" on `+ `storage pool "%s": %s`, s.volume.Name, s.pool.Name, customerr) @@ -709,8 +702,8 @@ func (s *storageCeph) ContainerCreate(container container) error { } }() - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, containerName, - storagePoolVolumeTypeNameContainer, s.UserName) + RBDDevPath, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, + containerName, storagePoolVolumeTypeNameContainer, s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for `+ `container "%s" on storage pool "%s": %s`, @@ -741,15 +734,6 @@ func (s *storageCeph) ContainerCreate(container container) error { `for container "%s" on storage pool "%s"`, RBDFilesystem, containerName, s.pool.Name) - // get rbd device path - RBDDevPath := getRBDDevPath( - s.OSDPoolName, - storagePoolVolumeTypeNameContainer, - containerName) - logger.Debugf(`Retrieved device path "%s" of RBD storage volume `+ - `for container "%s" on storage pool "%s"`, RBDDevPath, - containerName, s.pool.Name) - msg, err := makeFSType(RBDDevPath, RBDFilesystem) if err != nil { logger.Errorf(`Failed to create filesystem type "%s" on `+ @@ -876,7 +860,7 @@ func (s *storageCeph) ContainerCreateFromImage(container container, fingerprint } }() - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, containerName, + _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, containerName, storagePoolVolumeTypeNameContainer, s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for container `+ @@ -1293,7 +1277,7 @@ func (s *storageCeph) ContainerCopy(target container, source container, // registered one above for the dummy volume we created. // map the container's volume - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, + _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName, storagePoolVolumeTypeNameContainer, s.UserName) if err != nil { @@ -1339,7 +1323,6 @@ func (s *storageCeph) ContainerMount(c container) (bool, error) { logger.Debugf("Mounting RBD storage volume for container \"%s\" on storage pool \"%s\".", s.volume.Name, s.pool.Name) RBDFilesystem := s.getRBDFilesystem() - RBDDevPath := getRBDDevPath(s.OSDPoolName, storagePoolVolumeTypeNameContainer, name) containerMntPoint := getContainerMountPoint(s.pool.Name, name) if shared.IsSnapshot(name) { containerMntPoint = getSnapshotMountPoint(s.pool.Name, name) @@ -1361,11 +1344,17 @@ func (s *storageCeph) ContainerMount(c container) (bool, error) { lxdStorageOngoingOperationMap[containerMountLockID] = make(chan bool) lxdStorageMapLock.Unlock() + var ret int var mounterr error ourMount := false + RBDDevPath := "" if !shared.IsMountPoint(containerMntPoint) { + RBDDevPath, ret = getRBDMappedDevPath(s.ClusterName, + s.OSDPoolName, storagePoolVolumeTypeNameContainer, + name, true, s.UserName) mountFlags, mountOptions := lxdResolveMountoptions(s.getRBDMountOptions()) - mounterr = tryMount(RBDDevPath, containerMntPoint, RBDFilesystem, mountFlags, mountOptions) + mounterr = tryMount(RBDDevPath, containerMntPoint, + RBDFilesystem, mountFlags, mountOptions) ourMount = true } @@ -1376,7 +1365,7 @@ func (s *storageCeph) ContainerMount(c container) (bool, error) { } lxdStorageMapLock.Unlock() - if mounterr != nil { + if mounterr != nil || ret < 0 { logger.Errorf("Failed to mount RBD storage volume for container \"%s\": %s", s.volume.Name, mounterr) return false, mounterr } @@ -1464,8 +1453,8 @@ func (s *storageCeph) ContainerRename(c container, newName string) error { return } - err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, oldName, - storagePoolVolumeTypeNameContainer, s.UserName) + _, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, + oldName, storagePoolVolumeTypeNameContainer, s.UserName) if err != nil { logger.Warnf(`Failed to Map RBD storage volume `+ `for container "%s": %s`, oldName, err) @@ -1499,7 +1488,7 @@ func (s *storageCeph) ContainerRename(c container, newName string) error { }() // map - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, newName, + _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, newName, storagePoolVolumeTypeNameContainer, s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for `+ @@ -1934,8 +1923,8 @@ func (s *storageCeph) ContainerSnapshotStart(c container) (bool, error) { }() // map - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, cloneName, - "snapshots", s.UserName) + RBDDevPath, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, + cloneName, "snapshots", s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for `+ `container "%s" on storage pool "%s": %s`, @@ -1961,7 +1950,6 @@ func (s *storageCeph) ContainerSnapshotStart(c container) (bool, error) { containerMntPoint := getSnapshotMountPoint(s.pool.Name, containerName) RBDFilesystem := s.getRBDFilesystem() - RBDDevPath := getRBDDevPath(s.OSDPoolName, "snapshots", cloneName) mountFlags, mountOptions := lxdResolveMountoptions(s.getRBDMountOptions()) err = tryMount( RBDDevPath, @@ -2123,8 +2111,9 @@ func (s *storageCeph) ImageCreate(fingerprint string) error { } }() - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, - fingerprint, storagePoolVolumeTypeNameImage, s.UserName) + RBDDevPath, err := cephRBDVolumeMap(s.ClusterName, + s.OSDPoolName, fingerprint, + storagePoolVolumeTypeNameImage, s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for `+ `image "%s" on storage pool "%s": %s`, @@ -2152,9 +2141,6 @@ func (s *storageCeph) ImageCreate(fingerprint string) error { // get filesystem RBDFilesystem := s.getRBDFilesystem() - // get rbd device path - RBDDevPath := getRBDDevPath(s.OSDPoolName, - storagePoolVolumeTypeNameImage, fingerprint) msg, err := makeFSType(RBDDevPath, RBDFilesystem) if err != nil { logger.Errorf(`Failed to create filesystem "%s" for RBD `+ @@ -2456,9 +2442,10 @@ func (s *storageCeph) ImageMount(fingerprint string) (bool, error) { RBDFilesystem := s.getRBDFilesystem() RBDMountOptions := s.getRBDMountOptions() mountFlags, mountOptions := lxdResolveMountoptions(RBDMountOptions) - RBDDevPath := getRBDDevPath(s.OSDPoolName, storagePoolVolumeTypeNameImage, fingerprint) + RBDDevPath, ret := getRBDMappedDevPath(s.ClusterName, s.OSDPoolName, + storagePoolVolumeTypeNameImage, fingerprint, true, s.UserName) err := tryMount(RBDDevPath, imageMntPoint, RBDFilesystem, mountFlags, mountOptions) - if err != nil { + if err != nil || ret < 0 { logger.Errorf("Failed to mount RBD device %s onto %s: %s", RBDDevPath, imageMntPoint, err) return false, err } diff --git a/lxd/storage_ceph_utils.go b/lxd/storage_ceph_utils.go index 96785436c..1ce1e3f4f 100644 --- a/lxd/storage_ceph_utils.go +++ b/lxd/storage_ceph_utils.go @@ -3,8 +3,10 @@ package main import ( "encoding/json" "fmt" + "io/ioutil" "os" "os/exec" + "strconv" "strings" "syscall" @@ -56,17 +58,6 @@ func cephOSDPoolDestroy(clusterName string, poolName string, userName string) er return nil } -// getRBDDevPath constructs the path to a RBD block device. -// Note that for this path to be valid the corresponding volume has to be mapped -// first. -func getRBDDevPath(poolName string, volumeType string, volumeName string) string { - if volumeType == "" { - return fmt.Sprintf("/dev/rbd/%s/%s", poolName, volumeName) - } - - return fmt.Sprintf("/dev/rbd/%s/%s_%s", poolName, volumeType, volumeName) -} - // cephRBDVolumeCreate creates an RBD storage volume. // Note that the set of features is intentionally limited is intentionally // limited by passing --image-feature explicitly. This is done to ensure that @@ -129,8 +120,8 @@ func cephRBDVolumeDelete(clusterName string, poolName string, volumeName string, // This will ensure that the RBD storage volume is accessible as a block device // in the /dev directory and is therefore necessary in order to mount it. func cephRBDVolumeMap(clusterName string, poolName string, volumeName string, - volumeType string, userName string) error { - _, err := shared.RunCommand( + volumeType string, userName string) (string, error) { + devPath, err := shared.RunCommand( "rbd", "--id", userName, "--cluster", clusterName, @@ -138,21 +129,10 @@ func cephRBDVolumeMap(clusterName string, poolName string, volumeName string, "map", fmt.Sprintf("%s_%s", volumeType, volumeName)) if err != nil { - runError, ok := err.(shared.RunError) - if ok { - exitError, ok := runError.Err.(*exec.ExitError) - if ok { - waitStatus := exitError.Sys().(syscall.WaitStatus) - if waitStatus.ExitStatus() == 22 { - // EINVAL (already mapped) - return nil - } - } - } - return err + return "", err } - return nil + return strings.TrimSpace(devPath), nil } // cephRBDVolumeUnmap unmaps a given RBD storage volume @@ -694,7 +674,7 @@ func (s *storageCeph) copyWithoutSnapshotsFull(target container, return err } - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName, + _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName, storagePoolVolumeTypeNameContainer, s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for image `+ @@ -788,8 +768,9 @@ func (s *storageCeph) copyWithoutSnapshotsSparse(target container, return err } - err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName, - storagePoolVolumeTypeNameContainer, s.UserName) + _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, + targetContainerName, storagePoolVolumeTypeNameContainer, + s.UserName) if err != nil { logger.Errorf(`Failed to map RBD storage volume for image `+ `"%s" on storage pool "%s": %s`, targetContainerName, @@ -1394,3 +1375,90 @@ func parseClone(clone string) (string, string, string, error) { return poolName, volumeType, volumeName, nil } + +// getRBDMappedDevPath looks at sysfs to retrieve the device path +// "/dev/rbd<idx>" for an RBD image. If it doesn't find it it will map it if +// told to do so. +func getRBDMappedDevPath(clusterName string, poolName string, volumeType string, + volumeName string, doMap bool, userName string) (string, int) { + files, err := ioutil.ReadDir("/sys/devices/rbd") + if err != nil { + if os.IsNotExist(err) { + return "", 0 + } + + return "", -1 + } + + for _, f := range files { + if !f.IsDir() { + continue + } + + fName := f.Name() + idx, err := strconv.ParseUint(fName, 10, 64) + if err != nil { + continue + } + + tmp := fmt.Sprintf("/sys/devices/rbd/%/pool", fName) + contents, err := ioutil.ReadFile(tmp) + if err != nil { + if os.IsNotExist(err) { + continue + } + + return "", -1 + } + + detectedPoolName := strings.TrimSpace(string(contents)) + if detectedPoolName != poolName { + continue + } + + tmp = fmt.Sprintf("/sys/devices/rbd/%s/name", fName) + contents, err = ioutil.ReadFile(tmp) + if err != nil { + if os.IsNotExist(err) { + continue + } + + return "", -1 + } + + typedVolumeName := fmt.Sprintf("%s_%s", volumeType, volumeName) + detectedVolumeName := strings.TrimSpace(string(contents)) + if detectedVolumeName != typedVolumeName { + continue + } + + tmp = fmt.Sprintf("/sys/devices/rbd/%s/snap", fName) + contents, err = ioutil.ReadFile(tmp) + if err != nil { + if os.IsNotExist(err) { + continue + } + + return "", -1 + } + + detectedSnapName := strings.TrimSpace(string(contents)) + if detectedSnapName != "-" { + continue + } + + return fmt.Sprintf("/dev/rbd%d", idx), 1 + } + + if !doMap { + return "", 0 + } + + devPath, err := cephRBDVolumeMap(clusterName, poolName, + volumeName, volumeType, userName) + if err != nil { + return "", -1 + } + + return strings.TrimSpace(devPath), 2 +}
_______________________________________________ lxc-devel mailing list [email protected] http://lists.linuxcontainers.org/listinfo/lxc-devel
