Repository: mesos Updated Branches: refs/heads/master 554503985 -> 5a105893a
Updated filesystem layout for staging and mounting CSI volumes. To support the `STAGE_UNSTAGE_VOLUME` in CSI v0.2, a CSI volume is now staged at `<work_dir>/csi/<type>/<name>/mounts/<volume_id>/staging` and mounted at `<work_dir>/csi/<type>/<name>/mounts/<volume_id>/target`. The `DiskInfo.Source.Path.root` and `DiskInfo.Source.Mount.root` fields now stores `./csi/<type>/<name>/mounts` for PATH and MOUNT disk resources respectively, and the actual mount point is carved out from the `root` and `id` fields. In the future, the `SharedInfo` might also be used to construct the mount point for shared volumes. Review: https://reviews.apache.org/r/66574/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/02f9cb12 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/02f9cb12 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/02f9cb12 Branch: refs/heads/master Commit: 02f9cb127893b794e9fcc8b529f45bee33dd2e4c Parents: 5545039 Author: Chun-Hung Hsiao <chhs...@apache.org> Authored: Thu Apr 12 19:42:15 2018 -0700 Committer: Chun-Hung Hsiao <chhs...@mesosphere.io> Committed: Thu Apr 12 19:42:15 2018 -0700 ---------------------------------------------------------------------- src/CMakeLists.txt | 4 ++ src/Makefile.am | 19 +++++---- src/csi/paths.cpp | 36 +++++++++------- src/csi/paths.hpp | 15 ++++--- src/resource_provider/storage/provider.cpp | 56 +++++++++++++------------ src/slave/paths.cpp | 18 ++++++++ 6 files changed, 92 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/02f9cb12/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac176d1..31af9ea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -226,6 +226,9 @@ set(COMMON_SRC common/validation.cpp common/values.cpp) +set(CSI_SRC + csi/paths.cpp) + set(DOCKER_SRC docker/docker.cpp docker/spec.cpp) @@ -428,6 +431,7 @@ set(MESOS_SRC ${AUTHENTICATION_SRC} ${AUTHORIZER_SRC} ${COMMON_SRC} + ${CSI_SRC} ${DOCKER_SRC} ${EXECUTOR_SRC} ${FILES_SRC} http://git-wip-us.apache.org/repos/asf/mesos/blob/02f9cb12/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index b590928..07eb138 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1524,28 +1524,29 @@ libbuild_la_CPPFLAGS += -DBUILD_FLAGS="\"$$BUILD_FLAGS\"" libmesos_no_3rdparty_la_LIBADD += libbuild.la -if ENABLE_GRPC # Convenience library for build the CSI client. noinst_LTLIBRARIES += libcsi.la libcsi_la_SOURCES = \ - csi/paths.cpp \ - csi/utils.cpp + csi/paths.cpp + +libcsi_la_SOURCES += \ + csi/paths.hpp + +if ENABLE_GRPC libcsi_la_SOURCES += \ ../include/csi/spec.hpp \ - csi/paths.hpp \ + csi/client.cpp \ + csi/client.hpp \ csi/state.hpp \ csi/state.proto \ + csi/utils.cpp \ csi/utils.hpp - -libcsi_la_SOURCES += \ - csi/client.cpp \ - csi/client.hpp +endif nodist_libcsi_la_SOURCES = $(CXX_CSI_PROTOS) libcsi_la_CPPFLAGS = $(MESOS_CPPFLAGS) libmesos_no_3rdparty_la_LIBADD += libcsi.la -endif # Convenience library for building the replicated log in order to http://git-wip-us.apache.org/repos/asf/mesos/blob/02f9cb12/src/csi/paths.cpp ---------------------------------------------------------------------- diff --git a/src/csi/paths.cpp b/src/csi/paths.cpp index 0d42db2..cc5b9ec 100644 --- a/src/csi/paths.cpp +++ b/src/csi/paths.cpp @@ -42,18 +42,20 @@ namespace csi { namespace paths { // File names. -const char CONTAINER_INFO_FILE[] = "container.info"; -const char ENDPOINT_SOCKET_FILE[] = "endpoint.sock"; -const char VOLUME_STATE_FILE[] = "volume.state"; +constexpr char CONTAINER_INFO_FILE[] = "container.info"; +constexpr char ENDPOINT_SOCKET_FILE[] = "endpoint.sock"; +constexpr char VOLUME_STATE_FILE[] = "volume.state"; -const char CONTAINERS_DIR[] = "containers"; -const char VOLUMES_DIR[] = "volumes"; -const char MOUNTS_DIR[] = "mounts"; +constexpr char CONTAINERS_DIR[] = "containers"; +constexpr char VOLUMES_DIR[] = "volumes"; +constexpr char MOUNTS_DIR[] = "mounts"; +constexpr char STAGING_DIR[] = "staging"; +constexpr char TARGET_DIR[] = "target"; -const char ENDPOINT_DIR_SYMLINK[] = "endpoint"; -const char ENDPOINT_DIR[] = "mesos-csi-XXXXXX"; +constexpr char ENDPOINT_DIR_SYMLINK[] = "endpoint"; +constexpr char ENDPOINT_DIR[] = "mesos-csi-XXXXXX"; Try<list<string>> getContainerPaths( @@ -276,15 +278,19 @@ string getMountRootDir( } -string getMountPath( - const string& rootDir, - const string& type, - const string& name, +string getMountStagingPath( + const string& mountRootDir, const string& volumeId) { - return path::join( - getMountRootDir(rootDir, type, name), - http::encode(volumeId)); + return path::join(mountRootDir, http::encode(volumeId), STAGING_DIR); +} + + +string getMountTargetPath( + const string& mountRootDir, + const string& volumeId) +{ + return path::join(mountRootDir, http::encode(volumeId), TARGET_DIR); } } // namespace paths { http://git-wip-us.apache.org/repos/asf/mesos/blob/02f9cb12/src/csi/paths.hpp ---------------------------------------------------------------------- diff --git a/src/csi/paths.hpp b/src/csi/paths.hpp index 7892e5e..7a4e9e0 100644 --- a/src/csi/paths.hpp +++ b/src/csi/paths.hpp @@ -42,7 +42,9 @@ namespace paths { // | |-- <volume_id> // | |-- volume.state // |-- mounts -// |-- <volume_id> (mount point) +// |-- <volume_id> +// |- staging (staging mount point) +// |- target (mount point) struct ContainerPath @@ -134,10 +136,13 @@ std::string getMountRootDir( const std::string& name); -std::string getMountPath( - const std::string& rootDir, - const std::string& type, - const std::string& name, +std::string getMountStagingPath( + const std::string& mountRootDir, + const std::string& volumeId); + + +std::string getMountTargetPath( + const std::string& mountRootDir, const std::string& volumeId); } // namespace paths { http://git-wip-us.apache.org/repos/asf/mesos/blob/02f9cb12/src/resource_provider/storage/provider.cpp ---------------------------------------------------------------------- diff --git a/src/resource_provider/storage/provider.cpp b/src/resource_provider/storage/provider.cpp index ddbaa0f..ce94393 100644 --- a/src/resource_provider/storage/provider.cpp +++ b/src/resource_provider/storage/provider.cpp @@ -1785,24 +1785,23 @@ Future<csi::v0::Client> StorageLocalResourceProviderProcess::getService( endpointVolume->set_host_path(endpointDir); // Prepare the directory where the mount points will be placed. - const string mountDir = csi::paths::getMountRootDir( + const string mountRootDir = csi::paths::getMountRootDir( slave::paths::getCsiRootDir(workDir), info.storage().plugin().type(), info.storage().plugin().name()); - Try<Nothing> mkdir = os::mkdir(mountDir); + Try<Nothing> mkdir = os::mkdir(mountRootDir); if (mkdir.isError()) { return Failure( - "Failed to create directory '" + mountDir + - "': " + mkdir.error()); + "Failed to create directory '" + mountRootDir + "': " + mkdir.error()); } // Prepare a volume where the mount points will be placed. Volume* mountVolume = containerInfo.add_volumes(); mountVolume->set_mode(Volume::RW); - mountVolume->set_container_path(mountDir); + mountVolume->set_container_path(mountRootDir); mountVolume->mutable_source()->set_type(Volume::Source::HOST_PATH); - mountVolume->mutable_source()->mutable_host_path()->set_path(mountDir); + mountVolume->mutable_source()->mutable_host_path()->set_path(mountRootDir); mountVolume->mutable_source()->mutable_host_path() ->mutable_mount_propagation()->set_mode(MountPropagation::BIDIRECTIONAL); @@ -2194,16 +2193,17 @@ Future<Nothing> StorageLocalResourceProviderProcess::nodePublish( csi::v0::Client client) -> Future<Nothing> { VolumeData& volume = volumes.at(volumeId); - const string mountPath = csi::paths::getMountPath( - slave::paths::getCsiRootDir(workDir), - info.storage().plugin().type(), - info.storage().plugin().name(), + const string targetPath = csi::paths::getMountTargetPath( + csi::paths::getMountRootDir( + slave::paths::getCsiRootDir(workDir), + info.storage().plugin().type(), + info.storage().plugin().name()), volumeId); - Try<Nothing> mkdir = os::mkdir(mountPath); + Try<Nothing> mkdir = os::mkdir(targetPath); if (mkdir.isError()) { return Failure( - "Failed to create mount point '" + mountPath + "': " + + "Failed to create mount target path '" + targetPath + "': " + mkdir.error()); } @@ -2217,7 +2217,7 @@ Future<Nothing> StorageLocalResourceProviderProcess::nodePublish( csi::v0::NodePublishVolumeRequest request; request.set_volume_id(volumeId); *request.mutable_publish_info() = volume.state.publish_info(); - request.set_target_path(mountPath); + request.set_target_path(targetPath); request.mutable_volume_capability() ->CopyFrom(volume.state.volume_capability()); request.set_readonly(false); @@ -2252,13 +2252,14 @@ Future<Nothing> StorageLocalResourceProviderProcess::nodeUnpublish( .then(defer(self(), [this, volumeId](csi::v0::Client client) { VolumeData& volume = volumes.at(volumeId); - const string mountPath = csi::paths::getMountPath( - slave::paths::getCsiRootDir(workDir), - info.storage().plugin().type(), - info.storage().plugin().name(), + const string targetPath = csi::paths::getMountTargetPath( + csi::paths::getMountRootDir( + slave::paths::getCsiRootDir(workDir), + info.storage().plugin().type(), + info.storage().plugin().name()), volumeId); - CHECK(os::exists(mountPath)); + CHECK(os::exists(targetPath)); // A previously failed `NodePublishVolume` call can be recovered through // the current `NodeUnpublishVolume` call. See: @@ -2273,20 +2274,20 @@ Future<Nothing> StorageLocalResourceProviderProcess::nodeUnpublish( csi::v0::NodeUnpublishVolumeRequest request; request.set_volume_id(volumeId); - request.set_target_path(mountPath); + request.set_target_path(targetPath); return client.NodeUnpublishVolume(request) - .then(defer(self(), [this, volumeId, mountPath]() -> Future<Nothing> { + .then(defer(self(), [this, volumeId, targetPath]() -> Future<Nothing> { VolumeData& volume = volumes.at(volumeId); volume.state.set_state(VolumeState::NODE_READY); volume.state.clear_boot_id(); checkpointVolumeState(volumeId); - Try<Nothing> rmdir = os::rmdir(mountPath); + Try<Nothing> rmdir = os::rmdir(targetPath); if (rmdir.isError()) { return Failure( - "Failed to remove mount point '" + mountPath + "': " + + "Failed to remove mount point '" + targetPath + "': " + rmdir.error()); } @@ -2885,23 +2886,24 @@ StorageLocalResourceProviderProcess::applyCreateVolumeOrBlock( ->CopyFrom(convertStringMapToLabels(volumeState.volume_attributes())); } - const string mountPath = csi::paths::getMountPath( + const string mountRootDir = csi::paths::getMountRootDir( slave::paths::getCsiRootDir("."), info.storage().plugin().type(), - info.storage().plugin().name(), - volumeId); + info.storage().plugin().name()); switch (type) { case Resource::DiskInfo::Source::PATH: { // Set the root path relative to agent work dir. converted.mutable_disk()->mutable_source()->mutable_path() - ->set_root(mountPath); + ->set_root(mountRootDir); + break; } case Resource::DiskInfo::Source::MOUNT: { // Set the root path relative to agent work dir. converted.mutable_disk()->mutable_source()->mutable_mount() - ->set_root(mountPath); + ->set_root(mountRootDir); + break; } case Resource::DiskInfo::Source::BLOCK: { http://git-wip-us.apache.org/repos/asf/mesos/blob/02f9cb12/src/slave/paths.cpp ---------------------------------------------------------------------- diff --git a/src/slave/paths.cpp b/src/slave/paths.cpp index 8a7e162..690bfe3 100644 --- a/src/slave/paths.cpp +++ b/src/slave/paths.cpp @@ -36,6 +36,8 @@ #include "common/validation.hpp" +#include "csi/paths.hpp" + #include "messages/messages.hpp" #include "slave/paths.hpp" @@ -673,10 +675,18 @@ string getPersistentVolumePath( CHECK(volume.disk().source().has_path()); CHECK(volume.disk().source().path().has_root()); string root = volume.disk().source().path().root(); + if (!path::absolute(root)) { // A relative path in `root` is relative to agent work dir. root = path::join(workDir, root); } + + if (volume.disk().source().has_id()) { + // For a CSI volume the mount point is derived from `root` and `id`. + root = + csi::paths::getMountTargetPath(root, volume.disk().source().id()); + } + return getPersistentVolumePath( root, role, @@ -687,10 +697,18 @@ string getPersistentVolumePath( CHECK(volume.disk().source().has_mount()); CHECK(volume.disk().source().mount().has_root()); string root = volume.disk().source().mount().root(); + if (!path::absolute(root)) { // A relative path in `root` is relative to agent work dir. root = path::join(workDir, root); } + + if (volume.disk().source().has_id()) { + // For a CSI volume the mount point is derived from `root` and `id`. + root = + csi::paths::getMountTargetPath(root, volume.disk().source().id()); + } + return root; } case Resource::DiskInfo::Source::BLOCK:
