This is an automated email from the ASF dual-hosted git repository. saadurrahman pushed a commit to branch saadurrahman/3821-Remove-Deprecated-Volumes-K8s-dev in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 7bc2f02d70374d22378ead793f94aac6efd2d77b Author: Saad Ur Rahman <[email protected]> AuthorDate: Mon May 2 13:23:46 2022 -0400 [K8s] created Volume Mount in Volume Factory. Generating all volume mounts from within Volume Factory for maintainability. --- .../heron/scheduler/kubernetes/V1Controller.java | 6 +- .../apache/heron/scheduler/kubernetes/Volumes.java | 48 ++++++--- .../heron/scheduler/kubernetes/VolumesTests.java | 110 ++++++++++++++++++++- 3 files changed, 144 insertions(+), 20 deletions(-) diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java index 338e50bda51..a065790df5d 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java @@ -1239,7 +1239,7 @@ public class V1Controller extends KubernetesController { : mapOfOpts.entrySet()) { final String volumeName = configs.getKey(); final V1Volume volume = Volumes.get() - .create(Volumes.VolumeType.EmptyDir, volumeName, configs.getValue()); + .createVolume(Volumes.VolumeType.EmptyDir, volumeName, configs.getValue()); volumes.add(volume); volumeMounts.add(createVolumeMountsCLI(volumeName, configs.getValue())); } @@ -1260,7 +1260,7 @@ public class V1Controller extends KubernetesController { : mapOfOpts.entrySet()) { final String volumeName = configs.getKey(); final V1Volume volume = Volumes.get() - .create(Volumes.VolumeType.HostPath, volumeName, configs.getValue()); + .createVolume(Volumes.VolumeType.HostPath, volumeName, configs.getValue()); volumes.add(volume); volumeMounts.add(createVolumeMountsCLI(volumeName, configs.getValue())); } @@ -1281,7 +1281,7 @@ public class V1Controller extends KubernetesController { : mapOfOpts.entrySet()) { final String volumeName = configs.getKey(); final V1Volume volume = Volumes.get() - .create(Volumes.VolumeType.NetworkFileSystem, volumeName, configs.getValue()); + .createVolume(Volumes.VolumeType.NetworkFileSystem, volumeName, configs.getValue()); volumes.add(volume); volumeMounts.add(createVolumeMountsCLI(volumeName, configs.getValue())); } diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java index 038dd08a7d6..39e9a658d5d 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java @@ -27,6 +27,7 @@ import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim; import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeBuilder; import io.kubernetes.client.openapi.models.V1VolumeMount; +import io.kubernetes.client.openapi.models.V1VolumeMountBuilder; final class Volumes { @@ -34,8 +35,7 @@ final class Volumes { EmptyDir, HostPath, NetworkFileSystem, - PersistentVolumeClaim, - VolumeMount + PersistentVolumeClaim } private final Map<VolumeType, IVolumeFactory> volumes = new HashMap<>(); @@ -57,26 +57,45 @@ final class Volumes { * @param configs A map of configurations. * @return Fully configured <code>V1Volume</code>. */ - V1Volume create(VolumeType volumeType, String volumeName, - Map<KubernetesConstants.VolumeConfigKeys, String> configs) { + V1Volume createVolume(VolumeType volumeType, String volumeName, + Map<KubernetesConstants.VolumeConfigKeys, String> configs) { if (volumes.containsKey(volumeType)) { return volumes.get(volumeType).create(volumeName, configs); } return null; } - interface IVolumeFactory { - V1Volume create(String volumeName, Map<KubernetesConstants.VolumeConfigKeys, String> configs); - } - - interface IVolumeMountFactory { - V1VolumeMount create(String volumeName, - Map<KubernetesConstants.VolumeConfigKeys, String> configs); + /** + * Generates a <code>Volume Mount</code> from specifications. + * @param volumeName Name of the <code>Volume</code>. + * @param configs Mapping of <code>Volume</code> option <code>key-value</code> configuration pairs. + * @return A configured <code>V1VolumeMount</code>. + */ + V1VolumeMount createMount(String volumeName, + Map<KubernetesConstants.VolumeConfigKeys, String> configs) { + final V1VolumeMount volumeMount = new V1VolumeMountBuilder() + .withName(volumeName) + .build(); + for (Map.Entry<KubernetesConstants.VolumeConfigKeys, String> config : configs.entrySet()) { + switch (config.getKey()) { + case path: + volumeMount.mountPath(config.getValue()); + break; + case subPath: + volumeMount.subPath(config.getValue()); + break; + case readOnly: + volumeMount.readOnly(Boolean.parseBoolean(config.getValue())); + break; + default: + break; + } + } + return volumeMount; } - interface IPersistentVolumeClaimFactory { - V1PersistentVolumeClaim create(String volumeName, - Map<KubernetesConstants.VolumeConfigKeys, String> configs); + interface IVolumeFactory { + V1Volume create(String volumeName, Map<KubernetesConstants.VolumeConfigKeys, String> configs); } static class EmptyDirVolumeFactory implements IVolumeFactory { @@ -182,4 +201,5 @@ final class Volumes { return volume; } } + } diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java index 10bb4739bdf..a4b8c72be02 100644 --- a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java +++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java @@ -19,6 +19,8 @@ package org.apache.heron.scheduler.kubernetes; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import com.google.common.collect.ImmutableMap; @@ -26,8 +28,12 @@ import com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.junit.Test; +import org.apache.heron.common.basics.Pair; + import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeBuilder; +import io.kubernetes.client.openapi.models.V1VolumeMount; +import io.kubernetes.client.openapi.models.V1VolumeMountBuilder; public class VolumesTests { @@ -54,7 +60,7 @@ public class VolumesTests { .build(); final V1Volume actualVolume = Volumes.get() - .create(Volumes.VolumeType.EmptyDir, volumeName, config); + .createVolume(Volumes.VolumeType.EmptyDir, volumeName, config); Assert.assertEquals("Volume Factory Empty Directory", expectedVolume, actualVolume); } @@ -82,7 +88,7 @@ public class VolumesTests { .build(); final V1Volume actualVolume = Volumes.get() - .create(Volumes.VolumeType.HostPath, volumeName, config); + .createVolume(Volumes.VolumeType.HostPath, volumeName, config); Assert.assertEquals("Volume Factory Host Path", expectedVolume, actualVolume); } @@ -113,8 +119,106 @@ public class VolumesTests { .build(); final V1Volume actualVolume = Volumes.get() - .create(Volumes.VolumeType.NetworkFileSystem, volumeName, config); + .createVolume(Volumes.VolumeType.NetworkFileSystem, volumeName, config); Assert.assertEquals("Volume Factory Network File System", expectedVolume, actualVolume); } + + @Test + public void testVolumeMount() { + final String volumeNamePVC = "volume-name-pvc"; + final String volumeNameHostPath = "volume-name-host-path"; + final String volumeNameEmptyDir = "volume-name-empty-dir"; + final String volumeNameNFS = "volume-name-nfs"; + final String value = "inserted-value"; + + // Test case container. + // Input: [0] volume name, [1] volume options + // Output: The expected <V1VolumeMount>. + final List<KubernetesUtils.TestTuple< + Pair<String, Map<KubernetesConstants.VolumeConfigKeys, String>>, V1VolumeMount>> testCases = + new LinkedList<>(); + + // PVC. + final Map<KubernetesConstants.VolumeConfigKeys, String> configPVC = + ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder() + .put(KubernetesConstants.VolumeConfigKeys.claimName, value) + .put(KubernetesConstants.VolumeConfigKeys.storageClassName, value) + .put(KubernetesConstants.VolumeConfigKeys.sizeLimit, value) + .put(KubernetesConstants.VolumeConfigKeys.accessModes, value) + .put(KubernetesConstants.VolumeConfigKeys.volumeMode, value) + .put(KubernetesConstants.VolumeConfigKeys.path, value) + .put(KubernetesConstants.VolumeConfigKeys.subPath, value) + .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true") + .build(); + final V1VolumeMount volumeMountPVC = new V1VolumeMountBuilder() + .withName(volumeNamePVC) + .withMountPath(value) + .withSubPath(value) + .withReadOnly(true) + .build(); + testCases.add(new KubernetesUtils.TestTuple<>("PVC volume mount", + new Pair<>(volumeNamePVC, configPVC), volumeMountPVC)); + + // Host Path. + final Map<KubernetesConstants.VolumeConfigKeys, String> configHostPath = + ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder() + .put(KubernetesConstants.VolumeConfigKeys.type, "DirectoryOrCreate") + .put(KubernetesConstants.VolumeConfigKeys.pathOnHost, value) + .put(KubernetesConstants.VolumeConfigKeys.path, value) + .put(KubernetesConstants.VolumeConfigKeys.subPath, value) + .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true") + .build(); + final V1VolumeMount volumeMountHostPath = new V1VolumeMountBuilder() + .withName(volumeNameHostPath) + .withMountPath(value) + .withSubPath(value) + .withReadOnly(true) + .build(); + testCases.add(new KubernetesUtils.TestTuple<>("Host Path volume mount", + new Pair<>(volumeNameHostPath, configHostPath), volumeMountHostPath)); + + // Empty Dir. + final Map<KubernetesConstants.VolumeConfigKeys, String> configEmptyDir = + ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder() + .put(KubernetesConstants.VolumeConfigKeys.sizeLimit, value) + .put(KubernetesConstants.VolumeConfigKeys.medium, "Memory") + .put(KubernetesConstants.VolumeConfigKeys.path, value) + .put(KubernetesConstants.VolumeConfigKeys.subPath, value) + .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true") + .build(); + final V1VolumeMount volumeMountEmptyDir = new V1VolumeMountBuilder() + .withName(volumeNameEmptyDir) + .withMountPath(value) + .withSubPath(value) + .withReadOnly(true) + .build(); + testCases.add(new KubernetesUtils.TestTuple<>("Empty Dir volume mount", + new Pair<>(volumeNameEmptyDir, configEmptyDir), volumeMountEmptyDir)); + + // NFS. + final Map<KubernetesConstants.VolumeConfigKeys, String> configNFS = + ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder() + .put(KubernetesConstants.VolumeConfigKeys.server, "nfs.server.address") + .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true") + .put(KubernetesConstants.VolumeConfigKeys.pathOnNFS, value) + .put(KubernetesConstants.VolumeConfigKeys.path, value) + .put(KubernetesConstants.VolumeConfigKeys.subPath, value) + .build(); + final V1VolumeMount volumeMountNFS = new V1VolumeMountBuilder() + .withName(volumeNameNFS) + .withMountPath(value) + .withSubPath(value) + .withReadOnly(true) + .build(); + testCases.add(new KubernetesUtils.TestTuple<>("NFS volume mount", + new Pair<>(volumeNameNFS, configNFS), volumeMountNFS)); + + // Test loop. + for (KubernetesUtils.TestTuple<Pair<String, Map<KubernetesConstants.VolumeConfigKeys, String>>, + V1VolumeMount> testCase : testCases) { + V1VolumeMount actual = Volumes.get().createMount(testCase.input.first, testCase.input.second); + Assert.assertEquals(testCase.description, testCase.expected, actual); + } + } }
