This is an automated email from the ASF dual-hosted git repository. saadurrahman pushed a commit to branch saadurrahman/3821-Remove-Deprecated-Volumes-K8s-dev in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 35c023e106186fe972a82f6ce2c4d7bc9e35a409 Author: Saad Ur Rahman <[email protected]> AuthorDate: Mon May 2 15:22:06 2022 -0400 [K8s] created PVC generation in Volume Factory. --- .../apache/heron/scheduler/kubernetes/Volumes.java | 52 ++++++++- .../heron/scheduler/kubernetes/VolumesTests.java | 121 ++++++++++++++++++++- 2 files changed, 170 insertions(+), 3 deletions(-) diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java index 39e9a658d5d..d9e641f78cc 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java @@ -19,11 +19,14 @@ package org.apache.heron.scheduler.kubernetes; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder; +import io.kubernetes.client.openapi.models.V1ResourceRequirements; import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeBuilder; import io.kubernetes.client.openapi.models.V1VolumeMount; @@ -34,8 +37,7 @@ final class Volumes { public enum VolumeType { EmptyDir, HostPath, - NetworkFileSystem, - PersistentVolumeClaim + NetworkFileSystem } private final Map<VolumeType, IVolumeFactory> volumes = new HashMap<>(); @@ -94,6 +96,52 @@ final class Volumes { return volumeMount; } + /** + * Generates <code>Persistent Volume Claims Templates</code> from a mapping of <code>Volumes</code> + * to <code>key-value</code> pairs of configuration options and values. + * @param claimName Name to be assigned to <code>Persistent Volume Claims Template</code>. + * @param labels Labels to be attached to the <code>Persistent Volume Claims Template</code>. + * @param configs <code>Volume</code> to configuration <code>key-value</code> mappings. + * @return Fully populated dynamically backed <code>Persistent Volume Claims</code>. + */ + V1PersistentVolumeClaim createPersistentVolumeClaim(String claimName, Map<String, String> labels, + Map<KubernetesConstants.VolumeConfigKeys, String> configs) { + V1PersistentVolumeClaim claim = new V1PersistentVolumeClaimBuilder() + .withNewMetadata() + .withName(claimName) + .withLabels(labels) + .endMetadata() + .withNewSpec() + .withStorageClassName("") + .endSpec() + .build(); + + // Populate PVC options. + for (Map.Entry<KubernetesConstants.VolumeConfigKeys, String> option : configs.entrySet()) { + String optionValue = option.getValue(); + switch(option.getKey()) { + case storageClassName: + claim.getSpec().setStorageClassName(optionValue); + break; + case sizeLimit: + claim.getSpec().setResources( + new V1ResourceRequirements() + .putRequestsItem("storage", new Quantity(optionValue))); + break; + case accessModes: + claim.getSpec().setAccessModes(Arrays.asList(optionValue.split(","))); + break; + case volumeMode: + claim.getSpec().setVolumeMode(optionValue); + break; + // Valid ignored options not used in a PVC. + default: + break; + } + } + return claim; + } + interface IVolumeFactory { V1Volume create(String volumeName, Map<KubernetesConstants.VolumeConfigKeys, String> configs); } diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java index a4b8c72be02..1cd7889741b 100644 --- a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java +++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java @@ -19,6 +19,9 @@ package org.apache.heron.scheduler.kubernetes; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -30,6 +33,9 @@ import org.junit.Test; import org.apache.heron.common.basics.Pair; +import io.kubernetes.client.custom.Quantity; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder; import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeBuilder; import io.kubernetes.client.openapi.models.V1VolumeMount; @@ -216,9 +222,122 @@ public class VolumesTests { // Test loop. for (KubernetesUtils.TestTuple<Pair<String, Map<KubernetesConstants.VolumeConfigKeys, String>>, - V1VolumeMount> testCase : testCases) { + V1VolumeMount> testCase : testCases) { V1VolumeMount actual = Volumes.get().createMount(testCase.input.first, testCase.input.second); Assert.assertEquals(testCase.description, testCase.expected, actual); } } + + @Test + public void testPersistentVolumeClaim() { + final String topologyName = "topology-name"; + final String volumeNameOne = "volume-name-one"; + final String volumeNameTwo = "volume-name-two"; + final String volumeNameStatic = "volume-name-static"; + final String claimNameOne = "OnDemand"; + final String claimNameTwo = "claim-name-two"; + final String claimNameStatic = "OnDEmaND"; + final String storageClassName = "storage-class-name"; + final String sizeLimit = "555Gi"; + final String accessModesList = "ReadWriteOnce,ReadOnlyMany,ReadWriteMany"; + final String accessModes = "ReadOnlyMany"; + final String volumeMode = "VolumeMode"; + final String path = "/path/to/mount/"; + final String subPath = "/sub/path/to/mount/"; + final Map<String, String> labels = V1Controller.getPersistentVolumeClaimLabels(topologyName); + + final Map<KubernetesConstants.VolumeConfigKeys, String> volOneConfig = + new HashMap<KubernetesConstants.VolumeConfigKeys, String>() { + { + put(KubernetesConstants.VolumeConfigKeys.claimName, claimNameOne); + put(KubernetesConstants.VolumeConfigKeys.storageClassName, storageClassName); + put(KubernetesConstants.VolumeConfigKeys.sizeLimit, sizeLimit); + put(KubernetesConstants.VolumeConfigKeys.accessModes, accessModesList); + put(KubernetesConstants.VolumeConfigKeys.volumeMode, volumeMode); + put(KubernetesConstants.VolumeConfigKeys.path, path); + } + }; + + final Map<KubernetesConstants.VolumeConfigKeys, String> volTwoConfig = + new HashMap<KubernetesConstants.VolumeConfigKeys, String>() { + { + put(KubernetesConstants.VolumeConfigKeys.claimName, claimNameTwo); + put(KubernetesConstants.VolumeConfigKeys.storageClassName, storageClassName); + put(KubernetesConstants.VolumeConfigKeys.sizeLimit, sizeLimit); + put(KubernetesConstants.VolumeConfigKeys.accessModes, accessModesList); + put(KubernetesConstants.VolumeConfigKeys.volumeMode, volumeMode); + put(KubernetesConstants.VolumeConfigKeys.path, path); + put(KubernetesConstants.VolumeConfigKeys.subPath, subPath); + } + }; + + final Map<KubernetesConstants.VolumeConfigKeys, String> volStaticConfig = + new HashMap<KubernetesConstants.VolumeConfigKeys, String>() { + { + put(KubernetesConstants.VolumeConfigKeys.claimName, claimNameStatic); + put(KubernetesConstants.VolumeConfigKeys.sizeLimit, sizeLimit); + put(KubernetesConstants.VolumeConfigKeys.accessModes, accessModes); + put(KubernetesConstants.VolumeConfigKeys.volumeMode, volumeMode); + put(KubernetesConstants.VolumeConfigKeys.path, path); + put(KubernetesConstants.VolumeConfigKeys.subPath, subPath); + } + }; + + final V1PersistentVolumeClaim claimOne = new V1PersistentVolumeClaimBuilder() + .withNewMetadata() + .withName(volumeNameOne) + .withLabels(labels) + .endMetadata() + .withNewSpec() + .withStorageClassName(storageClassName) + .withAccessModes(Arrays.asList(accessModesList.split(","))) + .withVolumeMode(volumeMode) + .withNewResources() + .addToRequests("storage", new Quantity(sizeLimit)) + .endResources() + .endSpec() + .build(); + + final V1PersistentVolumeClaim claimTwo = new V1PersistentVolumeClaimBuilder() + .withNewMetadata() + .withName(volumeNameTwo) + .withLabels(labels) + .endMetadata() + .withNewSpec() + .withStorageClassName(storageClassName) + .withAccessModes(Arrays.asList(accessModesList.split(","))) + .withVolumeMode(volumeMode) + .withNewResources() + .addToRequests("storage", new Quantity(sizeLimit)) + .endResources() + .endSpec() + .build(); + + final V1PersistentVolumeClaim claimStatic = new V1PersistentVolumeClaimBuilder() + .withNewMetadata() + .withName(volumeNameStatic) + .withLabels(labels) + .endMetadata() + .withNewSpec() + .withStorageClassName("") + .withAccessModes(Collections.singletonList(accessModes)) + .withVolumeMode(volumeMode) + .withNewResources() + .addToRequests("storage", new Quantity(sizeLimit)) + .endResources() + .endSpec() + .build(); + + final V1PersistentVolumeClaim actualPVCOne = Volumes.get() + .createPersistentVolumeClaim(volumeNameOne, labels, volOneConfig); + Assert.assertEquals("Volume one PVC", claimOne, actualPVCOne); + + final V1PersistentVolumeClaim actualPVCTwo = Volumes.get() + .createPersistentVolumeClaim(volumeNameTwo, labels, volTwoConfig); + Assert.assertEquals("Volume two PVC", claimTwo, actualPVCTwo); + + final V1PersistentVolumeClaim actualPVCStatic = Volumes.get() + .createPersistentVolumeClaim(volumeNameStatic, labels, volStaticConfig); + Assert.assertEquals("Volume static PVC", claimStatic, actualPVCStatic); + } }
