This is an automated email from the ASF dual-hosted git repository. roryqi pushed a commit to branch support-pvc in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 3ec2d0a4f52e0811596e79fdc9428e5e6d2cf5fa Author: alexcqtan <[email protected]> AuthorDate: Tue Jan 10 10:52:44 2023 +0800 support mounting PVC for shuffleserver --- .../uniffle/v1alpha1/remoteshuffleservice_types.go | 23 +++ .../api/uniffle/v1alpha1/zz_generated.deepcopy.go | 28 +++ .../uniffle.apache.org_remoteshuffleservices.yaml | 192 +++++++++++++++++++++ .../controller/sync/shuffleserver/shuffleserver.go | 16 ++ 4 files changed, 259 insertions(+) diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go index cd53c2a12..978695343 100644 --- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go +++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go @@ -142,6 +142,29 @@ type ShuffleServerConfig struct { // UpgradeStrategy defines upgrade strategy of shuffle servers. UpgradeStrategy *ShuffleServerUpgradeStrategy `json:"upgradeStrategy"` + + // volumeClaimTemplates is a list of claims that pods are allowed to reference. + // The StatefulSet controller is responsible for mapping network identities to + // claims in a way that maintains the identity of a pod. Every claim in + // this list must have at least one matching (by name) volumeMount in one + // container in the template. A claim in this list takes precedence over + // any volumes in the template, with the same name. + // +optional + VolumeClaimTemplates []ShuffleServerPersistentVolumeClaimTemplate `json:"volumeClaimTemplates,omitempty" protobuf:"bytes,4,rep,name=volumeClaimTemplates"` +} + +type ShuffleServerPersistentVolumeClaimTemplate struct { + // May contain labels and annotations that will be copied into the PVC + // when creating it. No other fields are allowed and will be rejected during + // validation. + // + VolumeNameTemplate *string `json:"volumeNameTemplate"` + + // The specification for the PersistentVolumeClaim. The entire content is + // copied unchanged into the PVC that gets created from this + // template. The same fields as in a PersistentVolumeClaim + // are also valid here. + Spec corev1.PersistentVolumeClaimSpec `json:"spec" protobuf:"bytes,2,name=spec"` } // ShuffleServerUpgradeStrategy defines upgrade strategy of shuffle servers. diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go index 8d4d40def..0c7389f29 100644 --- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go @@ -443,6 +443,13 @@ func (in *ShuffleServerConfig) DeepCopyInto(out *ShuffleServerConfig) { *out = new(ShuffleServerUpgradeStrategy) (*in).DeepCopyInto(*out) } + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]ShuffleServerPersistentVolumeClaimTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerConfig. @@ -455,6 +462,27 @@ func (in *ShuffleServerConfig) DeepCopy() *ShuffleServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopyInto(out *ShuffleServerPersistentVolumeClaimTemplate) { + *out = *in + if in.VolumeNameTemplate != nil { + in, out := &in.VolumeNameTemplate, &out.VolumeNameTemplate + *out = new(string) + **out = **in + } + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerPersistentVolumeClaimTemplate. +func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopy() *ShuffleServerPersistentVolumeClaimTemplate { + if in == nil { + return nil + } + out := new(ShuffleServerPersistentVolumeClaimTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShuffleServerUpgradeStrategy) DeepCopyInto(out *ShuffleServerUpgradeStrategy) { *out = *in diff --git a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml index 314304cd5..818df4bc8 100644 --- a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml +++ b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml @@ -8362,6 +8362,198 @@ spec: required: - type type: object + volumeClaimTemplates: + description: volumeClaimTemplates is a list of claims that pods + are allowed to reference. The StatefulSet controller is responsible + for mapping network identities to claims in a way that maintains + the identity of a pod. Every claim in this list must have at + least one matching (by name) volumeMount in one container in + the template. A claim in this list takes precedence over any + volumes in the template, with the same name. + items: + properties: + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC that + gets created from this template. The same fields as in + a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on + the contents of the specified data source. If the + AnyVolumeDataSource feature gate is enabled, this + field will always have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API + group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the DataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource and + DataSourceRef) will be set to the same value automatically + if one of them is empty and the other is non-empty. + There are two important differences between DataSource + and DataSourceRef: * While DataSource only allows + two specific types of objects, DataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by the + claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + volumeNameTemplate: + description: May contain labels and annotations that will + be copied into the PVC when creating it. No other fields + are allowed and will be rejected during validation. + type: string + required: + - spec + - volumeNameTemplate + type: object + type: array volumeMounts: description: VolumeMounts indicates describes mountings of volumes within shuffle servers' container. diff --git a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go index a96c0eba0..9d5dc17d8 100644 --- a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go +++ b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go @@ -198,6 +198,17 @@ func GenerateSts(kubeClient kubernetes.Interface, rss *unifflev1alpha1.RemoteShu sts.Spec.Template.Spec.RuntimeClassName = rss.Spec.ShuffleServer.RuntimeClassName } + // add VolumeClaimTemplates, support cloud storage + sts.Spec.VolumeClaimTemplates = make([]corev1.PersistentVolumeClaim, 0, len(rss.Spec.ShuffleServer.VolumeClaimTemplates)) + for _, pvcTemplate := range rss.Spec.ShuffleServer.VolumeClaimTemplates { + sts.Spec.VolumeClaimTemplates = append(sts.Spec.VolumeClaimTemplates, corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: *pvcTemplate.VolumeNameTemplate, + }, + Spec: pvcTemplate.Spec, + }) + } + // add init containers, the main container and other containers. sts.Spec.Template.Spec.InitContainers = util.GenerateInitContainers(rss.Spec.ShuffleServer.RSSPodSpec) containers := []corev1.Container{*generateMainContainer(rss)} @@ -238,6 +249,11 @@ func generateStorageBasePath(rss *unifflev1alpha1.RemoteShuffleService) string { } paths = append(paths, strings.TrimSuffix(v, "/")+"/"+controllerconstants.RssDataDir) } + + for _, vm := range rss.Spec.ShuffleServer.VolumeMounts { + paths = append(paths, strings.TrimSuffix(vm.MountPath, "/"+"rssdata")) + } + sort.Strings(paths) return strings.Join(paths, ",") }
