This is an automated email from the ASF dual-hosted git repository. duanzhengqiang pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/shardingsphere-on-cloud.git
commit 1841beb23226e4772b018448d048ff78904424bc Author: Ghoul_Lee <[email protected]> AuthorDate: Wed Jul 20 20:17:22 2022 +0800 feat(HPA): Added ability to automatically create or delete HPA based on configuration (#19) * feat(HPA): add create HPA Signed-off-by: windghoul <[email protected]> * fix(HPA): fix deployment replicas Signed-off-by: windghoul <[email protected]> --- .../templates/shardingsphere_v1alpha1_proxy.yaml | 3 + charts/shardingsphere-cluster/values.yaml | 14 +- .../crds/shardingsphere.sphere-ex.com_proxies.yaml | 1134 ++++++++++---------- .../shardingsphere.sphere-ex.com_proxyconfigs.yaml | 322 +++--- .../templates/deployment.yaml | 2 +- charts/shardingsphere-operator/values.yaml | 4 +- .../api/v1alpha1/proxy_status.go | 3 + .../api/v1alpha1/proxy_types.go | 21 +- .../api/v1alpha1/proxy_webhook.go | 48 +- .../pkg/controllers/proxy_controller.go | 53 +- shardingsphere-operator/pkg/reconcile/resource.go | 104 +- 11 files changed, 970 insertions(+), 738 deletions(-) diff --git a/charts/shardingsphere-cluster/templates/shardingsphere_v1alpha1_proxy.yaml b/charts/shardingsphere-cluster/templates/shardingsphere_v1alpha1_proxy.yaml index b41fa39..98fc647 100644 --- a/charts/shardingsphere-cluster/templates/shardingsphere_v1alpha1_proxy.yaml +++ b/charts/shardingsphere-cluster/templates/shardingsphere_v1alpha1_proxy.yaml @@ -25,6 +25,9 @@ spec: {{- if .Values.replicaCount }} replicas: {{ .Values.replicaCount }} {{- end}} + {{- if .Values.automaticScaling.enable }} + automaticScaling: {{- toYaml .Values.automaticScaling | nindent 4}} + {{- end }} proxyConfigName: {{ printf "%s-configuration" (include "common.names.fullname" .) | quote}} port: {{ .Values.startPort }} {{- if .Values.mySQLDriver.version }} diff --git a/charts/shardingsphere-cluster/values.yaml b/charts/shardingsphere-cluster/values.yaml index 7aed5b1..614d1fa 100644 --- a/charts/shardingsphere-cluster/values.yaml +++ b/charts/shardingsphere-cluster/values.yaml @@ -42,11 +42,7 @@ automaticScaling: ## requests: ## cpu: 2 ## -resources: - limits: - cpu: '2' - requests: - cpu: '1' +resources: {} ## @param service.type ShardingSphere-Proxy external exposure mode ## @param service.port ShardingSphere-Proxy exposes port ## @@ -60,6 +56,12 @@ startPort: 3307 ## mySQLDriver: version: "5.1.47" +## @param imagePullSecrets ShardingSphere-Proxy pull private image repository key +## e.g: +## imagePullSecrets: +## - name: mysecret +## +imagePullSecrets: [] ## @section ShardingSphere-Proxy ServerConfiguration parameters ## NOTE: If you use the sub-charts to deploy Zookeeper, the server-lists field must be "{{ printf \"%s-zookeeper.%s:2181\" .Release.Name .Release.Namespace }}", ## otherwise please fill in the correct zookeeper address @@ -117,7 +119,7 @@ zookeeper: enabled: true ## @param zookeeper.replicaCount Number of ZooKeeper nodes ## - replicaCount: 3 + replicaCount: 1 ## ZooKeeper Persistence parameters ## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ ## @param zookeeper.persistence.enabled Enable persistence on ZooKeeper using PVC(s) diff --git a/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxies.yaml b/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxies.yaml index bfeae40..f0a3903 100644 --- a/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxies.yaml +++ b/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxies.yaml @@ -31,372 +31,404 @@ spec: singular: proxy scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .status.readyNodes - name: ReadyNodes - type: integer - - jsonPath: .status.phase - name: Phase - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: Proxy is the Schema for the proxies API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation + - additionalPrinterColumns: + - jsonPath: .status.readyNodes + name: ReadyNodes + type: integer + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Proxy is the Schema for the proxies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ProxySpec defines the desired state of Proxy - properties: - livenessProbe: - description: Probes are not allowed for ephemeral containers. + type: string + metadata: + type: object + spec: + description: ProxySpec defines the desired state of Proxy + properties: + automaticScaling: + description: AutomaticScaling HPA configuration + properties: + enable: + type: boolean + maxInstance: + format: int32 + type: integer + minInstance: + format: int32 + type: integer + scaleDownWindows: + format: int32 + type: integer + scaleUpWindows: + format: int32 + type: integer + target: + format: int32 + type: integer + type: object + imagePullSecrets: + items: + description: LocalObjectReference contains enough information to + let you locate the referenced object inside the same namespace. properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute inside - the container, the working directory for the command is - root ('/') in the container's filesystem. The command is - simply exec'd, it is not run inside a shell, so traditional - shell instructions ('|', etc) won't work. To use a shell, - you need to explicitly call out to that shell. Exit status - of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe to be - considered failed after having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. This - is an alpha field and requires enabling GRPCContainerProbe feature - gate. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the service to place + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + type: array + livenessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command is + simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. This + is an alpha field and requires enabling GRPCContainerProbe feature + gate. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to the pod - IP. You probably want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows - repeated headers. - items: - description: HTTPHeader describes a custom header to be - used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access on the container. - Number must be in the range 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the host. Defaults - to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container has started + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod + IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. + Number must be in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults + to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. Default - to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe to be - considered successful after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default + to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be + considered successful after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access on the container. - Number must be in the range 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs to terminate - gracefully upon probe failure. The grace period is the duration - in seconds after the processes running in the pod are sent a - termination signal and the time when the processes are forcibly - halted with a kill signal. Set this value longer than the expected - cleanup time for your process. If this value is nil, the pod's - terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. Value must - be non-negative integer. The value zero indicates stop immediately - via the kill signal (no opportunity to shut down). This is a - beta field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which the probe times out. + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. + Number must be in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate + gracefully upon probe failure. The grace period is the duration + in seconds after the processes running in the pod are sent a + termination signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates stop immediately + via the kill signal (no opportunity to shut down). This is a + beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - mySQLDriver: - description: MySQLDriver Defines the mysql-driven version in ShardingSphere-proxy - properties: - version: - description: mysql-driven version,must be x.y.z - pattern: ^([1-9]\d|[1-9])(\.([1-9]\d|\d)){2}$ - type: string - required: - - version - type: object - port: - description: Port is ShardingSphere-Proxy startup port - format: int32 - minimum: 0 - type: integer - proxyConfigName: - description: ProxyConfigName is the name of the ProxyConfig CRD - minLength: 0 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - readinessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute inside - the container, the working directory for the command is - root ('/') in the container's filesystem. The command is - simply exec'd, it is not run inside a shell, so traditional - shell instructions ('|', etc) won't work. To use a shell, - you need to explicitly call out to that shell. Exit status - of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe to be - considered failed after having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. This - is an alpha field and requires enabling GRPCContainerProbe feature - gate. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the service to place + format: int32 + type: integer + type: object + mySQLDriver: + description: MySQLDriver Defines the mysql-driven version in ShardingSphere-proxy + properties: + version: + description: mysql-driven version,must be x.y.z + pattern: ^([1-9]\d|[1-9])(\.([1-9]\d|\d)){2}$ + type: string + required: + - version + type: object + port: + description: Port is ShardingSphere-Proxy startup port + format: int32 + minimum: 0 + type: integer + proxyConfigName: + description: ProxyConfigName is the name of the ProxyConfig CRD + minLength: 0 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + readinessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command is + simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. This + is an alpha field and requires enabling GRPCContainerProbe feature + gate. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to the pod - IP. You probably want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows - repeated headers. - items: - description: HTTPHeader describes a custom header to be - used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access on the container. - Number must be in the range 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the host. Defaults - to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container has started + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod + IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. + Number must be in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults + to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. Default - to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe to be - considered successful after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default + to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be + considered successful after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access on the container. - Number must be in the range 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs to terminate - gracefully upon probe failure. The grace period is the duration - in seconds after the processes running in the pod are sent a - termination signal and the time when the processes are forcibly - halted with a kill signal. Set this value longer than the expected - cleanup time for your process. If this value is nil, the pod's - terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. Value must - be non-negative integer. The value zero indicates stop immediately - via the kill signal (no opportunity to shut down). This is a - beta field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - replicas: - description: Replicas is the expected number of replicas of ShardingSphere-Proxy - format: int32 - type: integer - resources: - description: ResourceRequirements describes the compute resource requirements. - properties: - limits: - additionalProperties: + type: string + port: anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + - type: integer + - type: string + description: Number or name of the port to access on the container. + Number must be in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate + gracefully upon probe failure. The grace period is the duration + in seconds after the processes running in the pod are sent a + termination signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates stop immediately + via the kill signal (no opportunity to shut down). This is a + beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + replicas: + description: Replicas is the expected number of replicas of ShardingSphere-Proxy + format: int32 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - serviceType: - description: ServiceType defines the Service in Kubernetes of ShardingSphere-Proxy - properties: - nodePort: - description: 'The port on each node on which this service is exposed + type: object + type: object + serviceType: + description: ServiceType defines the Service in Kubernetes of ShardingSphere-Proxy + properties: + nodePort: + description: 'The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not @@ -405,226 +437,226 @@ spec: does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' - format: int32 - minimum: 0 - type: integer - type: - description: Service Type string describes ingress methods for - a service - enum: - - ClusterIP - - NodePort - - LoadBalancer - - ExternalName - type: string - required: - - type - type: object - startupProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: Command is the command line to execute inside - the container, the working directory for the command is - root ('/') in the container's filesystem. The command is - simply exec'd, it is not run inside a shell, so traditional - shell instructions ('|', etc) won't work. To use a shell, - you need to explicitly call out to that shell. Exit status - of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe to be - considered failed after having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. This - is an alpha field and requires enabling GRPCContainerProbe feature - gate. - properties: - port: - description: Port number of the gRPC service. Number must - be in the range 1 to 65535. - format: int32 - type: integer - service: - description: "Service is the name of the service to place + format: int32 + minimum: 0 + type: integer + type: + description: Service Type string describes ingress methods for + a service + enum: + - ClusterIP + - NodePort + - LoadBalancer + - ExternalName + type: string + required: + - type + type: object + startupProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command is + simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. This + is an alpha field and requires enabling GRPCContainerProbe feature + gate. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC." - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: Host name to connect to, defaults to the pod - IP. You probably want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows - repeated headers. - items: - description: HTTPHeader describes a custom header to be - used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port to access on the container. - Number must be in the range 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting to the host. Defaults - to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the container has started - before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. Default - to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe to be - considered successful after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults - to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port to access on the container. - Number must be in the range 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs to terminate - gracefully upon probe failure. The grace period is the duration - in seconds after the processes running in the pod are sent a - termination signal and the time when the processes are forcibly - halted with a kill signal. Set this value longer than the expected - cleanup time for your process. If this value is nil, the pod's - terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. Value must - be non-negative integer. The value zero indicates stop immediately - via the kill signal (no opportunity to shut down). This is a - beta field and requires enabling ProbeTerminationGracePeriod - feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds - is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: 'Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - version: - description: Version is the version of ShardingSphere-Proxy - type: string - required: - - port - - proxyConfigName - - replicas - - serviceType - - version - type: object - status: - description: ProxyStatus defines the observed state of Proxy - properties: - conditions: - description: Conditions The conditions array, the reason and message - fields - items: - description: Condition | **condition** | **status** | **directions**| - | ------------- | ---------- | ---------------------------------------------------- - | | Initialized | true | Initialization successful| | - Initialized | false | initialization failed| | Started | - true | pod started successfully but not ready| | Started | - false | pod started failed| | Ready | true | - The pod is ready and can provide external services| | Unknown | - true | ShardingSphere-Proxy failed to start correctly due - to some problems | + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. properties: - lastUpdateTime: - format: date-time + host: + description: Host name to connect to, defaults to the pod + IP. You probably want to set "Host" in httpHeaders instead. type: string - status: + httpHeaders: + description: Custom headers to set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. type: string - type: + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. + Number must be in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults + to HTTP. type: string required: - - status - - type + - port type: object - type: array - phase: - description: 'ShardingSphere-Proxy phase are a brief summary of the + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default + to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be + considered successful after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. + Number must be in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate + gracefully upon probe failure. The grace period is the duration + in seconds after the processes running in the pod are sent a + termination signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates stop immediately + via the kill signal (no opportunity to shut down). This is a + beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + version: + description: Version is the version of ShardingSphere-Proxy + type: string + required: + - port + - proxyConfigName + - replicas + - serviceType + - version + type: object + status: + description: ProxyStatus defines the observed state of Proxy + properties: + conditions: + description: Conditions The conditions array, the reason and message + fields + items: + description: Condition | **condition** | **status** | **directions**| + | ------------- | ---------- | ---------------------------------------------------- + | | Initialized | true | Initialization successful| | + Initialized | false | initialization failed| | Started | + true | pod started successfully but not ready| | Started | + false | pod started failed| | Ready | true | + The pod is ready and can provide external services| | Unknown | + true | ShardingSphere-Proxy failed to start correctly due + to some problems | + properties: + lastUpdateTime: + format: date-time + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + description: 'ShardingSphere-Proxy phase are a brief summary of the ShardingSphere-Proxy life cycle There are two possible phase values: Ready: ShardingSphere-Proxy can already provide external services NotReady: ShardingSphere-Proxy cannot provide external services' - type: string - readyNodes: - description: ReadyNodes shows the number of replicas that ShardingSphere-Proxy - is running normally - format: int32 - type: integer - required: - - conditions - - phase - - readyNodes - type: object - type: object - served: true - storage: true - subresources: - status: { } + type: string + readyNodes: + description: ReadyNodes shows the number of replicas that ShardingSphere-Proxy + is running normally + format: int32 + type: integer + required: + - conditions + - phase + - readyNodes + type: object + type: object + served: true + storage: true + subresources: + status: {} status: acceptedNames: kind: "" plural: "" - conditions: [ ] - storedVersions: [ ] + conditions: [] + storedVersions: [] diff --git a/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxyconfigs.yaml b/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxyconfigs.yaml index 38416c6..6e3cd8e 100644 --- a/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxyconfigs.yaml +++ b/charts/shardingsphere-operator/crds/shardingsphere.sphere-ex.com_proxyconfigs.yaml @@ -31,188 +31,188 @@ spec: singular: proxyconfig scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .status.metadataRepository - name: MetadataRepository - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: ProxyConfig is the Schema for the proxyconfigs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation + - additionalPrinterColumns: + - jsonPath: .status.metadataRepository + name: MetadataRepository + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ProxyConfig is the Schema for the proxyconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ProxyConfigSpec defines the desired state of ProxyConfig - properties: - authority: - description: Auth is used to set up initial user to login compute - node, and authority data of storage node. - properties: - privilege: - description: Privilege for storage node, the default value is - ALL_PRIVILEGES_PERMITTED + type: string + metadata: + type: object + spec: + description: ProxyConfigSpec defines the desired state of ProxyConfig + properties: + authority: + description: Auth is used to set up initial user to login compute + node, and authority data of storage node. + properties: + privilege: + description: Privilege for storage node, the default value is + ALL_PRIVILEGES_PERMITTED + properties: + type: + type: string + required: + - type + type: object + users: + items: + description: 'User is a slice about authorized host and password + for compute node. Format: user:<username>@<hostname>,hostname + is % or empty string means do not care about authorized host + password:<password>' properties: - type: + password: + type: string + user: type: string required: - - type + - password + - user type: object - users: - items: - description: 'User is a slice about authorized host and password - for compute node. Format: user:<username>@<hostname>,hostname - is % or empty string means do not care about authorized host - password:<password>' + type: array + required: + - users + type: object + mode: + description: ClusterConfig needs to fill in the relevant configuration + required by Cluster mode + properties: + overwrite: + description: Whether overwrite persistent configuration with local + configuration If it is false, it will not upload the local configuration + to ZooKeeper. If it is true, it will upload the local configuration + to ZooKeeper and overwrite the original configuration of ZooKeeper. + type: boolean + repository: + description: Persist repository configuration + properties: + props: + description: Properties of persist repository properties: - password: - type: string - user: + digest: + description: Password of login type: string - required: - - password - - user - type: object - type: array - required: - - users - type: object - mode: - description: ClusterConfig needs to fill in the relevant configuration - required by Cluster mode - properties: - overwrite: - description: Whether overwrite persistent configuration with local - configuration If it is false, it will not upload the local configuration - to ZooKeeper. If it is true, it will upload the local configuration - to ZooKeeper and overwrite the original configuration of ZooKeeper. - type: boolean - repository: - description: Persist repository configuration - properties: - props: - description: Properties of persist repository - properties: - digest: - description: Password of login - type: string - maxRetries: - description: 'MaxRetries Max retries of client connection. + maxRetries: + description: 'MaxRetries Max retries of client connection. default: 3' - type: integer - namespace: - description: Namespace of registry center - type: string - operationTimeoutMilliseconds: - description: 'OperationTimeoutMilliseconds Milliseconds + type: integer + namespace: + description: Namespace of registry center + type: string + operationTimeoutMilliseconds: + description: 'OperationTimeoutMilliseconds Milliseconds of operation timeout. default: 500' - type: integer - retryIntervalMilliseconds: - description: 'RetryIntervalMilliseconds Milliseconds of + type: integer + retryIntervalMilliseconds: + description: 'RetryIntervalMilliseconds Milliseconds of retry interval. default: 500' - type: integer - server-lists: - description: Server lists of registry center - type: string - timeToLiveSeconds: - description: 'TimeToLiveSeconds Seconds of ephemeral data + type: integer + server-lists: + description: Server lists of registry center + type: string + timeToLiveSeconds: + description: 'TimeToLiveSeconds Seconds of ephemeral data live.default: 60' - type: integer - required: - - namespace - - server-lists - type: object - type: - description: Type of persist repository - enum: - - ZooKeeper - type: string - required: - - props - - type - type: object - type: - description: 'Type of mode configuration. Values only support: - Cluster' - enum: - - Cluster - type: string - required: - - overwrite - - repository + type: integer + required: + - namespace + - server-lists + type: object + type: + description: Type of persist repository + enum: + - ZooKeeper + type: string + required: + - props - type - type: object - props: - description: Props Apache ShardingSphere provides the way of property - configuration to configure system level configuration. - properties: - check-duplicate-table-enabled: - description: Whether validate duplicate table when application - startup or updated. - type: boolean - check-table-metadata-enabled: - description: Whether validate table meta data consistency when - application startup or updated. - type: boolean - kernel-executor-size: - description: The max thread size of worker group to execute SQL. - One ShardingSphereDataSource will use a independent thread pool, - it does not share thread pool even different data source in - same JVM. - type: integer - proxy-backend-driver-type: - type: string - proxy-backend-executor-suitable: - description: 'Available options of proxy backend executor suitable: + type: object + type: + description: 'Type of mode configuration. Values only support: + Cluster' + enum: + - Cluster + type: string + required: + - overwrite + - repository + - type + type: object + props: + description: Props Apache ShardingSphere provides the way of property + configuration to configure system level configuration. + properties: + check-duplicate-table-enabled: + description: Whether validate duplicate table when application + startup or updated. + type: boolean + check-table-metadata-enabled: + description: Whether validate table meta data consistency when + application startup or updated. + type: boolean + kernel-executor-size: + description: The max thread size of worker group to execute SQL. + One ShardingSphereDataSource will use a independent thread pool, + it does not share thread pool even different data source in + same JVM. + type: integer + proxy-backend-driver-type: + type: string + proxy-backend-executor-suitable: + description: 'Available options of proxy backend executor suitable: OLAP(default), OLTP. The OLTP option may reduce time cost of writing packets to client, but it may increase the latency of SQL execution and block other clients if client connections are more than proxy-frontend-executor-size, especially executing slow SQL.' - type: string - proxy-backend-query-fetch-size: - description: Proxy backend query fetch size. A larger value may - increase the memory usage of ShardingSphere Proxy. The default - value is -1, which means set the minimum value for different - JDBC drivers. - type: integer - proxy-frontend-database-protocol-type: - type: string - proxy-frontend-executor-size: - description: Proxy frontend Netty executor size. The default value - is 0, which means let Netty decide. - type: integer - type: object - required: - - authority - - mode - type: object - status: - description: ProxyConfigStatus defines the observed state of ProxyConfig - properties: - metadataRepository: - type: string - required: - - metadataRepository - type: object - type: object - served: true - storage: true - subresources: - status: { } + type: string + proxy-backend-query-fetch-size: + description: Proxy backend query fetch size. A larger value may + increase the memory usage of ShardingSphere Proxy. The default + value is -1, which means set the minimum value for different + JDBC drivers. + type: integer + proxy-frontend-database-protocol-type: + type: string + proxy-frontend-executor-size: + description: Proxy frontend Netty executor size. The default value + is 0, which means let Netty decide. + type: integer + type: object + required: + - authority + - mode + type: object + status: + description: ProxyConfigStatus defines the observed state of ProxyConfig + properties: + metadataRepository: + type: string + required: + - metadataRepository + type: object + type: object + served: true + storage: true + subresources: + status: {} status: acceptedNames: kind: "" plural: "" - conditions: [ ] - storedVersions: [ ] + conditions: [] + storedVersions: [] diff --git a/charts/shardingsphere-operator/templates/deployment.yaml b/charts/shardingsphere-operator/templates/deployment.yaml index f059423..e85c58b 100644 --- a/charts/shardingsphere-operator/templates/deployment.yaml +++ b/charts/shardingsphere-operator/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: spec: containers: - args: - - --health-probe-bind-address=: {{ .Values.health.healthProbePort }} + - --health-probe-bind-address=:{{ .Values.health.healthProbePort }} - --leader-elect command: - /manager diff --git a/charts/shardingsphere-operator/values.yaml b/charts/shardingsphere-operator/values.yaml index 32f4106..b69cdba 100644 --- a/charts/shardingsphere-operator/values.yaml +++ b/charts/shardingsphere-operator/values.yaml @@ -21,13 +21,13 @@ replicaCount: 2 image: ## @param image.repository operator image name ## - repository: "sahrdingsphere-operator" + repository: "ghcr.io/sphereex/shardingsphere-on-cloud" ## @param image.pullPolicy image pull policy ## pullPolicy: IfNotPresent ## @param image.tag image tag ## - tag: "0.0.1" + tag: "0.1.0" ## @param imagePullSecrets image pull secret of private repository ## e.g: ## imagePullSecrets: diff --git a/shardingsphere-operator/api/v1alpha1/proxy_status.go b/shardingsphere-operator/api/v1alpha1/proxy_status.go index 58d1093..8f3ae69 100644 --- a/shardingsphere-operator/api/v1alpha1/proxy_status.go +++ b/shardingsphere-operator/api/v1alpha1/proxy_status.go @@ -127,3 +127,6 @@ func (p *Proxy) SetFailed() { LastUpdateTime: metav1.Now(), }) } +func (p *Proxy) UpdateReadyNodes(readyNodes int32) { + p.Status.ReadyNodes = readyNodes +} diff --git a/shardingsphere-operator/api/v1alpha1/proxy_types.go b/shardingsphere-operator/api/v1alpha1/proxy_types.go index 452c96f..8e1ba42 100644 --- a/shardingsphere-operator/api/v1alpha1/proxy_types.go +++ b/shardingsphere-operator/api/v1alpha1/proxy_types.go @@ -52,6 +52,22 @@ type MySQLDriver struct { Version string `json:"version"` } +// AutomaticScaling HPA configuration +type AutomaticScaling struct { + // +optional + Enable bool `json:"enable,omitempty"` + // +optional + ScaleUpWindows int32 `json:"scaleUpWindows,omitempty"` + // +optional + ScaleDownWindows int32 `json:"scaleDownWindows,omitempty"` + // +optional + Target int32 `json:"target,omitempty"` + // +optional + MaxInstance int32 `json:"maxInstance,omitempty"` + // +optional + MinInstance int32 `json:"minInstance,omitempty"` +} + // ProxySpec defines the desired state of Proxy type ProxySpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster @@ -62,7 +78,10 @@ type ProxySpec struct { ServiceType ServiceType `json:"serviceType"` //Replicas is the expected number of replicas of ShardingSphere-Proxy Replicas int32 `json:"replicas"` - + // +optional + AutomaticScaling *AutomaticScaling `json:"automaticScaling,omitempty"` + // +optional + ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"` // +kubebuilder:validation:MinLength=0 // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` diff --git a/shardingsphere-operator/api/v1alpha1/proxy_webhook.go b/shardingsphere-operator/api/v1alpha1/proxy_webhook.go index 67d6b5c..b216a20 100644 --- a/shardingsphere-operator/api/v1alpha1/proxy_webhook.go +++ b/shardingsphere-operator/api/v1alpha1/proxy_webhook.go @@ -56,7 +56,7 @@ func (r *Proxy) Default() { }, }, PeriodSeconds: 5, - FailureThreshold: 12, + FailureThreshold: 2, } } if r.Spec.ReadinessProbe == nil { @@ -83,12 +83,18 @@ func (r *Proxy) Default() { if r.Spec.Resources == nil { // The application for resources is based on the upper limit of cpu: 2core memory: 2Gi to apply for computing resources. // The cpu is applied according to 10%, and the memory is applied according to 80% - cpu, _ := resource.ParseQuantity("0.2") - memory, _ := resource.ParseQuantity("1.6Gi") + cpuLimits, _ := resource.ParseQuantity("4") + memoryLimits, _ := resource.ParseQuantity("2Gi") + cpuRequest, _ := resource.ParseQuantity("2.8") + memoryRequest, _ := resource.ParseQuantity("1600Mbi") r.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": cpuLimits, + "memory": memoryLimits, + }, Requests: v1.ResourceList{ - "cpu": cpu, - "memory": memory, + "cpu": cpuRequest, + "memory": memoryRequest, }, } } @@ -101,13 +107,29 @@ var _ webhook.Validator = &Proxy{} // ValidateCreate implements webhook.Validator so a webhook will be registered for the type func (r *Proxy) ValidateCreate() error { proxylog.Info("validate create", "name", r.Name) - return r.validateService() + err := r.validateService() + if err != nil { + return err + } + err = r.validateInstance() + if err != nil { + return err + } + return nil } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type func (r *Proxy) ValidateUpdate(old runtime.Object) error { proxylog.Info("validate update", "name", r.Name) - return r.validateService() + err := r.validateService() + if err != nil { + return err + } + err = r.validateInstance() + if err != nil { + return err + } + return nil } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type @@ -126,3 +148,15 @@ func (r *Proxy) validateService() error { } return nil } + +func (r *Proxy) validateInstance() error { + var allErrs field.ErrorList + if r.Spec.AutomaticScaling != nil && r.Spec.AutomaticScaling.MaxInstance < 1 { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("MaxInstance"), r.Spec.ServiceType, errors.New("If automatic scaling is enabled, the number of instances must be filled in.").Error())) + return apierrors.NewInvalid(schema.GroupKind{ + Group: "shardingsphere.sphere-ex.com", + Kind: "Proxy", + }, r.Name, allErrs) + } + return nil +} diff --git a/shardingsphere-operator/pkg/controllers/proxy_controller.go b/shardingsphere-operator/pkg/controllers/proxy_controller.go index 404774b..5e69311 100644 --- a/shardingsphere-operator/pkg/controllers/proxy_controller.go +++ b/shardingsphere-operator/pkg/controllers/proxy_controller.go @@ -20,6 +20,7 @@ package controllers import ( "context" appsv1 "k8s.io/api/apps/v1" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -34,6 +35,8 @@ import ( const ( //WaitingForReady Time selection reference kubelet restart time WaitingForReady = 10 * time.Second + //miniReadyCount Minimum number of replicas that can be served + miniReadyCount = 1 ) // ProxyReconciler reconciles a Proxy object @@ -49,6 +52,7 @@ type ProxyReconciler struct { //+kubebuilder:rbac:groups=apps,resources=deployment/status,verbs=get;list //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=pods/status,verbs=get;list;watch; +//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -68,6 +72,10 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl runtimeDeployment := &appsv1.Deployment{} err = r.Get(ctx, req.NamespacedName, runtimeDeployment) + if err != nil && !apierrors.IsNotFound(err) { + log.Error(err, "Error getting cascaded HPA") + return ctrl.Result{}, err + } if apierrors.IsNotFound(err) { cascadingDeployment := reconcile.ConstructCascadingDeployment(run) err = r.Create(ctx, cascadingDeployment) @@ -77,9 +85,6 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl log.Error(err, "Error creating cascaded deployment") return ctrl.Result{}, err } - } else if err != nil { - log.Error(err, "Error getting cascaded deployment") - return ctrl.Result{}, err } else { originDeployment := runtimeDeployment.DeepCopy() reconcile.UpdateDeployment(run, originDeployment) @@ -89,9 +94,44 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{Requeue: true}, err } } + runtimeHPA := &autoscalingv2beta2.HorizontalPodAutoscaler{} + err = r.Get(ctx, req.NamespacedName, runtimeHPA) + if err != nil && !apierrors.IsNotFound(err) { + log.Error(err, "Error getting cascaded HPA") + return ctrl.Result{}, err + } + if apierrors.IsNotFound(err) && run.Spec.AutomaticScaling != nil { + cascadingHPA := reconcile.ConstructHPA(run) + err = r.Create(ctx, cascadingHPA) + if err != nil { + run.SetInitializationFailed() + _ = r.Status().Update(ctx, run) + log.Error(err, "Error creating cascaded HPA") + return ctrl.Result{}, err + } + + } else if run.Spec.AutomaticScaling == nil { + err = r.Delete(ctx, runtimeHPA) + if err != nil { + log.Error(err, "Error delete cascaded HPA") + return ctrl.Result{}, err + } + } else { + originHPA := runtimeHPA.DeepCopy() + reconcile.UpdateHPA(run, originHPA) + err = r.Update(ctx, originHPA) + if err != nil { + log.Error(err, "Error updating cascaded HPA") + return ctrl.Result{}, err + } + } runtimeService := &v1.Service{} err = r.Get(ctx, req.NamespacedName, runtimeService) + if err != nil && !apierrors.IsNotFound(err) { + log.Error(err, "Error getting cascaded HPA") + return ctrl.Result{}, err + } if apierrors.IsNotFound(err) { cascadingService := reconcile.ConstructCascadingService(run) err = r.Create(ctx, cascadingService) @@ -103,9 +143,6 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } run.SetInitialized() return ctrl.Result{RequeueAfter: WaitingForReady}, nil - } else if err != nil { - log.Error(err, "Error getting cascaded service") - return ctrl.Result{}, err } else { originService := runtimeService.DeepCopy() reconcile.UpdateService(run, originService) @@ -126,7 +163,7 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl result := ctrl.Result{} readyNodes := reconcile.CountingReadyPods(podList) if reconcile.IsRunning(podList) { - if readyNodes != run.Spec.Replicas { + if readyNodes < miniReadyCount { result.RequeueAfter = WaitingForReady if readyNodes != run.Status.ReadyNodes { run.SetPodStarted(readyNodes) @@ -135,6 +172,8 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl if run.Status.Phase != v1alpha1.StatusReady { log.Info("Status is now ready!") run.SetReady(readyNodes) + } else if readyNodes != *runtimeDeployment.Spec.Replicas { + run.UpdateReadyNodes(readyNodes) } } } else { diff --git a/shardingsphere-operator/pkg/reconcile/resource.go b/shardingsphere-operator/pkg/reconcile/resource.go index 34a0d7a..6829e94 100644 --- a/shardingsphere-operator/pkg/reconcile/resource.go +++ b/shardingsphere-operator/pkg/reconcile/resource.go @@ -22,6 +22,7 @@ import ( "gopkg.in/yaml.v2" "html/template" appsv1 "k8s.io/api/apps/v1" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -32,6 +33,39 @@ import ( const imageName = "apache/shardingsphere-proxy" +var logback = `<?xml version="1.0"?> +<configuration> + <appender name="console" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %logger{36} - %msg%n</pattern> + </encoder> + </appender> + <appender name="sqlConsole" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] [%X{database}] [%X{user}] [%X{host}] %logger{36} - %msg%n</pattern> + </encoder> + </appender> + + <logger name="ShardingSphere-SQL" level="info" additivity="false"> + <appender-ref ref="sqlConsole" /> + </logger> + <logger name="org.apache.shardingsphere" level="info" additivity="false"> + <appender-ref ref="console" /> + </logger> + + <logger name="com.zaxxer.hikari" level="error" /> + + <logger name="com.atomikos" level="error" /> + + <logger name="io.netty" level="error" /> + + <root> + <level value="info" /> + <appender-ref ref="console" /> + </root> +</configuration> +` + func ConstructCascadingDeployment(proxy *v1alpha1.Proxy) *appsv1.Deployment { dp := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -45,7 +79,6 @@ func ConstructCascadingDeployment(proxy *v1alpha1.Proxy) *appsv1.Deployment { Strategy: appsv1.DeploymentStrategy{ Type: appsv1.RecreateDeploymentStrategyType, }, - Replicas: &proxy.Spec.Replicas, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "apps": proxy.Name, @@ -98,10 +131,16 @@ func ConstructCascadingDeployment(proxy *v1alpha1.Proxy) *appsv1.Deployment { }, }, } + if proxy.Spec.AutomaticScaling == nil { + dp.Spec.Replicas = &proxy.Spec.Replicas + } dp.Spec.Template.Spec.Containers[0].Resources = *proxy.Spec.Resources dp.Spec.Template.Spec.Containers[0].LivenessProbe = proxy.Spec.LivenessProbe dp.Spec.Template.Spec.Containers[0].ReadinessProbe = proxy.Spec.ReadinessProbe dp.Spec.Template.Spec.Containers[0].StartupProbe = proxy.Spec.StartupProbe + if len(proxy.Spec.ImagePullSecrets) > 0 { + dp.Spec.Template.Spec.ImagePullSecrets = proxy.Spec.ImagePullSecrets + } return processOptionalParameter(proxy, dp) } @@ -195,6 +234,57 @@ func ConstructCascadingConfigmap(proxyConfig *v1alpha1.ProxyConfig) *v1.ConfigMa }, Data: map[string]string{ "server.yaml": y, + "logback.xml": logback, + }, + } + +} + +// ConstructHPA Create HPA if you need +func ConstructHPA(proxy *v1alpha1.Proxy) *autoscalingv2beta2.HorizontalPodAutoscaler { + return &autoscalingv2beta2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: proxy.Name, + Namespace: proxy.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(proxy.GetObjectMeta(), proxy.GroupVersionKind()), + }, + }, + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: proxy.Name, + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + MinReplicas: &proxy.Spec.AutomaticScaling.MinInstance, + MaxReplicas: proxy.Spec.AutomaticScaling.MaxInstance, + Metrics: []autoscalingv2beta2.MetricSpec{ + { + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: "cpu", + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.UtilizationMetricType, + AverageUtilization: &proxy.Spec.AutomaticScaling.Target, + }, + }, + }, + }, + Behavior: &autoscalingv2beta2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2beta2.HPAScalingRules{ + StabilizationWindowSeconds: &proxy.Spec.AutomaticScaling.ScaleUpWindows, + }, + ScaleDown: &autoscalingv2beta2.HPAScalingRules{ + StabilizationWindowSeconds: &proxy.Spec.AutomaticScaling.ScaleDownWindows, + Policies: []autoscalingv2beta2.HPAScalingPolicy{ + { + Type: autoscalingv2beta2.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 30, + }, + }, + }, + }, }, } @@ -209,7 +299,9 @@ func toYaml(proxyConfig *v1alpha1.ProxyConfig) string { // UpdateDeployment FIXME:merge UpdateDeployment and ConstructCascadingDeployment func UpdateDeployment(proxy *v1alpha1.Proxy, runtimeDeployment *appsv1.Deployment) { runtimeDeployment.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", imageName, proxy.Spec.Version) - runtimeDeployment.Spec.Replicas = &proxy.Spec.Replicas + if proxy.Spec.AutomaticScaling == nil { + runtimeDeployment.Spec.Replicas = &proxy.Spec.Replicas + } runtimeDeployment.Spec.Template.Spec.Volumes[0].ConfigMap.Name = proxy.Spec.ProxyConfigName runtimeDeployment.Spec.Template.Spec.Containers[0].Env[0].Value = strconv.FormatInt(int64(proxy.Spec.Port), 10) runtimeDeployment.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = proxy.Spec.Port @@ -232,6 +324,14 @@ func UpdateService(proxy *v1alpha1.Proxy, runtimeService *v1.Service) { } } +func UpdateHPA(proxy *v1alpha1.Proxy, runtimeHPA *autoscalingv2beta2.HorizontalPodAutoscaler) { + runtimeHPA.Spec.Metrics[0].Resource.Target.AverageUtilization = &proxy.Spec.AutomaticScaling.Target + runtimeHPA.Spec.Behavior.ScaleDown.StabilizationWindowSeconds = &proxy.Spec.AutomaticScaling.ScaleDownWindows + runtimeHPA.Spec.Behavior.ScaleUp.StabilizationWindowSeconds = &proxy.Spec.AutomaticScaling.ScaleUpWindows + runtimeHPA.Spec.MaxReplicas = proxy.Spec.AutomaticScaling.MaxInstance + runtimeHPA.Spec.MinReplicas = &proxy.Spec.AutomaticScaling.MinInstance +} + func fromInt32(val int32) intstr.IntOrString { return intstr.IntOrString{Type: intstr.Int, IntVal: val} }
