This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 4a76a07c9c [Improvement][Helm] Make configmap of
api/master/worker/alert configuration (#16058)
4a76a07c9c is described below
commit 4a76a07c9c94e7a7505b63c02c5b78c8da635baa
Author: JohnHuang <[email protected]>
AuthorDate: Fri Jun 21 04:37:19 2024 -0400
[Improvement][Helm] Make configmap of api/master/worker/alert configuration
(#16058)
Update
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
* Update deploy/kubernetes/dolphinscheduler/values.yaml
---
deploy/kubernetes/dolphinscheduler/README.md | 9 +-
.../configmap-dolphinscheduler-alert.yaml | 97 +--------
.../templates/configmap-dolphinscheduler-api.yaml | 240 +--------------------
.../configmap-dolphinscheduler-master.yaml | 155 +------------
.../configmap-dolphinscheduler-worker.yaml | 90 +-------
.../deployment-dolphinscheduler-alert.yaml | 4 +
.../templates/deployment-dolphinscheduler-api.yaml | 4 +
.../statefulset-dolphinscheduler-master.yaml | 4 +
.../statefulset-dolphinscheduler-worker.yaml | 4 +
deploy/kubernetes/dolphinscheduler/values.yaml | 53 ++++-
10 files changed, 98 insertions(+), 562 deletions(-)
diff --git a/deploy/kubernetes/dolphinscheduler/README.md
b/deploy/kubernetes/dolphinscheduler/README.md
index 4b83f5b160..4a6b880293 100644
--- a/deploy/kubernetes/dolphinscheduler/README.md
+++ b/deploy/kubernetes/dolphinscheduler/README.md
@@ -14,6 +14,8 @@ Please refer to the [Quick Start in
Kubernetes](../../../docs/docs/en/guide/inst
|-----|------|---------|-------------|
| alert.affinity | object | `{}` | Affinity is a group of affinity scheduling
rules. If specified, the pod's scheduling constraints. More info:
[node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity)
|
| alert.annotations | object | `{}` | You can use annotations to attach
arbitrary non-identifying metadata to objects. Clients such as tools and
libraries can retrieve this metadata. |
+| alert.customizedConfig | object | `{}` | configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
|
+| alert.enableCustomizedConfig | bool | `false` | enable configure custom
config |
| alert.enabled | bool | `true` | Enable or disable the Alert-Server component
|
| alert.env.JAVA_OPTS | string | `"-Xms512m -Xmx512m -Xmn256m"` | The jvm
options for alert server |
| alert.livenessProbe | object |
`{"enabled":true,"failureThreshold":"3","initialDelaySeconds":"30","periodSeconds":"30","successThreshold":"1","timeoutSeconds":"5"}`
| Periodic probe of container liveness. Container will be restarted if the
probe fails. More info:
[container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
|
@@ -52,6 +54,8 @@ Please refer to the [Quick Start in
Kubernetes](../../../docs/docs/en/guide/inst
| alert.tolerations | list | `[]` | Tolerations are appended (excluding
duplicates) to pods running with this RuntimeClass during admission,
effectively unioning the set of nodes tolerated by the pod and the
RuntimeClass. |
| api.affinity | object | `{}` | Affinity is a group of affinity scheduling
rules. If specified, the pod's scheduling constraints. More info:
[node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity)
|
| api.annotations | object | `{}` | You can use annotations to attach
arbitrary non-identifying metadata to objects. Clients such as tools and
libraries can retrieve this metadata. |
+| api.customizedConfig | object | `{}` | configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-api/src/main/resources/application.yaml
|
+| api.enableCustomizedConfig | bool | `false` | enable configure custom config
|
| api.enabled | bool | `true` | Enable or disable the API-Server component |
| api.env.JAVA_OPTS | string | `"-Xms512m -Xmx512m -Xmn256m"` | The jvm
options for api server |
| api.livenessProbe | object |
`{"enabled":true,"failureThreshold":"3","initialDelaySeconds":"30","periodSeconds":"30","successThreshold":"1","timeoutSeconds":"5"}`
| Periodic probe of container liveness. Container will be restarted if the
probe fails. More info:
[container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
|
@@ -191,6 +195,8 @@ Please refer to the [Quick Start in
Kubernetes](../../../docs/docs/en/guide/inst
| initImage.pullPolicy | string | `"IfNotPresent"` | Image pull policy.
Options: Always, Never, IfNotPresent |
| master.affinity | object | `{}` | Affinity is a group of affinity scheduling
rules. If specified, the pod's scheduling constraints. More info:
[node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity)
|
| master.annotations | object | `{}` | You can use annotations to attach
arbitrary non-identifying metadata to objects. Clients such as tools and
libraries can retrieve this metadata. |
+| master.customizedConfig | object | `{}` | configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-master/src/main/resources/application.yaml
|
+| master.enableCustomizedConfig | bool | `false` | enable configure custom
config |
| master.enabled | bool | `true` | Enable or disable the Master component |
| master.env.JAVA_OPTS | string | `"-Xms1g -Xmx1g -Xmn512m"` | The jvm options
for master server |
| master.env.MASTER_DISPATCH_TASK_NUM | string | `"3"` | Master dispatch task
number per batch |
@@ -297,6 +303,8 @@ Please refer to the [Quick Start in
Kubernetes](../../../docs/docs/en/guide/inst
| timezone | string | `"Asia/Shanghai"` | World time and date for cities in
all time zones |
| worker.affinity | object | `{}` | Affinity is a group of affinity scheduling
rules. If specified, the pod's scheduling constraints. More info:
[node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity)
|
| worker.annotations | object | `{}` | You can use annotations to attach
arbitrary non-identifying metadata to objects. Clients such as tools and
libraries can retrieve this metadata. |
+| worker.customizedConfig | object | `{}` | configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-worker/src/main/resources/application.yaml
|
+| worker.enableCustomizedConfig | bool | `false` | enable configure custom
config |
| worker.enabled | bool | `true` | Enable or disable the Worker component |
| worker.env.DEFAULT_TENANT_ENABLED | bool | `false` | If set true, will use
worker bootstrap user as the tenant to execute task when the tenant is
`default`; |
| worker.env.WORKER_EXEC_THREADS | string | `"100"` | Worker execute thread
number to limit task instances |
@@ -315,7 +323,6 @@ Please refer to the [Quick Start in
Kubernetes](../../../docs/docs/en/guide/inst
| worker.keda.minReplicaCount | int | `0` | Minimum number of workers created
by keda |
| worker.keda.namespaceLabels | object | `{}` | Keda namespace labels |
| worker.keda.pollingInterval | int | `5` | How often KEDA polls the
DolphinScheduler DB to report new scale requests to the HPA |
-| worker.livenessProbe | object |
`{"enabled":true,"failureThreshold":"3","initialDelaySeconds":"30","periodSeconds":"30","successThreshold":"1","timeoutSeconds":"5"}`
| Periodic probe of container liveness. Container will be restarted if the
probe fails. More info:
[container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
|
| worker.livenessProbe.enabled | bool | `true` | Turn on and off liveness
probe |
| worker.livenessProbe.failureThreshold | string | `"3"` | Minimum consecutive
failures for the probe |
| worker.livenessProbe.initialDelaySeconds | string | `"30"` | Delay before
liveness probe is initiated |
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
index fd89c1ed15..15bffe06f2 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-{{- if and .Values.alert.enabled }}
+{{- if .Values.alert.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@@ -23,93 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
data:
- application.yaml: |
- spring:
- profiles:
- active: {{ .Values.datasource.profile }}
- jackson:
- time-zone: UTC
- date-format: "yyyy-MM-dd HH:mm:ss"
- banner:
- charset: UTF-8
- datasource:
- driver-class-name: org.postgresql.Driver
- url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
- username: root
- password: root
- hikari:
- connection-test-query: select 1
- pool-name: DolphinScheduler
-
- # Mybatis-plus configuration, you don't need to change it
- mybatis-plus:
- mapper-locations:
classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
- type-aliases-package: org.apache.dolphinscheduler.dao.entity
- configuration:
- cache-enabled: false
- call-setters-on-nulls: true
- map-underscore-to-camel-case: true
- jdbc-type-for-null: NULL
- global-config:
- db-config:
- id-type: auto
- banner: false
-
- server:
- port: 50053
-
- management:
- endpoints:
- web:
- exposure:
- include: health,metrics,prometheus
- endpoint:
- health:
- enabled: true
- show-details: always
- health:
- db:
- enabled: true
- defaults:
- enabled: false
- metrics:
- tags:
- application: ${spring.application.name}
-
- alert:
- port: 50052
- # Mark each alert of alert server if late after x milliseconds as failed.
- # Define value is (0 = infinite), and alert server would be waiting
alert result.
- wait-timeout: 0
- max-heartbeat-interval: 60s
- query_alert_threshold: 100
-
- registry:
- type: zookeeper
- zookeeper:
- namespace: dolphinscheduler
- connect-string: localhost:2181
- retry-policy:
- base-sleep-time: 60ms
- max-sleep: 300ms
- max-retries: 5
- session-timeout: 30s
- connection-timeout: 9s
- block-until-connected: 600ms
- digest: ~
-
- metrics:
- enabled: true
-
- # Override by profile
- ---
- spring:
- config:
- activate:
- on-profile: mysql
- datasource:
- driver-class-name: com.mysql.cj.jdbc.Driver
- url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
- username: root
- password: root
-{{- end }}
+{{- range $path, $config := .Values.alert.customizedConfig }}
+ {{ $path }}: |
+{{ $config | indent 4 -}}
+{{- end -}}
+{{- end -}}
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
index f115490c31..211d3dfda9 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-{{- if and .Values.api.enabled }}
+{{- if .Values.api.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@@ -23,236 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
{{- include "dolphinscheduler.api.labels" . | nindent 4 }}
data:
- application.yaml: |
- server:
- port: 12345
- servlet:
- session:
- timeout: 120m
- context-path: /dolphinscheduler/
- compression:
- enabled: true
- mime-types:
text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
- jetty:
- max-http-form-post-size: 5000000
- accesslog:
- enabled: true
- custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms'
-
- spring:
- profiles:
- active: {{ .Values.datasource.profile }}
- banner:
- charset: UTF-8
- jackson:
- time-zone: UTC
- date-format: "yyyy-MM-dd HH:mm:ss"
- servlet:
- multipart:
- max-file-size: 1024MB
- max-request-size: 1024MB
- messages:
- basename: i18n/messages
- datasource:
- driver-class-name: org.postgresql.Driver
- url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
- username: root
- password: root
- hikari:
- connection-test-query: select 1
- pool-name: DolphinScheduler
- quartz:
- auto-startup: false
- job-store-type: jdbc
- jdbc:
- initialize-schema: never
- properties:
- org.quartz.jobStore.isClustered: true
- org.quartz.jobStore.class:
org.springframework.scheduling.quartz.LocalDataSourceJobStore
- org.quartz.scheduler.instanceId: AUTO
- org.quartz.jobStore.tablePrefix: QRTZ_
- org.quartz.jobStore.acquireTriggersWithinLock: true
- org.quartz.scheduler.instanceName: DolphinScheduler
- org.quartz.threadPool.class:
org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool
- org.quartz.jobStore.useProperties: false
- org.quartz.jobStore.misfireThreshold: 60000
- org.quartz.scheduler.makeSchedulerThreadDaemon: true
- org.quartz.jobStore.driverDelegateClass:
org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
- org.quartz.jobStore.clusterCheckinInterval: 5000
- org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
- mvc:
- pathmatch:
- matching-strategy: ANT_PATH_MATCHER
- static-path-pattern: /static/**
- springdoc:
- swagger-ui:
- path: /swagger-ui.html
- packages-to-scan: org.apache.dolphinscheduler.api
-
- # Mybatis-plus configuration, you don't need to change it
- mybatis-plus:
- mapper-locations:
classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
- type-aliases-package: org.apache.dolphinscheduler.dao.entity
- configuration:
- cache-enabled: false
- call-setters-on-nulls: true
- map-underscore-to-camel-case: true
- jdbc-type-for-null: NULL
- global-config:
- db-config:
- id-type: auto
- banner: false
-
- management:
- endpoints:
- web:
- exposure:
- include: health,metrics,prometheus
- endpoint:
- health:
- enabled: true
- show-details: always
- health:
- db:
- enabled: true
- defaults:
- enabled: false
- metrics:
- tags:
- application: ${spring.application.name}
-
- registry:
- type: zookeeper
- zookeeper:
- namespace: dolphinscheduler
- connect-string: localhost:2181
- retry-policy:
- base-sleep-time: 60ms
- max-sleep: 300ms
- max-retries: 5
- session-timeout: 60s
- connection-timeout: 15s
- block-until-connected: 15s
- digest: ~
-
- api:
- audit-enable: false
- # Traffic control, if you turn on this config, the maximum number of
request/s will be limited.
- # global max request number per second
- # default tenant-level max request number
- traffic-control:
- global-switch: false
- max-global-qps-rate: 300
- tenant-switch: false
- default-tenant-qps-rate: 10
- #customize-tenant-qps-rate:
- # eg.
- #tenant1: 11
- #tenant2: 20
- python-gateway:
- # Weather enable python gateway server or not. The default value is
false.
- enabled: false
- # Authentication token for connection from python api to python
gateway server. Should be changed the default value
- # when you deploy in public network.
- auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
- # The address of Python gateway server start. Set its value to
`0.0.0.0` if your Python API run in different
- # between Python gateway server. It could be be specific to other
address like `127.0.0.1` or `localhost`
- gateway-server-address: 0.0.0.0
- # The port of Python gateway server start. Define which port you could
connect to Python gateway server from
- # Python API side.
- gateway-server-port: 25333
- # The address of Python callback client.
- python-address: 127.0.0.1
- # The port of Python callback client.
- python-port: 25334
- # Close connection of socket server if no other request accept after x
milliseconds. Define value is (0 = infinite),
- # and socket server would never close even though no requests accept
- connect-timeout: 0
- # Close each active connection of socket server if python program not
active after x milliseconds. Define value is
- # (0 = infinite), and socket server would never close even though no
requests accept
- read-timeout: 0
-
- metrics:
- enabled: true
-
- security:
- authentication:
- # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
- type: PASSWORD
- # IF you set type `LDAP`, below config will be effective
- ldap:
- # ldap server config
- urls: ldap://ldap.forumsys.com:389/
- base-dn: dc=example,dc=com
- username: cn=read-only-admin,dc=example,dc=com
- password: password
- user:
- # admin userId when you use LDAP login
- admin: read-only-admin
- identity-attribute: uid
- email-attribute: mail
- # action when ldap user is not exist (supported types: CREATE,DENY)
- not-exist-action: CREATE
- ssl:
- enable: false
- # jks file absolute path && password
- trust-store: "/ldapkeystore.jks"
- trust-store-password: "password"
- casdoor:
- user:
- admin: ""
- oauth2:
- enable: false
- provider:
- github:
- authorizationUri: ""
- redirectUri: ""
- clientId: ""
- clientSecret: ""
- tokenUri: ""
- userInfoUri: ""
- callbackUrl: ""
- iconUri: ""
- provider: github
- google:
- authorizationUri: ""
- redirectUri: ""
- clientId: ""
- clientSecret: ""
- tokenUri: ""
- userInfoUri: ""
- callbackUrl: ""
- iconUri: ""
- provider: google
- casdoor:
- # Your Casdoor server url
- endpoint: ""
- client-id: ""
- client-secret: ""
- # The certificate may be multi-line, you can use `|-` for ease
- certificate: ""
- # Your organization name added in Casdoor
- organization-name: ""
- # Your application name added in Casdoor
- application-name: ""
- # Doplhinscheduler login url
- redirect-url: ""
-
- # Override by profile
- ---
- spring:
- config:
- activate:
- on-profile: mysql
- datasource:
- driver-class-name: com.mysql.cj.jdbc.Driver
- url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
- username: root
- password: root
- quartz:
- properties:
- org.quartz.jobStore.driverDelegateClass:
org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-{{- end }}
-
-
-
+{{- range $path, $config := .Values.api.customizedConfig }}
+ {{ $path }}: |
+{{ $config | indent 4 -}}
+{{- end -}}
+{{- end -}}
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
index 0a2c8bf2a4..9bcb7dd411 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-{{- if and .Values.master.enabled }}
+{{- if .Values.master.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@@ -23,151 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
{{- include "dolphinscheduler.master.labels" . | nindent 4 }}
data:
- application.yaml: |
- spring:
- profiles:
- active: {{ .Values.datasource.profile }}
- banner:
- charset: UTF-8
- jackson:
- time-zone: UTC
- date-format: "yyyy-MM-dd HH:mm:ss"
- datasource:
- driver-class-name: org.postgresql.Driver
- url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
- username: root
- password: root
- hikari:
- connection-test-query: select 1
- pool-name: DolphinScheduler
- quartz:
- job-store-type: jdbc
- jdbc:
- initialize-schema: never
- properties:
- org.quartz.threadPool.threadPriority: 5
- org.quartz.jobStore.isClustered: true
- org.quartz.jobStore.class:
org.springframework.scheduling.quartz.LocalDataSourceJobStore
- org.quartz.scheduler.instanceId: AUTO
- org.quartz.jobStore.tablePrefix: QRTZ_
- org.quartz.jobStore.acquireTriggersWithinLock: true
- org.quartz.scheduler.instanceName: DolphinScheduler
- org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
- org.quartz.jobStore.useProperties: false
- org.quartz.threadPool.makeThreadsDaemons: true
- org.quartz.threadPool.threadCount: 25
- org.quartz.jobStore.misfireThreshold: 60000
- org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
- org.quartz.scheduler.makeSchedulerThreadDaemon: true
- org.quartz.jobStore.driverDelegateClass:
org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
- org.quartz.jobStore.clusterCheckinInterval: 5000
-
- # Mybatis-plus configuration, you don't need to change it
- mybatis-plus:
- mapper-locations:
classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
- type-aliases-package: org.apache.dolphinscheduler.dao.entity
- configuration:
- cache-enabled: false
- call-setters-on-nulls: true
- map-underscore-to-camel-case: true
- jdbc-type-for-null: NULL
- global-config:
- db-config:
- id-type: auto
- banner: false
-
-
- registry:
- type: zookeeper
- zookeeper:
- namespace: dolphinscheduler
- connect-string: localhost:2181
- retry-policy:
- base-sleep-time: 60ms
- max-sleep: 300ms
- max-retries: 5
- session-timeout: 30s
- connection-timeout: 9s
- block-until-connected: 600ms
- digest: ~
-
- master:
- listen-port: 5678
- # master fetch command num
- fetch-command-num: 10
- # master prepare execute thread number to limit handle commands in
parallel
- pre-exec-threads: 10
- # master execute thread number to limit process instances in parallel
- exec-threads: 100
- # master dispatch task number per batch, if all the tasks dispatch
failed in a batch, will sleep 1s.
- dispatch-task-number: 3
- # master host selector to select a suitable worker, default value:
LowerWeight. Optional values include random, round_robin, lower_weight
- host-selector: lower_weight
- # master heartbeat interval
- max-heartbeat-interval: 10s
- # master commit task retry times
- task-commit-retry-times: 5
- # master commit task interval
- task-commit-interval: 1s
- state-wheel-interval: 5s
- server-load-protection:
- # If set true, will open master overload protection
- enabled: true
- # Master max system cpu usage, when the master's system cpu usage is
smaller then this value, master server can execute workflow.
- max-system-cpu-usage-percentage-thresholds: 0.7
- # Master max jvm cpu usage, when the master's jvm cpu usage is smaller
then this value, master server can execute workflow.
- max-jvm-cpu-usage-percentage-thresholds: 0.7
- # Master max System memory usage , when the master's system memory
usage is smaller then this value, master server can execute workflow.
- max-system-memory-usage-percentage-thresholds: 0.7
- # Master max disk usage , when the master's disk usage is smaller then
this value, master server can execute workflow.
- max-disk-usage-percentage-thresholds: 0.7
- # failover interval, the unit is minute
- failover-interval: 10m
- # kill yarn / k8s application when failover taskInstance, default true
- kill-application-when-task-failover: true
- registry-disconnect-strategy:
- # The disconnect strategy: stop, waiting
- strategy: waiting
- # The max waiting time to reconnect to registry if you set the
strategy to waiting
- max-waiting-time: 100s
- worker-group-refresh-interval: 10s
-
- server:
- port: 5679
-
- management:
- endpoints:
- web:
- exposure:
- include: health,metrics,prometheus
- endpoint:
- health:
- enabled: true
- show-details: always
- health:
- db:
- enabled: true
- defaults:
- enabled: false
- metrics:
- tags:
- application: ${spring.application.name}
-
- metrics:
- enabled: true
-
- # Override by profile
- ---
- spring:
- config:
- activate:
- on-profile: mysql
- datasource:
- driver-class-name: com.mysql.cj.jdbc.Driver
- url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
- username: root
- password: root
- quartz:
- properties:
- org.quartz.jobStore.driverDelegateClass:
org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-{{- end }}
+{{- range $path, $config := .Values.master.customizedConfig }}
+ {{ $path }}: |
+{{ $config | indent 4 -}}
+{{- end -}}
+{{- end -}}
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
index b15cad5649..c1d81a1802 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-{{- if and .Values.worker.enabled }}
+{{- if .Values.worker.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@@ -23,86 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
{{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
data:
- application.yaml: |
- spring:
- banner:
- charset: UTF-8
- jackson:
- time-zone: UTC
- date-format: "yyyy-MM-dd HH:mm:ss"
- autoconfigure:
- exclude:
- -
org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
-
- registry:
- type: zookeeper
- zookeeper:
- namespace: dolphinscheduler
- connect-string: localhost:2181
- retry-policy:
- base-sleep-time: 60ms
- max-sleep: 300ms
- max-retries: 5
- session-timeout: 30s
- connection-timeout: 9s
- block-until-connected: 600ms
- digest: ~
-
- worker:
- # worker listener port
- listen-port: 1234
- # worker execute thread number to limit task instances in parallel
- exec-threads: 100
- # worker heartbeat interval
- max-heartbeat-interval: 10s
- # worker host weight to dispatch tasks, default value 100
- host-weight: 100
- server-load-protection:
- # If set true, will open worker overload protection
- enabled: true
- # Worker max system cpu usage, when the worker's system cpu usage is
smaller then this value, worker server can be dispatched tasks.
- max-system-cpu-usage-percentage-thresholds: 0.7
- # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller
then this value, worker server can be dispatched tasks.
- max-jvm-cpu-usage-percentage-thresholds: 0.7
- # Worker max System memory usage , when the master's system memory
usage is smaller then this value, master server can execute workflow.
- max-system-memory-usage-percentage-thresholds: 0.7
- # Worker max disk usage , when the worker's disk usage is smaller then
this value, worker server can be dispatched tasks.
- max-disk-usage-percentage-thresholds: 0.7
- registry-disconnect-strategy:
- # The disconnect strategy: stop, waiting
- strategy: waiting
- # The max waiting time to reconnect to registry if you set the
strategy to waiting
- max-waiting-time: 100s
- task-execute-threads-full-policy: REJECT
- tenant-config:
- # tenant corresponds to the user of the system, which is used by the
worker to submit the job. If system does not have this user, it will be
automatically created after the parameter worker.tenant.auto.create is true.
- auto-create-tenant-enabled: true
- # Scenes to be used for distributed users. For example, users created
by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this
parameter is true, auto-create-tenant-enabled has no effect and will not
automatically create tenants.
- distributed-tenant-enabled: false
- # If set true, will use worker bootstrap user as the tenant to execute
task when the tenant is `default`.
- default-tenant-enabled: false
-
- server:
- port: 1235
-
- management:
- endpoints:
- web:
- exposure:
- include: health,metrics,prometheus
- endpoint:
- health:
- enabled: true
- show-details: always
- health:
- db:
- enabled: true
- defaults:
- enabled: false
- metrics:
- tags:
- application: ${spring.application.name}
-
- metrics:
- enabled: true
-{{- end }}
+{{- range $path, $config := .Values.worker.customizedConfig }}
+ {{ $path }}: |
+{{ $config | indent 4 -}}
+{{- end -}}
+{{- end -}}
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
index 6a5560b3d4..7a90d08025 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
@@ -115,9 +115,11 @@ spec:
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
subPath: common.properties
+ {{- if .Values.alert.enableCustomizedConfig }}
- name: alert-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
+ {{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- if .Values.alert.persistentVolumeClaim.enabled }}
@@ -129,7 +131,9 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
+ {{- if .Values.alert.enableCustomizedConfig }}
- name: alert-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-alert
+ {{- end }}
{{- end }}
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
index f36d2442ae..b4fa07256c 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
@@ -121,9 +121,11 @@ spec:
mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml
subPath: task-type-config.yaml
{{- end }}
+ {{- if .Values.api.enableCustomizedConfig }}
- name: api-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
+ {{- end }}
{{- include "dolphinscheduler.sharedStorage.volumeMount" . |
nindent 12 }}
{{- include "dolphinscheduler.fsFileResource.volumeMount" . |
nindent 12 }}
{{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12
}}
@@ -139,9 +141,11 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
+ {{- if .Values.api.enableCustomizedConfig }}
- name: api-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-api
+ {{- end }}
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
{{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
index 1d9d517d0e..c4174b9ca0 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
@@ -109,9 +109,11 @@ spec:
volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-master
+ {{- if .Values.master.enableCustomizedConfig }}
- name: master-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
+ {{- end }}
{{- include "dolphinscheduler.sharedStorage.volumeMount" . |
nindent 12 }}
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
@@ -125,9 +127,11 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
+ {{- if .Values.master.enableCustomizedConfig }}
- name: master-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-master
+ {{- end }}
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
- name: config-volume
configMap:
diff --git
a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
index 8d55be6f70..4c66ca5ffe 100644
---
a/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
+++
b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
@@ -111,9 +111,11 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-worker-data
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
+ {{- if .Values.worker.enableCustomizedConfig }}
- name: worker-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
+ {{- end }}
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
subPath: common.properties
@@ -142,9 +144,11 @@ spec:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
emptyDir: {}
{{- end }}
+ {{- if .Values.worker.enableCustomizedConfig }}
- name: worker-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-worker
+ {{- end }}
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml
b/deploy/kubernetes/dolphinscheduler/values.yaml
index e7e42c6d09..5d924d6bba 100644
--- a/deploy/kubernetes/dolphinscheduler/values.yaml
+++ b/deploy/kubernetes/dolphinscheduler/values.yaml
@@ -449,7 +449,19 @@ master:
# requests:
# memory: "2Gi"
# cpu: "500m"
-
+ # -- enable configure custom config
+ enableCustomizedConfig: false
+ # -- configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-master/src/main/resources/application.yaml
+ customizedConfig: { }
+ # customizedConfig:
+ # application.yaml: |
+ # profiles:
+ # active: postgresql
+ # banner:
+ # charset: UTF-8
+ # jackson:
+ # time-zone: UTC
+ # date-format: "yyyy-MM-dd HH:mm:ss"
# -- Periodic probe of container liveness. Container will be restarted if
the probe fails.
# More info:
[container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
livenessProbe:
@@ -578,6 +590,17 @@ worker:
# -- Periodic probe of container liveness. Container will be restarted if
the probe fails.
# More info:
[container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
+ # -- enable configure custom config
+ enableCustomizedConfig: false
+ # -- configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-worker/src/main/resources/application.yaml
+ customizedConfig: { }
+# customizedConfig:
+# application.yaml: |
+# banner:
+# charset: UTF-8
+# jackson:
+# time-zone: UTC
+# date-format: "yyyy-MM-dd HH:mm:ss"
livenessProbe:
# -- Turn on and off liveness probe
enabled: true
@@ -740,7 +763,19 @@ alert:
# requests:
# memory: "1Gi"
# cpu: "500m"
-
+ # -- enable configure custom config
+ enableCustomizedConfig: false
+ # -- configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
+ customizedConfig: { }
+ # customizedConfig:
+ # application.yaml: |
+ # profiles:
+ # active: postgresql
+ # banner:
+ # charset: UTF-8
+ # jackson:
+ # time-zone: UTC
+ # date-format: "yyyy-MM-dd HH:mm:ss"
# -- Periodic probe of container liveness. Container will be restarted if
the probe fails.
# More info:
[container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
livenessProbe:
@@ -840,7 +875,19 @@ api:
# requests:
# memory: "1Gi"
# cpu: "500m"
-
+ # -- enable configure custom config
+ enableCustomizedConfig: false
+ # -- configure aligned with
https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-api/src/main/resources/application.yaml
+ customizedConfig: { }
+ # customizedConfig:
+ # application.yaml: |
+ # profiles:
+ # active: postgresql
+ # banner:
+ # charset: UTF-8
+ # jackson:
+ # time-zone: UTC
+ # date-format: "yyyy-MM-dd HH:mm:ss"
# -- Periodic probe of container liveness. Container will be restarted if
the probe fails.
# More info:
[container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
livenessProbe:
