This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/dolphinscheduler-operator.git
commit 83c2ab54426e11e9dfd025920776f50e2691f9e9 Author: nobolity <[email protected]> AuthorDate: Fri May 27 11:52:45 2022 +0800 fix(operator): delete the configmap --- api/v1alpha1/ds_public.go | 11 ++ api/v1alpha1/dsmaster_types.go | 1 + api/v1alpha1/dsworker_types.go | 3 + api/v1alpha1/zz_generated.deepcopy.go | 25 ++++ config/configmap/alert/application.yaml | 70 ---------- config/configmap/alert/ds-alert-configmap.yaml | 7 - config/configmap/api/application.yaml | 143 --------------------- config/configmap/api/ds-api-configmap.yaml | 7 - config/configmap/api/ds-api-deployment.yaml | 37 ------ config/configmap/master/application.yaml | 142 -------------------- config/configmap/worker/application.yaml | 91 ------------- .../ds.apache.dolphinscheduler.dev_dsmasters.yaml | 17 +++ .../ds.apache.dolphinscheduler.dev_dsworkers.yaml | 18 +++ .../alert/ds-alert-deployment.yaml | 22 ++-- .../{configmap => ds}/alert/ds-alert-service.yaml | 0 config/ds/api/ds-api-deployment.yaml | 32 +++++ config/{configmap => ds}/api/ds-api-service.yaml | 0 config/{configmap => ds}/ds-pv.yaml | 0 config/{configmap => ds}/ds-pvc.yaml | 0 .../postgreSQL/postgres-configmap.yaml | 0 .../postgreSQL/postgres-deployment.yaml | 0 .../{configmap => ds}/postgreSQL/postgres-pv.yaml | 0 .../{configmap => ds}/postgreSQL/postgres-pvc.yaml | 0 .../postgreSQL/postgres-service.yaml | 0 config/{configmap => ds}/zookeeper/zookeeper.yaml | 0 config/manager/kustomization.yaml | 10 +- config/samples/ds_v1alpha1_dsmaster.yaml | 8 +- config/samples/ds_v1alpha1_dsworker.yaml | 11 +- controllers/master_reconcile.go | 37 +++--- controllers/pod.go | 80 +----------- controllers/worker_reconcile.go | 20 ++- 31 files changed, 172 insertions(+), 620 deletions(-) diff --git a/api/v1alpha1/ds_public.go b/api/v1alpha1/ds_public.go index 78f31d0..c971727 100644 --- a/api/v1alpha1/ds_public.go +++ b/api/v1alpha1/ds_public.go @@ -30,6 +30,10 @@ const ( EnvZookeeper = "REGISTRY_ZOOKEEPER_CONNECT_STRING" DsServiceLabel = "service-name" DsServiceLabelValue = "ds-service" + DataSourceDriveName = "SPRING_DATASOURCE_DRIVER_CLASS_NAME" + DataSourceUrl = "SPRING_DATASOURCE_URL" + DataSourceUserName = "SPRING_DATASOURCE_USERNAME" + DataSourcePassWord = "SPRING_DATASOURCE_PASSWORD" ) // DsCondition represents one current condition of a ds cluster. @@ -118,3 +122,10 @@ func RandStr(length int) string { } return string(result) } + +type DateSourceTemplate struct { + DriveName string `json:"drive_name"` + Url string `json:"url"` + UserName string `json:"username"` + Password string `json:"password"` +} diff --git a/api/v1alpha1/dsmaster_types.go b/api/v1alpha1/dsmaster_types.go index 7909da7..581835c 100644 --- a/api/v1alpha1/dsmaster_types.go +++ b/api/v1alpha1/dsmaster_types.go @@ -25,6 +25,7 @@ import ( // DSMasterSpec defines the desired state of DSMaster type DSMasterSpec struct { + Datasource *DateSourceTemplate `json:"datasource"` // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file diff --git a/api/v1alpha1/dsworker_types.go b/api/v1alpha1/dsworker_types.go index d33ca6e..d64db8f 100644 --- a/api/v1alpha1/dsworker_types.go +++ b/api/v1alpha1/dsworker_types.go @@ -25,6 +25,8 @@ import ( // DSWorkerSpec defines the desired state of DSWorker type DSWorkerSpec struct { + //Datasource is the config of database + Datasource *DateSourceTemplate `json:"datasource"` // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file @@ -52,6 +54,7 @@ type DSWorkerSpec struct { //ZookeeperConnect is the address string of zookeeper ,and it will be written to ENV ZookeeperConnect string `json:"zookeeper_connect,omitempty"` + // Pod defines the policy to create pod for the dm-worker pod. // Updating Pod does not take effect on any existing dm-worker pods. Pod *PodPolicy `json:"pod,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ee5ab85..6c0f493 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -88,6 +88,11 @@ func (in *DSMasterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DSMasterSpec) DeepCopyInto(out *DSMasterSpec) { *out = *in + if in.Datasource != nil { + in, out := &in.Datasource, &out.Datasource + *out = new(DateSourceTemplate) + **out = **in + } if in.Pod != nil { in, out := &in.Pod, &out.Pod *out = new(PodPolicy) @@ -188,6 +193,11 @@ func (in *DSWorkerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DSWorkerSpec) DeepCopyInto(out *DSWorkerSpec) { *out = *in + if in.Datasource != nil { + in, out := &in.Datasource, &out.Datasource + *out = new(DateSourceTemplate) + **out = **in + } if in.Pod != nil { in, out := &in.Pod, &out.Pod *out = new(PodPolicy) @@ -226,6 +236,21 @@ func (in *DSWorkerStatus) DeepCopy() *DSWorkerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DateSourceTemplate) DeepCopyInto(out *DateSourceTemplate) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DateSourceTemplate. +func (in *DateSourceTemplate) DeepCopy() *DateSourceTemplate { + if in == nil { + return nil + } + out := new(DateSourceTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DsCondition) DeepCopyInto(out *DsCondition) { *out = *in diff --git a/config/configmap/alert/application.yaml b/config/configmap/alert/application.yaml deleted file mode 100644 index ea30a8d..0000000 --- a/config/configmap/alert/application.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -spring: - application: - name: alert-server - jackson: - time-zone: UTC - date-format: "yyyy-MM-dd HH:mm:ss" - banner: - charset: UTF-8 - datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://172.17.0.3:5432/dolphinscheduler - username: postgresadmin - password: admin12345 - hikari: - connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 - pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 - -server: - port: 50053 - -management: - endpoints: - web: - exposure: - include: '*' - metrics: - tags: - application: ${spring.application.name} - -alert: - port: 50052 - -metrics: - enabled: true - -# Override by profile - ---- -spring: - config: - activate: - on-profile: mysql - datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 diff --git a/config/configmap/alert/ds-alert-configmap.yaml b/config/configmap/alert/ds-alert-configmap.yaml deleted file mode 100644 index 12a0065..0000000 --- a/config/configmap/alert/ds-alert-configmap.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ds-alert-configmap - labels: - app: ds-alert - namespace: ds diff --git a/config/configmap/api/application.yaml b/config/configmap/api/application.yaml deleted file mode 100644 index a11d1b7..0000000 --- a/config/configmap/api/application.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -server: - port: 12345 - servlet: - session: - timeout: 120m - context-path: /dolphinscheduler/ - compression: - enabled: true - mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml - jetty: - max-http-form-post-size: 5000000 - -spring: - application: - name: api-server - banner: - charset: UTF-8 - jackson: - time-zone: UTC - date-format: "yyyy-MM-dd HH:mm:ss" - servlet: - multipart: - max-file-size: 1024MB - max-request-size: 1024MB - messages: - basename: i18n/messages - datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://172.17.0.3:5432/dolphinscheduler - username: postgresadmin - password: admin12345 - hikari: - connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 - pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 - quartz: - auto-startup: false - job-store-type: jdbc - jdbc: - initialize-schema: never - properties: - org.quartz.threadPool:threadPriority: 5 - org.quartz.jobStore.isClustered: true - org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX - org.quartz.scheduler.instanceId: AUTO - org.quartz.jobStore.tablePrefix: QRTZ_ - org.quartz.jobStore.acquireTriggersWithinLock: true - org.quartz.scheduler.instanceName: DolphinScheduler - org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool - org.quartz.jobStore.useProperties: false - org.quartz.threadPool.makeThreadsDaemons: true - org.quartz.threadPool.threadCount: 25 - org.quartz.jobStore.misfireThreshold: 60000 - org.quartz.scheduler.makeSchedulerThreadDaemon: true - org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate - org.quartz.jobStore.clusterCheckinInterval: 5000 - -management: - endpoints: - web: - exposure: - include: '*' - metrics: - tags: - application: ${spring.application.name} - -registry: - type: zookeeper - zookeeper: - namespace: dolphinscheduler - connect-string: localhost:2181 - retry-policy: - base-sleep-time: 60ms - max-sleep: 300ms - max-retries: 5 - session-timeout: 30s - connection-timeout: 9s - block-until-connected: 600ms - digest: ~ - -audit: - enabled: false - -metrics: - enabled: true - -python-gateway: - # Weather enable python gateway server or not. The default value is true. - enabled: true - # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different - # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` - gateway-server-address: 0.0.0.0 - # The port of Python gateway server start. Define which port you could connect to Python gateway server from - # Python API side. - gateway-server-port: 25333 - # The address of Python callback client. - python-address: 127.0.0.1 - # The port of Python callback client. - python-port: 25334 - # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), - # and socket server would never close even though no requests accept - connect-timeout: 0 - # Close each active connection of socket server if python program not active after x milliseconds. Define value is - # (0 = infinite), and socket server would never close even though no requests accept - read-timeout: 0 - -# Override by profile - ---- -spring: - config: - activate: - on-profile: mysql - datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 - quartz: - properties: - org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/config/configmap/api/ds-api-configmap.yaml b/config/configmap/api/ds-api-configmap.yaml deleted file mode 100644 index b786e1c..0000000 --- a/config/configmap/api/ds-api-configmap.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ds-api-configmap - labels: - app: ds-api - namespace: ds diff --git a/config/configmap/api/ds-api-deployment.yaml b/config/configmap/api/ds-api-deployment.yaml deleted file mode 100644 index 2451d10..0000000 --- a/config/configmap/api/ds-api-deployment.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ds-api-deployment - namespace: ds -spec: - replicas: 1 - selector: - matchLabels: - app: ds-api - template: - metadata: - labels: - app: ds-api - spec: - containers: - - name: ds-api - image: apache/dolphinscheduler-api:3.0.0-alpha - imagePullPolicy: "IfNotPresent" - env: - - name: REGISTRY_ZOOKEEPER_CONNECT_STRING - value: 172.17.0.4:2181 - ports: - - containerPort: 12345 - volumeMounts: - - mountPath: /opt/dolphinscheduler/conf/application.yaml - name: config-map - subPath: application.yaml - # command: - # - "/bin/sh" - # args: - # - "-c" - # - "while true; do echo hello; sleep 10;done" - volumes: - - name: config-map - configMap: - name: ds-api-config \ No newline at end of file diff --git a/config/configmap/master/application.yaml b/config/configmap/master/application.yaml deleted file mode 100644 index d92656e..0000000 --- a/config/configmap/master/application.yaml +++ /dev/null @@ -1,142 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -spring: - banner: - charset: UTF-8 - application: - name: master-server - jackson: - time-zone: UTC - date-format: "yyyy-MM-dd HH:mm:ss" - cache: - # default enable cache, you can disable by `type: none` - type: none - cache-names: - - tenant - - user - - processDefinition - - processTaskRelation - - taskDefinition - caffeine: - spec: maximumSize=100,expireAfterWrite=300s,recordStats - datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://172.17.0.3:5432/dolphinscheduler - username: postgresadmin - password: admin12345 - hikari: - connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 - pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 - quartz: - job-store-type: jdbc - jdbc: - initialize-schema: never - properties: - org.quartz.threadPool:threadPriority: 5 - org.quartz.jobStore.isClustered: true - org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX - org.quartz.scheduler.instanceId: AUTO - org.quartz.jobStore.tablePrefix: QRTZ_ - org.quartz.jobStore.acquireTriggersWithinLock: true - org.quartz.scheduler.instanceName: DolphinScheduler - org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool - org.quartz.jobStore.useProperties: false - org.quartz.threadPool.makeThreadsDaemons: true - org.quartz.threadPool.threadCount: 25 - org.quartz.jobStore.misfireThreshold: 60000 - org.quartz.scheduler.makeSchedulerThreadDaemon: true - org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate - org.quartz.jobStore.clusterCheckinInterval: 5000 - -registry: - type: zookeeper - zookeeper: - namespace: dolphinscheduler - connect-string: 172.17.0.11:2181 - retry-policy: - base-sleep-time: 60ms - max-sleep: 300ms - max-retries: 5 - session-timeout: 30s - connection-timeout: 9s - block-until-connected: 600ms - digest: ~ - -master: - listen-port: 5678 - # master fetch command num - fetch-command-num: 10 - # master prepare execute thread number to limit handle commands in parallel - pre-exec-threads: 10 - # master execute thread number to limit process instances in parallel - exec-threads: 100 - # master dispatch task number per batch - dispatch-task-number: 3 - # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight - host-selector: lower_weight - # master heartbeat interval, the unit is second - heartbeat-interval: 10 - # master commit task retry times - task-commit-retry-times: 5 - # master commit task interval, the unit is millisecond - task-commit-interval: 1000 - state-wheel-interval: 5 - # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2 - max-cpu-load-avg: -1 - # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G - reserved-memory: 0.3 - # failover interval, the unit is minute - failover-interval: 10 - # kill yarn jon when failover taskInstance, default true - kill-yarn-job-when-task-failover: true - -server: - port: 5679 - -management: - endpoints: - web: - exposure: - include: '*' - metrics: - tags: - application: ${spring.application.name} - -metrics: - enabled: true - -# Override by profile - ---- -spring: - config: - activate: - on-profile: mysql - datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 - quartz: - properties: - org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/config/configmap/worker/application.yaml b/config/configmap/worker/application.yaml deleted file mode 100644 index 48c89ae..0000000 --- a/config/configmap/worker/application.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -spring: - banner: - charset: UTF-8 - application: - name: worker-server - jackson: - time-zone: UTC - date-format: "yyyy-MM-dd HH:mm:ss" - datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://172.17.0.3:5432/dolphinscheduler - username: postgresadmin - password: admin12345 - hikari: - connection-test-query: select 1 - minimum-idle: 5 - auto-commit: true - validation-timeout: 3000 - pool-name: DolphinScheduler - maximum-pool-size: 50 - connection-timeout: 30000 - idle-timeout: 600000 - leak-detection-threshold: 0 - initialization-fail-timeout: 1 - -registry: - type: zookeeper - zookeeper: - namespace: dolphinscheduler - connect-string: localhost:2181 - retry-policy: - base-sleep-time: 60ms - max-sleep: 300ms - max-retries: 5 - session-timeout: 30s - connection-timeout: 9s - block-until-connected: 600ms - digest: ~ - -worker: - # worker listener port - listen-port: 1234 - # worker execute thread number to limit task instances in parallel - exec-threads: 100 - # worker heartbeat interval, the unit is second - heartbeat-interval: 10 - # worker host weight to dispatch tasks, default value 100 - host-weight: 100 - # worker tenant auto create - tenant-auto-create: true - # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2 - max-cpu-load-avg: -1 - # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G - reserved-memory: 0.3 - # default worker groups separated by comma, like 'worker.groups=default,test' - groups: - - default - # alert server listen host - alert-listen-host: localhost - alert-listen-port: 50052 - -server: - port: 1235 - -management: - endpoints: - web: - exposure: - include: '*' - metrics: - tags: - application: ${spring.application.name} - -metrics: - enabled: true diff --git a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml index 8250f94..0aff315 100644 --- a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml +++ b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml @@ -35,6 +35,22 @@ spec: spec: description: DSMasterSpec defines the desired state of DSMaster properties: + datasource: + properties: + drive_name: + type: string + password: + type: string + url: + type: string + username: + type: string + required: + - drive_name + - password + - url + - username + type: object log_pvc_name: description: LogPvcName defines the log capacity of application ,the position is /opt/dolphinscheduler/logs eg 20Gi @@ -1490,6 +1506,7 @@ spec: ,and it will be written to ENV type: string required: + - datasource - replicas type: object status: diff --git a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml index 9ab3831..e10149e 100644 --- a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml +++ b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml @@ -35,6 +35,23 @@ spec: spec: description: DSWorkerSpec defines the desired state of DSWorker properties: + datasource: + description: Datasource is the config of database + properties: + drive_name: + type: string + password: + type: string + url: + type: string + username: + type: string + required: + - drive_name + - password + - url + - username + type: object lib_pvc_name: description: LibPvcName define the address of lib pvc,the position is /opt/soft @@ -1494,6 +1511,7 @@ spec: ,and it will be written to ENV type: string required: + - datasource - replicas type: object status: diff --git a/config/configmap/alert/ds-alert-deployment.yaml b/config/ds/alert/ds-alert-deployment.yaml similarity index 50% rename from config/configmap/alert/ds-alert-deployment.yaml rename to config/ds/alert/ds-alert-deployment.yaml index c752995..627e1d6 100644 --- a/config/configmap/alert/ds-alert-deployment.yaml +++ b/config/ds/alert/ds-alert-deployment.yaml @@ -19,16 +19,12 @@ spec: imagePullPolicy: "IfNotPresent" ports: - containerPort: 50052 - volumeMounts: - - mountPath: /opt/dolphinscheduler/conf/application.yaml - name: config-map - subPath: application.yaml - # command: - # - "/bin/sh" - # args: - # - "-c" - # - "while true; do echo hello; sleep 10;done" - volumes: - - name: config-map - configMap: - name: ds-alert-config \ No newline at end of file + env: + - name: SPRING_DATASOURCE_DRIVER_CLASS_NAME + value: org.postgresql.Driver + - name: SPRING_DATASOURCE_URL + value: "jdbc:postgresql://172.17.0.4:5432/dolphinscheduler" + - name: SPRING_DATASOURCE_USERNAME + value: "postgresadmin" + - name: SPRING_DATASOURCE_PASSWORD + value: "admin12345" \ No newline at end of file diff --git a/config/configmap/alert/ds-alert-service.yaml b/config/ds/alert/ds-alert-service.yaml similarity index 100% rename from config/configmap/alert/ds-alert-service.yaml rename to config/ds/alert/ds-alert-service.yaml diff --git a/config/ds/api/ds-api-deployment.yaml b/config/ds/api/ds-api-deployment.yaml new file mode 100644 index 0000000..9264c94 --- /dev/null +++ b/config/ds/api/ds-api-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ds-api-deployment + namespace: ds +spec: + replicas: 1 + selector: + matchLabels: + app: ds-api + template: + metadata: + labels: + app: ds-api + spec: + containers: + - name: ds-api + image: apache/dolphinscheduler-api:3.0.0-alpha + imagePullPolicy: "IfNotPresent" + env: + - name: REGISTRY_ZOOKEEPER_CONNECT_STRING + value: 172.17.0.5:2181 + - name: SPRING_DATASOURCE_DRIVER_CLASS_NAME + value: org.postgresql.Driver + - name: SPRING_DATASOURCE_URL + value: "jdbc:postgresql://172.17.0.4:5432/dolphinscheduler" + - name: SPRING_DATASOURCE_USERNAME + value: "postgresadmin" + - name: SPRING_DATASOURCE_PASSWORD + value: "admin12345" + ports: + - containerPort: 12345 \ No newline at end of file diff --git a/config/configmap/api/ds-api-service.yaml b/config/ds/api/ds-api-service.yaml similarity index 100% rename from config/configmap/api/ds-api-service.yaml rename to config/ds/api/ds-api-service.yaml diff --git a/config/configmap/ds-pv.yaml b/config/ds/ds-pv.yaml similarity index 100% rename from config/configmap/ds-pv.yaml rename to config/ds/ds-pv.yaml diff --git a/config/configmap/ds-pvc.yaml b/config/ds/ds-pvc.yaml similarity index 100% rename from config/configmap/ds-pvc.yaml rename to config/ds/ds-pvc.yaml diff --git a/config/configmap/postgreSQL/postgres-configmap.yaml b/config/ds/postgreSQL/postgres-configmap.yaml similarity index 100% rename from config/configmap/postgreSQL/postgres-configmap.yaml rename to config/ds/postgreSQL/postgres-configmap.yaml diff --git a/config/configmap/postgreSQL/postgres-deployment.yaml b/config/ds/postgreSQL/postgres-deployment.yaml similarity index 100% rename from config/configmap/postgreSQL/postgres-deployment.yaml rename to config/ds/postgreSQL/postgres-deployment.yaml diff --git a/config/configmap/postgreSQL/postgres-pv.yaml b/config/ds/postgreSQL/postgres-pv.yaml similarity index 100% rename from config/configmap/postgreSQL/postgres-pv.yaml rename to config/ds/postgreSQL/postgres-pv.yaml diff --git a/config/configmap/postgreSQL/postgres-pvc.yaml b/config/ds/postgreSQL/postgres-pvc.yaml similarity index 100% rename from config/configmap/postgreSQL/postgres-pvc.yaml rename to config/ds/postgreSQL/postgres-pvc.yaml diff --git a/config/configmap/postgreSQL/postgres-service.yaml b/config/ds/postgreSQL/postgres-service.yaml similarity index 100% rename from config/configmap/postgreSQL/postgres-service.yaml rename to config/ds/postgreSQL/postgres-service.yaml diff --git a/config/configmap/zookeeper/zookeeper.yaml b/config/ds/zookeeper/zookeeper.yaml similarity index 100% rename from config/configmap/zookeeper/zookeeper.yaml rename to config/ds/zookeeper/zookeeper.yaml diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 2bcd3ee..5e793dd 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,6 +5,12 @@ generatorOptions: disableNameSuffixHash: true configMapGenerator: -- name: manager-config - files: +- files: - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: controller + newTag: latest diff --git a/config/samples/ds_v1alpha1_dsmaster.yaml b/config/samples/ds_v1alpha1_dsmaster.yaml index 034b762..8f2a2a3 100644 --- a/config/samples/ds_v1alpha1_dsmaster.yaml +++ b/config/samples/ds_v1alpha1_dsmaster.yaml @@ -7,7 +7,13 @@ metadata: app: ds-master spec: replicas: 1 - zookeeper_connect: "172.17.0.4:2181" + zookeeper_connect: "172.17.0.5:2181" version: 3.0.0-alpha repository: apache/dolphinscheduler-master + datasource: + drive_name: "org.postgresql.Driver" + url: "jdbc:postgresql://172.17.0.4:5432/dolphinscheduler" + username: "postgresadmin" + password: "admin12345" + diff --git a/config/samples/ds_v1alpha1_dsworker.yaml b/config/samples/ds_v1alpha1_dsworker.yaml index f3421b9..c1f1097 100644 --- a/config/samples/ds_v1alpha1_dsworker.yaml +++ b/config/samples/ds_v1alpha1_dsworker.yaml @@ -6,7 +6,12 @@ metadata: labels: app: ds-worker spec: - replicas: 3 - zookeeper_connect: "172.17.0.4:2181" + replicas: 1 + zookeeper_connect: "172.17.0.5:2181" version: 3.0.0-alpha - repository: apache/dolphinscheduler-worker \ No newline at end of file + repository: apache/dolphinscheduler-worker + datasource: + drive_name: "org.postgresql.Driver" + url: "jdbc:postgresql://172.17.0.4:5432/dolphinscheduler" + username: "postgresadmin" + password: "admin12345" \ No newline at end of file diff --git a/controllers/master_reconcile.go b/controllers/master_reconcile.go index 09dfe38..b42c58f 100644 --- a/controllers/master_reconcile.go +++ b/controllers/master_reconcile.go @@ -108,16 +108,6 @@ func newDSMasterPod(cr *dsv1alpha1.DSMaster) *corev1.Pod { dsv1alpha1.DsServiceLabel: dsv1alpha1.DsServiceLabelValue}, }, Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: dsMasterConfig, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: dsMasterConfig}, - }, - }, - }, - }, Hostname: podName, Subdomain: dsv1alpha1.DsServiceLabelValue, SetHostnameAsFQDN: &isSetHostnameAsFQDN, @@ -126,16 +116,25 @@ func newDSMasterPod(cr *dsv1alpha1.DSMaster) *corev1.Pod { Name: cr.Name, Image: ImageName(cr.Spec.Repository, cr.Spec.Version), ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{{ - Name: dsv1alpha1.EnvZookeeper, - Value: cr.Spec.ZookeeperConnect, - }}, - VolumeMounts: []corev1.VolumeMount{ + Env: []corev1.EnvVar{ + { + Name: dsv1alpha1.EnvZookeeper, + Value: cr.Spec.ZookeeperConnect, + }, { + Name: dsv1alpha1.DataSourceDriveName, + Value: cr.Spec.Datasource.DriveName, + }, + { + Name: dsv1alpha1.DataSourceUrl, + Value: cr.Spec.Datasource.Url, + }, + { + Name: dsv1alpha1.DataSourceUserName, + Value: cr.Spec.Datasource.UserName, + }, { - Name: dsMasterConfig, - MountPath: "/opt/dolphinscheduler/conf/application.yaml", - ReadOnly: false, - SubPath: "application.yaml", + Name: dsv1alpha1.DataSourcePassWord, + Value: cr.Spec.Datasource.Password, }, }, }, diff --git a/controllers/pod.go b/controllers/pod.go index f8c9e45..35caf01 100644 --- a/controllers/pod.go +++ b/controllers/pod.go @@ -20,35 +20,17 @@ import ( dsv1alpha1 "dolphinscheduler-operator/api/v1alpha1" "fmt" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) const ( dsLogVolumeName = "ds-log" dsLogVolumeMountDir = "/opt/dolphinscheduler/logs" - dsLogPVName = "pv" dsShareVolumeName = "ds-soft" dsShareVolumeMountDir = "/opt/soft" ) -func GetDsVersion(pod *corev1.Pod) string { - return pod.Labels[dsv1alpha1.DsVersionLabel] -} - -func SetDSVersion(pod *corev1.Pod, version string) { - pod.Labels[dsv1alpha1.DsVersionLabel] = version -} - -func GetPodNames(pods []*corev1.Pod) []string { - if len(pods) == 0 { - return nil - } - res := make([]string, 0) - for _, p := range pods { - res = append(res, p.Name) - } - return res +type PodTemplate struct { } func applyPodPolicy(pod *corev1.Pod, policy *dsv1alpha1.PodPolicy) { @@ -88,11 +70,6 @@ func containerWithRequirements(c corev1.Container, r corev1.ResourceRequirements return c } -// PVCNameFromMember the way we get PVC name from the member name -func PVCNameFromMember(memberName string) string { - return memberName -} - func ImageName(repo, version string) string { return fmt.Sprintf("%s:%v", repo, version) } @@ -106,27 +83,10 @@ func LabelsForCluster(lbs string) map[string]string { return labels.Set{dsv1alpha1.DsAppName: lbs} } -func LabelsForPV() map[string]string { - return labels.Set{dsv1alpha1.DsAppName: dsLogPVName} -} - func LabelsForService() map[string]string { return labels.Set{dsv1alpha1.DsServiceLabel: dsv1alpha1.DsServiceLabelValue} } -// AddDSVolumeToPod abstract the process of appending volume spec to pod spec -func AddDSVolumeToPod(pod *corev1.Pod, pvc *corev1.PersistentVolumeClaim) { - vol := corev1.Volume{Name: dsLogVolumeName} - if pvc != nil { - vol.VolumeSource = corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc.Name}, - } - } else { - vol.VolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} - } - pod.Spec.Volumes = append(pod.Spec.Volumes, vol) -} - // AddLogVolumeToPod abstract the process of appending volume spec to pod spec func AddLogVolumeToPod(pod *corev1.Pod, pvcName string) { vol := corev1.Volume{Name: dsLogVolumeName} @@ -172,44 +132,6 @@ func AddLibVolumeToPod(pod *corev1.Pod, pvcName string) { pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, vom) } -// NewLogPVC create PVC from dsMaster pod's PVC spec -//func NewLogPVC(cluster *dsv1alpha1.DSMaster, pod *corev1.Pod, storageClassName string) *corev1.PersistentVolumeClaim { -// -// pvc := &corev1.PersistentVolumeClaim{ -// ObjectMeta: metav1.ObjectMeta{ -// GenerateName: cluster.Name + "-pvc", -// Namespace: cluster.Namespace, -// Labels: LabelsForCluster(dsLogVolumeName), -// }, -// Spec: corev1.PersistentVolumeClaimSpec{ -// AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, -// Resources: corev1.ResourceRequirements{ -// Requests: corev1.ResourceList{ -// corev1.ResourceStorage: resource.MustParse(cluster.Spec.LogCapacity), -// }, -// }, -// StorageClassName: &storageClassName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: LabelsForPV(), -// }, -// }, -// } -// return pvc -//} - -// NewDSWorkerPodPVC create PVC object from dsMaster pod's PVC spec -func NewDSWorkerPodPVC(cluster *dsv1alpha1.DSWorker, pod *corev1.Pod, lbs string) *corev1.PersistentVolumeClaim { - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: PVCNameFromMember(pod.Name), - Namespace: cluster.Namespace, - Labels: LabelsForCluster(lbs), - }, - Spec: *cluster.Spec.Pod.PersistentVolumeClaimSpec, - } - return pvc -} - // mergeLabels merges l2 into l1. Conflicting label will be skipped. func mergeLabels(l1, l2 map[string]string) { for k, v := range l2 { diff --git a/controllers/worker_reconcile.go b/controllers/worker_reconcile.go index 8a3b2ad..b13c422 100644 --- a/controllers/worker_reconcile.go +++ b/controllers/worker_reconcile.go @@ -127,13 +127,21 @@ func newDSWorkerPod(cr *dsv1alpha1.DSWorker) *corev1.Pod { Env: []corev1.EnvVar{{ Name: dsv1alpha1.EnvZookeeper, Value: cr.Spec.ZookeeperConnect, - }}, - VolumeMounts: []corev1.VolumeMount{ + }, { + Name: dsv1alpha1.DataSourceDriveName, + Value: cr.Spec.Datasource.DriveName, + }, + { + Name: dsv1alpha1.DataSourceUrl, + Value: cr.Spec.Datasource.Url, + }, + { + Name: dsv1alpha1.DataSourceUserName, + Value: cr.Spec.Datasource.UserName, + }, { - Name: dsWorkerConfig, - MountPath: "/opt/dolphinscheduler/conf/application.yaml", - ReadOnly: false, - SubPath: "application.yaml", + Name: dsv1alpha1.DataSourcePassWord, + Value: cr.Spec.Datasource.Password, }, }, },
