This is an automated email from the ASF dual-hosted git repository. vvraskin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new 3701d2a Refactor database-related variables to avoid duplication in every environment. (#3867) 3701d2a is described below commit 3701d2a3ebc51c08d46de578c195a8056f14c067 Author: Christian Bickel <git...@cbickel.de> AuthorDate: Thu Jul 12 17:24:01 2018 +0200 Refactor database-related variables to avoid duplication in every environment. (#3867) * Factor out couch admin credentials to global group_vars/all file. * Refactor db_provider. * Refactor db_protocol. * Refactor db_protocol. * Refactor db_host. --- ansible/couchdb.yml | 4 +-- ansible/environments/docker-machine/group_vars/all | 8 ----- ansible/environments/local/group_vars/all | 8 ----- ansible/environments/local/hosts.j2.ini | 2 +- ansible/environments/vagrant/group_vars/all | 8 ----- ansible/environments/vagrant/hosts | 2 +- ansible/group_vars/all | 8 +++++ ansible/roles/controller/tasks/deploy.yml | 12 +++---- ansible/roles/couchdb/tasks/deploy.yml | 40 +++++++++++----------- ansible/roles/invoker/tasks/deploy.yml | 12 +++---- ansible/tasks/db/checkDb.yml | 8 ++--- ansible/tasks/db/recreateDb.yml | 24 ++++++------- ansible/tasks/db/recreateDoc.yml | 12 +++---- ansible/templates/whisk.conf.j2 | 12 +++---- ansible/templates/whisk.properties.j2 | 12 +++---- tests/src/test/resources/application.conf.j2 | 12 +++---- 16 files changed, 84 insertions(+), 100 deletions(-) diff --git a/ansible/couchdb.yml b/ansible/couchdb.yml index 00d15dd..bcdea9b 100644 --- a/ansible/couchdb.yml +++ b/ansible/couchdb.yml @@ -8,12 +8,12 @@ - name: check if db_local.ini exists? tags: ini stat: path="{{ playbook_dir }}/db_local.ini" - register: db + register: db_check - name: prepare db_local.ini tags: ini local_action: template src="db_local.ini.j2" dest="{{ playbook_dir }}/db_local.ini" - when: not db.stat.exists + when: not db_check.stat.exists - hosts: db roles: diff --git a/ansible/environments/docker-machine/group_vars/all b/ansible/environments/docker-machine/group_vars/all index 91471c8..a17696a 100644 --- a/ansible/environments/docker-machine/group_vars/all +++ b/ansible/environments/docker-machine/group_vars/all @@ -19,14 +19,6 @@ whisk_api_localhost_name: "openwhisk" # Hardcoded for docker-machine since db init runs on host not inside VM db_prefix: whisk_dockermachine_ -# Auto lookup to find the db credentials -db_provider: "{{ lookup('ini', 'db_provider section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_username: "{{ lookup('ini', 'db_username section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_password: "{{ lookup('ini', 'db_password section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_protocol: "{{ lookup('ini', 'db_protocol section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_host: "{{ lookup('ini', 'db_host section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_port: "{{ lookup('ini', 'db_port section=db_creds file={{ playbook_dir }}/db_local.ini') }}" - # API GW connection configuration apigw_auth_user: "" apigw_auth_pwd: "" diff --git a/ansible/environments/local/group_vars/all b/ansible/environments/local/group_vars/all index 338e10a..30464ef 100755 --- a/ansible/environments/local/group_vars/all +++ b/ansible/environments/local/group_vars/all @@ -11,14 +11,6 @@ invoker_use_runc: "{{ ansible_distribution != 'MacOSX' }}" db_prefix: whisk_local_ -# Auto lookup to find the db credentials -db_provider: "{{ lookup('ini', 'db_provider section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_username: "{{ lookup('ini', 'db_username section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_password: "{{ lookup('ini', 'db_password section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_protocol: "{{ lookup('ini', 'db_protocol section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_host: "{{ lookup('ini', 'db_host section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_port: "{{ lookup('ini', 'db_port section=db_creds file={{ playbook_dir }}/db_local.ini') }}" - # API GW connection configuration apigw_auth_user: "" apigw_auth_pwd: "" diff --git a/ansible/environments/local/hosts.j2.ini b/ansible/environments/local/hosts.j2.ini index a687adf..210df4c 100644 --- a/ansible/environments/local/hosts.j2.ini +++ b/ansible/environments/local/hosts.j2.ini @@ -27,7 +27,7 @@ invoker0 ansible_host=172.17.0.1 ansible_connection=local invoker1 ansible_host=172.17.0.1 ansible_connection=local {% endif %} -; db group is only used if db_provider is CouchDB +; db group is only used if db.provider is CouchDB [db] 172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local diff --git a/ansible/environments/vagrant/group_vars/all b/ansible/environments/vagrant/group_vars/all index 614e0e2..b114445 100755 --- a/ansible/environments/vagrant/group_vars/all +++ b/ansible/environments/vagrant/group_vars/all @@ -11,14 +11,6 @@ invoker_use_runc: "{{ ansible_distribution != 'MacOSX' }}" db_prefix: whisk_local_ -# Auto lookup to find the db credentials -db_provider: "{{ lookup('ini', 'db_provider section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_username: "{{ lookup('ini', 'db_username section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_password: "{{ lookup('ini', 'db_password section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_protocol: "{{ lookup('ini', 'db_protocol section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_host: "{{ lookup('ini', 'db_host section=db_creds file={{ playbook_dir }}/db_local.ini') }}" -db_port: "{{ lookup('ini', 'db_port section=db_creds file={{ playbook_dir }}/db_local.ini') }}" - # API GW connection configuration apigw_auth_user: "" apigw_auth_pwd: "" diff --git a/ansible/environments/vagrant/hosts b/ansible/environments/vagrant/hosts index b630812..e32f943 100644 --- a/ansible/environments/vagrant/hosts +++ b/ansible/environments/vagrant/hosts @@ -18,7 +18,7 @@ kafkas [invokers] invoker0 ansible_host=172.17.0.1 ansible_connection=local -; db group is only used if db_provider is CouchDB +; db group is only used if db.provider is CouchDB [db] 172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local diff --git a/ansible/group_vars/all b/ansible/group_vars/all index b36d2e4..dc5d942 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -217,6 +217,10 @@ nginx: # The key db.whisk.auth is the name of the authentication database where all keys of all users are stored. # The db_prefix is defined for each environment on its own. The CouchDb credentials are also defined for each environment on its own. db: + provider: "{{ db_provider | default(lookup('ini', 'db_provider section=db_creds file={{ playbook_dir }}/db_local.ini')) }}" + protocol: "{{ db_protocol | default(lookup('ini', 'db_protocol section=db_creds file={{ playbook_dir }}/db_local.ini')) }}" + port: "{{ dp_port | default(lookup('ini', 'db_port section=db_creds file={{ playbook_dir }}/db_local.ini')) }}" + host: "{{ db_host | default(lookup('ini', 'db_host section=db_creds file={{ playbook_dir }}/db_local.ini')) }}" instances: "{{ groups['db'] | length }}" authkeys: - guest @@ -225,6 +229,10 @@ db: actions: "{{ db_prefix }}whisks" activations: "{{ db_prefix }}activations" auth: "{{ db_prefix }}subjects" + credentials: + admin: + user: "{{ db_username | default(lookup('ini', 'db_username section=db_creds file={{ playbook_dir }}/db_local.ini')) }}" + pass: "{{ db_password | default(lookup('ini', 'db_password section=db_creds file={{ playbook_dir }}/db_local.ini')) }}" apigateway: port: diff --git a/ansible/roles/controller/tasks/deploy.yml b/ansible/roles/controller/tasks/deploy.yml index 9d14751..e4b0885 100644 --- a/ansible/roles/controller/tasks/deploy.yml +++ b/ansible/roles/controller/tasks/deploy.yml @@ -159,12 +159,12 @@ "CONFIG_whisk_kafka_common_sslKeystorePassword": "{{ kafka.ssl.keystore.password }}" - "CONFIG_whisk_couchdb_protocol": "{{ db_protocol }}" - "CONFIG_whisk_couchdb_host": "{{ db_host }}" - "CONFIG_whisk_couchdb_port": "{{ db_port }}" - "CONFIG_whisk_couchdb_username": "{{ db_username }}" - "CONFIG_whisk_couchdb_password": "{{ db_password }}" - "CONFIG_whisk_couchdb_provider": "{{ db_provider }}" + "CONFIG_whisk_couchdb_protocol": "{{ db.protocol }}" + "CONFIG_whisk_couchdb_host": "{{ db.host }}" + "CONFIG_whisk_couchdb_port": "{{ db.port }}" + "CONFIG_whisk_couchdb_username": "{{ db.credentials.admin.user }}" + "CONFIG_whisk_couchdb_password": "{{ db.credentials.admin.pass }}" + "CONFIG_whisk_couchdb_provider": "{{ db.provider }}" "CONFIG_whisk_couchdb_databases_WhiskAuth": "{{ db.whisk.auth }}" "CONFIG_whisk_couchdb_databases_WhiskEntity": "{{ db.whisk.actions }}" "CONFIG_whisk_couchdb_databases_WhiskActivation": diff --git a/ansible/roles/couchdb/tasks/deploy.yml b/ansible/roles/couchdb/tasks/deploy.yml index a4365fb..2e3b75d 100644 --- a/ansible/roles/couchdb/tasks/deploy.yml +++ b/ansible/roles/couchdb/tasks/deploy.yml @@ -8,8 +8,8 @@ coordinator: "{{ groups['db'][0] }}" - name: check if db credentials are valid for CouchDB - fail: msg="The db provider in your {{ hosts_dir }}/group_vars/all is {{ db_provider }}, it has to be CouchDB, pls double check" - when: db_provider != "CouchDB" + fail: msg="The db provider in your {{ hosts_dir }}/group_vars/all is {{ db.provider }}, it has to be CouchDB, pls double check" + when: db.provider != "CouchDB" - name: check for persistent disk shell: df -h @@ -34,18 +34,18 @@ restart_policy: "{{ docker.restart.policy }}" volumes: "{{volume_dir | default([])}}" ports: - - "{{ db_port }}:5984" + - "{{ db.port }}:5984" - "4369:4369" - "9100:9100" env: - COUCHDB_USER: "{{ db_username }}" - COUCHDB_PASSWORD: "{{ db_password }}" + COUCHDB_USER: "{{ db.credentials.admin.user }}" + COUCHDB_PASSWORD: "{{ db.credentials.admin.pass }}" NODENAME: "{{ ansible_host }}" pull: "{{ couchdb.pull_couchdb | default(true) }}" - name: wait until CouchDB in this host is up and running uri: - url: "{{ db_protocol }}://{{ ansible_host }}:{{ db_port }}/_utils/" + url: "{{ db.protocol }}://{{ ansible_host }}:{{ db.port }}/_utils/" register: result until: result.status == 200 retries: 12 @@ -53,49 +53,49 @@ - name: create '_users' database for singleton mode uri: - url: "{{ db_protocol }}://{{ ansible_host }}:{{ db_port }}/_users" + url: "{{ db.protocol }}://{{ ansible_host }}:{{ db.port }}/_users" method: PUT status_code: 201 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes when: (couchdb.version|version_compare('2.0','>=')) and (db.instances|int == 1) - name: enable the cluster setup mode uri: - url: "{{ db_protocol }}://{{ ansible_host }}:{{ db_port }}/_cluster_setup" + url: "{{ db.protocol }}://{{ ansible_host }}:{{ db.port }}/_cluster_setup" method: POST body: > - {"action": "enable_cluster", "bind_address":"0.0.0.0", "username": "{{ db_username }}", "password":"{{ db_password }}", "port": {{ db_port }}, "node_count": "{{ groups['db'] | length }}", "remote_node": "{{ ansible_host }}", "remote_current_user": "{{ db_username }}", "remote_current_password": "{{ db_password }}"} + {"action": "enable_cluster", "bind_address":"0.0.0.0", "username": "{{ db.credentials.admin.user }}", "password":"{{ db.credentials.admin.pass }}", "port": {{ db.port }}, "node_count": "{{ groups['db'] | length }}", "remote_node": "{{ ansible_host }}", "remote_current_user": "{{ db.credentials.admin.user }}", "remote_current_password": "{{ db.credentials.admin.pass }}"} body_format: json status_code: 201 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes when: (inventory_hostname == coordinator) and (db.instances|int >= 2) - name: add remote nodes to the cluster uri: - url: "{{ db_protocol }}://{{ coordinator }}:{{ db_port }}/_cluster_setup" + url: "{{ db.protocol }}://{{ coordinator }}:{{ db.port }}/_cluster_setup" method: POST body: > - {"action": "add_node", "host":"{{ ansible_host }}", "port": {{ db_port }}, "username": "{{ db_username }}", "password":"{{ db_password }}"} + {"action": "add_node", "host":"{{ ansible_host }}", "port": {{ db.port }}, "username": "{{ db.credentials.admin.user }}", "password":"{{ db.credentials.admin.pass }}"} body_format: json status_code: 201 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes when: (inventory_hostname != coordinator) and (db.instances|int >= 2) - name: finish the cluster setup mode uri: - url: "{{ db_protocol }}://{{ ansible_host }}:{{ db_port }}/_cluster_setup" + url: "{{ db.protocol }}://{{ ansible_host }}:{{ db.port }}/_cluster_setup" method: POST body: > {"action": "finish_cluster"} body_format: json status_code: 201 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes when: (inventory_hostname == coordinator) and (db.instances|int >= 2) diff --git a/ansible/roles/invoker/tasks/deploy.yml b/ansible/roles/invoker/tasks/deploy.yml index 65d741c..edfde37 100644 --- a/ansible/roles/invoker/tasks/deploy.yml +++ b/ansible/roles/invoker/tasks/deploy.yml @@ -179,12 +179,12 @@ "CONFIG_whisk_kafka_common_sslKeystorePassword": "{{ kafka.ssl.keystore.password }}" "CONFIG_whisk_userEvents_enabled": "{{ user_events }}" "ZOOKEEPER_HOSTS": "{{ zookeeper_connect_string }}" - "CONFIG_whisk_couchdb_protocol": "{{ db_protocol }}" - "CONFIG_whisk_couchdb_host": "{{ db_host }}" - "CONFIG_whisk_couchdb_port": "{{ db_port }}" - "CONFIG_whisk_couchdb_username": "{{ db_username }}" - "CONFIG_whisk_couchdb_password": "{{ db_password }}" - "CONFIG_whisk_couchdb_provider": "{{ db_provider }}" + "CONFIG_whisk_couchdb_protocol": "{{ db.protocol }}" + "CONFIG_whisk_couchdb_host": "{{ db.host }}" + "CONFIG_whisk_couchdb_port": "{{ db.port }}" + "CONFIG_whisk_couchdb_username": "{{ db.credentials.admin.user }}" + "CONFIG_whisk_couchdb_password": "{{ db.credentials.admin.pass }}" + "CONFIG_whisk_couchdb_provider": "{{ db.provider }}" "CONFIG_whisk_couchdb_databases_WhiskAuth": "{{ db.whisk.auth }}" "CONFIG_whisk_couchdb_databases_WhiskEntity": "{{ db.whisk.actions }}" "CONFIG_whisk_couchdb_databases_WhiskActivation": "{{ db.whisk.activations }}" diff --git a/ansible/tasks/db/checkDb.yml b/ansible/tasks/db/checkDb.yml index a694ca2..728e60a 100644 --- a/ansible/tasks/db/checkDb.yml +++ b/ansible/tasks/db/checkDb.yml @@ -4,11 +4,11 @@ # Checks, that the Database exists # dbName - name of the database to check -- name: check if {{ dbName }} with {{ db_provider }} exists +- name: check if {{ dbName }} with {{ db.provider }} exists uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/{{ dbName }}" + url: "{{ db.protocol }}://{{ db.host }}:{{ db.port }}/{{ dbName }}" method: HEAD status_code: 200 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes diff --git a/ansible/tasks/db/recreateDb.yml b/ansible/tasks/db/recreateDb.yml index abc83ab..145d285 100644 --- a/ansible/tasks/db/recreateDb.yml +++ b/ansible/tasks/db/recreateDb.yml @@ -5,32 +5,32 @@ # dbName - name of the database to (re)create # forceRecreation - if true, the databases will be deleted (if it exists) and recreated. If false, it will not be recreated. -- name: check if {{ dbName }} with {{ db_provider }} exists +- name: check if {{ dbName }} with {{ db.provider }} exists uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/{{ dbName }}" + url: "{{ db.protocol }}://{{ db.host }}:{{ db.port }}/{{ dbName }}" method: HEAD status_code: 200,404 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes register: response -- name: delete the {{ dbName }} with {{ db_provider }} +- name: delete the {{ dbName }} with {{ db.provider }} uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/{{ dbName }}" + url: "{{ db.protocol }}://{{ db.host }}:{{ db.port }}/{{ dbName }}" method: DELETE status_code: 200,404 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes when: forceRecreation == True and response.status == 200 -- name: create {{ dbName }} with {{ db_provider }} +- name: create {{ dbName }} with {{ db.provider }} uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/{{ dbName }}" + url: "{{ db.protocol }}://{{ db.host }}:{{ db.port }}/{{ dbName }}" method: PUT status_code: 200,201 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes when: forceRecreation == True or response.status == 404 diff --git a/ansible/tasks/db/recreateDoc.yml b/ansible/tasks/db/recreateDoc.yml index 56eff7a..c3529b5 100644 --- a/ansible/tasks/db/recreateDoc.yml +++ b/ansible/tasks/db/recreateDoc.yml @@ -13,12 +13,12 @@ # fetches the revision of previous view (to update it) if it exists - name: check for {{ doc['_id'] }} document in {{ dbName }} database uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/{{ dbName }}/{{ doc['_id'] }}" + url: "{{ db.protocol }}://{{ db.host }}:{{ db.port }}/{{ dbName }}/{{ doc['_id'] }}" return_content: yes method: GET status_code: 200, 404 - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes register: response @@ -44,12 +44,12 @@ - name: recreate or update the document on the {{ dbName }} database uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/{{ dbName }}" + url: "{{ db.protocol }}://{{ db.host }}:{{ db.port }}/{{ dbName }}" method: POST status_code: 200, 201 body_format: json body: "{{ document }}" - user: "{{ db_username }}" - password: "{{ db_password }}" + user: "{{ db.credentials.admin.user }}" + password: "{{ db.credentials.admin.pass }}" force_basic_auth: yes when: create == True diff --git a/ansible/templates/whisk.conf.j2 b/ansible/templates/whisk.conf.j2 index 58f0ea0..9bb9d53 100644 --- a/ansible/templates/whisk.conf.j2 +++ b/ansible/templates/whisk.conf.j2 @@ -2,12 +2,12 @@ include classpath("application.conf") whisk { couchdb { - protocol = "{{ db_protocol }}" - host = "{{ db_host }}" - port = "{{ db_port }}" - username = "{{ db_username }}" - password = "{{ db_password }}" - provider = "{{ db_provider }}" + protocol = "{{ db.protocol }}" + host = "{{ db.host }}" + port = "{{ db.port }}" + username = "{{ db.credentials.admin.user }}" + password = "{{ db.credentials.admin.pass }}" + provider = "{{ db.provider }}" databases { WhiskAuth = "{{ db.whisk.auth }}" WhiskEntity = "{{ db.whisk.actions }}" diff --git a/ansible/templates/whisk.properties.j2 b/ansible/templates/whisk.properties.j2 index f367bf0..bfdba3b 100644 --- a/ansible/templates/whisk.properties.j2 +++ b/ansible/templates/whisk.properties.j2 @@ -76,12 +76,12 @@ docker.addHost.cmd= docker.dns.cmd={{ docker_dns }} docker.restart.opts={{ docker.restart.policy }} -db.provider={{ db_provider }} -db.protocol={{ db_protocol }} -db.host={{ db_host }} -db.port={{ db_port }} -db.username={{ db_username }} -db.password={{ db_password }} +db.provider={{ db.provider }} +db.protocol={{ db.protocol }} +db.host={{ db.host }} +db.port={{ db.port }} +db.username={{ db.credentials.admin.user }} +db.password={{ db.credentials.admin.pass }} db.prefix={{ db_prefix }} db.whisk.auths={{ db.whisk.auth }} db.whisk.actions={{ db.whisk.actions }} diff --git a/tests/src/test/resources/application.conf.j2 b/tests/src/test/resources/application.conf.j2 index 59e33f0..3b75461 100644 --- a/tests/src/test/resources/application.conf.j2 +++ b/tests/src/test/resources/application.conf.j2 @@ -33,12 +33,12 @@ whisk { } couchdb { - protocol = "{{ db_protocol }}" - host = "{{ db_host }}" - port = "{{ db_port }}" - username = "{{ db_username }}" - password = "{{ db_password }}" - provider = "{{ db_provider }}" + protocol = "{{ db.protocol }}" + host = "{{ db.host }}" + port = "{{ db.port }}" + username = "{{ db.credentials.admin.user }}" + password = "{{ db.credentials.admin.pass }}" + provider = "{{ db.provider }}" databases { WhiskAuth = "{{ db.whisk.auth }}" WhiskEntity = "{{ db.whisk.actions }}"