This is an automated email from the ASF dual-hosted git repository.
houshengbo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new aad27be Improve the efficiency of Jenkins pipeline (#4391)
aad27be is described below
commit aad27bed182f709fd7d25bc7a01f7e0fe985b027
Author: Vincent <[email protected]>
AuthorDate: Tue Mar 26 09:08:32 2019 -0400
Improve the efficiency of Jenkins pipeline (#4391)
This PR will utilize three VMs as the edge machines, so that we can run 3
openwhisk depoyments at the same time instead of one.
---
Jenkinsfile | 98 +++++++++++++---------
.../group_vars/{all => openwhisk-vm1-he-de} | 0
.../group_vars/{all => openwhisk-vm2-he-de} | 3 +-
.../group_vars/{all => openwhisk-vm3-he-de} | 2 +-
.../{hosts.j2.ini => openwhisk-vm1-he-de.j2.ini} | 0
.../jenkins/openwhisk-vm2-he-de.j2.ini | 36 ++++++++
.../jenkins/openwhisk-vm3-he-de.j2.ini | 36 ++++++++
7 files changed, 133 insertions(+), 42 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 1a7118d..262cd44 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -16,57 +16,75 @@
* limitations under the License.
*/
-timeout(time: 4, unit: 'HOURS') {
+timeout(time: 12, unit: 'HOURS') {
- def domainName = "openwhisk-vm1-he-de.apache.org"
def port = "444"
def cert = "domain.crt"
def key = "domain.key"
- node("openwhisk1") {
- properties([disableConcurrentBuilds()])
- deleteDir()
- stage ('Checkout') {
- checkout scm
- }
-
- stage ('Build') {
- // Set up a private docker registry service, accessed by all the
OpenWhisk VMs.
+ node("openwhisk") {
+ def hostName = sh(returnStdout: true, script: 'hostname').trim()
+ def domainName = hostName+".apache.org"
+ lock("${hostName}") {
try {
- sh "docker container stop registry && docker container rm -v
registry"
- } catch (exp) {
- println("Unable to stop and remove the container registry.")
- }
+ deleteDir()
+ stage('Checkout') {
+ checkout scm
+ }
+
+ stage('Build') {
+ // Set up a private docker registry service, accessed by
all the OpenWhisk VMs.
+ try {
+ sh "docker container stop registry && docker container
rm -v registry"
+ } catch (exp) {
+ println("Unable to stop and remove the container
registry.")
+ }
- sh "docker run -d --restart=always --name registry -v
\"$HOME\"/certs:/certs \
+ sh "docker run -d --restart=always --name registry -v
\"$HOME\"/certs:/certs \
-e REGISTRY_HTTP_ADDR=0.0.0.0:${port} -e
REGISTRY_HTTP_TLS_CERTIFICATE=/certs/${cert} \
-e REGISTRY_HTTP_TLS_KEY=/certs/${key} -p ${port}:${port}
registry:2"
- // Build the controller and invoker images.
- sh "./gradlew distDocker -PdockerRegistry=${domainName}:${port}"
- }
+ // Build the controller and invoker images.
+ sh "./gradlew distDocker
-PdockerRegistry=${domainName}:${port}"
+ }
- stage('Deploy') {
- dir("ansible") {
- // Copy the jenkins ansible configuration under the directory
ansible. This can make sure the SSH is used to
- // access the VMs of invokers by the VM of the controller.
- sh '[ -f "environments/jenkins/ansible_jenkins.cfg" ] && cp
environments/jenkins/ansible_jenkins.cfg ansible.cfg'
- sh 'ansible-playbook -i environments/jenkins setup.yml'
- sh 'ansible-playbook -i environments/jenkins openwhisk.yml -e
mode=clean'
- sh 'ansible-playbook -i environments/jenkins apigateway.yml -e
mode=clean'
- sh 'ansible-playbook -i environments/jenkins couchdb.yml -e
mode=clean'
- sh 'ansible-playbook -i environments/jenkins couchdb.yml'
- sh 'ansible-playbook -i environments/jenkins initdb.yml'
- sh 'ansible-playbook -i environments/jenkins wipe.yml'
- sh 'ansible-playbook -i environments/jenkins apigateway.yml'
- sh 'ansible-playbook -i environments/jenkins openwhisk.yml'
- sh 'ansible-playbook -i environments/jenkins properties.yml'
- sh 'ansible-playbook -i environments/jenkins routemgmt.yml'
- sh 'ansible-playbook -i environments/jenkins postdeploy.yml'
- }
- }
+ stage('Deploy') {
+ dir("ansible") {
+ // Copy the jenkins ansible configuration under the
directory ansible. This can make sure the SSH is used to
+ // access the VMs of invokers by the VM of the
controller.
+ sh '[ -f "environments/jenkins/ansible_jenkins.cfg" ]
&& cp environments/jenkins/ansible_jenkins.cfg ansible.cfg'
+ }
+
+ dir("ansible/environments/jenkins") {
+ sh "cp ${hostName}.j2.ini hosts.j2.ini"
+ }
- stage('Test') {
- sh './gradlew :tests:test'
+ dir("ansible/environments/jenkins/group_vars") {
+ sh "cp ${hostName} all"
+ }
+
+ dir("ansible") {
+ sh 'ansible-playbook -i environments/jenkins setup.yml'
+ sh 'ansible-playbook -i environments/jenkins
openwhisk.yml -e mode=clean'
+ sh 'ansible-playbook -i environments/jenkins
apigateway.yml -e mode=clean'
+ sh 'ansible-playbook -i environments/jenkins
couchdb.yml -e mode=clean'
+ sh 'ansible-playbook -i environments/jenkins
couchdb.yml'
+ sh 'ansible-playbook -i environments/jenkins
initdb.yml'
+ sh 'ansible-playbook -i environments/jenkins wipe.yml'
+ sh 'ansible-playbook -i environments/jenkins
apigateway.yml'
+ sh 'ansible-playbook -i environments/jenkins
openwhisk.yml'
+ sh 'ansible-playbook -i environments/jenkins
properties.yml'
+ sh 'ansible-playbook -i environments/jenkins
routemgmt.yml'
+ sh 'ansible-playbook -i environments/jenkins
postdeploy.yml'
+ }
+ }
+ stage('Test') {
+ sh './gradlew :tests:test'
+ }
+ } catch (exp) {
+ println("Exception:" + exp)
+ } finally {
+ step([$class: 'JUnitResultArchiver', testResults:
'**/test-results/**/TEST-*.xml'])
+ }
}
}
}
diff --git a/ansible/environments/jenkins/group_vars/all
b/ansible/environments/jenkins/group_vars/openwhisk-vm1-he-de
similarity index 100%
copy from ansible/environments/jenkins/group_vars/all
copy to ansible/environments/jenkins/group_vars/openwhisk-vm1-he-de
diff --git a/ansible/environments/jenkins/group_vars/all
b/ansible/environments/jenkins/group_vars/openwhisk-vm2-he-de
similarity index 93%
copy from ansible/environments/jenkins/group_vars/all
copy to ansible/environments/jenkins/group_vars/openwhisk-vm2-he-de
index 659fdc2..814a060 100755
--- a/ansible/environments/jenkins/group_vars/all
+++ b/ansible/environments/jenkins/group_vars/openwhisk-vm2-he-de
@@ -6,7 +6,7 @@ config_root_dir: "{{ openwhisk_tmp_dir }}/wskconf"
whisk_logs_dir: "{{ openwhisk_tmp_dir }}/wsklogs"
coverage_enabled: "{{ lookup('env', 'GRADLE_COVERAGE') | default('false',
true) | bool}}"
coverage_logs_dir: "{{ openwhisk_tmp_dir }}/wskcov"
-docker_registry: "openwhisk-vm1-he-de.apache.org:444/"
+docker_registry: "openwhisk-vm2-he-de.apache.org:444/"
docker_dns: ""
runtimes_bypass_pull_for_local_images: true
invoker_use_runc: "{{ ansible_distribution != 'MacOSX' }}"
@@ -34,3 +34,4 @@ env_hosts_dir: "{{ playbook_dir }}/environments/jenkins"
container_pool_akka_client: true
runtimes_enable_concurrency: true
limit_action_concurrency_max: 500
+limit_invocations_per_minute: 120
diff --git a/ansible/environments/jenkins/group_vars/all
b/ansible/environments/jenkins/group_vars/openwhisk-vm3-he-de
similarity index 95%
rename from ansible/environments/jenkins/group_vars/all
rename to ansible/environments/jenkins/group_vars/openwhisk-vm3-he-de
index 659fdc2..e5ed3f2 100755
--- a/ansible/environments/jenkins/group_vars/all
+++ b/ansible/environments/jenkins/group_vars/openwhisk-vm3-he-de
@@ -6,7 +6,7 @@ config_root_dir: "{{ openwhisk_tmp_dir }}/wskconf"
whisk_logs_dir: "{{ openwhisk_tmp_dir }}/wsklogs"
coverage_enabled: "{{ lookup('env', 'GRADLE_COVERAGE') | default('false',
true) | bool}}"
coverage_logs_dir: "{{ openwhisk_tmp_dir }}/wskcov"
-docker_registry: "openwhisk-vm1-he-de.apache.org:444/"
+docker_registry: "openwhisk-vm3-he-de.apache.org:444/"
docker_dns: ""
runtimes_bypass_pull_for_local_images: true
invoker_use_runc: "{{ ansible_distribution != 'MacOSX' }}"
diff --git a/ansible/environments/jenkins/hosts.j2.ini
b/ansible/environments/jenkins/openwhisk-vm1-he-de.j2.ini
similarity index 100%
rename from ansible/environments/jenkins/hosts.j2.ini
rename to ansible/environments/jenkins/openwhisk-vm1-he-de.j2.ini
diff --git a/ansible/environments/jenkins/openwhisk-vm2-he-de.j2.ini
b/ansible/environments/jenkins/openwhisk-vm2-he-de.j2.ini
new file mode 100644
index 0000000..b6be1a4
--- /dev/null
+++ b/ansible/environments/jenkins/openwhisk-vm2-he-de.j2.ini
@@ -0,0 +1,36 @@
+; the first parameter in a host is the inventory_hostname
+
+; used for local actions only
+ansible ansible_connection=local
+
+[edge]
+5.9.20.92 ansible_host=5.9.20.92 ansible_connection=local
+
+[controllers]
+controller0 ansible_host=5.9.20.92 ansible_connection=local
+;{% if mode is defined and 'HA' in mode %}
+;controller1 ansible_host=5.9.20.92 ansible_connection=local
+;{% endif %}
+
+[kafkas]
+kafka0 ansible_host=5.9.20.92 ansible_connection=local
+{% if mode is defined and 'HA' in mode %}
+kafka1 ansible_host=5.9.20.92 ansible_connection=local
+{% endif %}
+
+[zookeepers:children]
+kafkas
+
+[invokers]
+invoker0 ansible_host=144.76.27.183 ansible_connection=ssh
+invoker1 ansible_host=144.76.63.94 ansible_connection=ssh
+
+; db group is only used if db.provider is CouchDB
+[db]
+5.9.20.92 ansible_host=5.9.20.92 ansible_connection=local
+
+[redis]
+5.9.20.92 ansible_host=5.9.20.92 ansible_connection=local
+
+[apigateway]
+5.9.20.92 ansible_host=5.9.20.92 ansible_connection=local
diff --git a/ansible/environments/jenkins/openwhisk-vm3-he-de.j2.ini
b/ansible/environments/jenkins/openwhisk-vm3-he-de.j2.ini
new file mode 100644
index 0000000..c150271
--- /dev/null
+++ b/ansible/environments/jenkins/openwhisk-vm3-he-de.j2.ini
@@ -0,0 +1,36 @@
+; the first parameter in a host is the inventory_hostname
+
+; used for local actions only
+ansible ansible_connection=local
+
+[edge]
+144.76.27.183 ansible_host=144.76.27.183 ansible_connection=local
+
+[controllers]
+controller0 ansible_host=144.76.27.183 ansible_connection=local
+;{% if mode is defined and 'HA' in mode %}
+;controller1 ansible_host=144.76.27.183 ansible_connection=local
+;{% endif %}
+
+[kafkas]
+kafka0 ansible_host=144.76.27.183 ansible_connection=local
+{% if mode is defined and 'HA' in mode %}
+kafka1 ansible_host=144.76.27.183 ansible_connection=local
+{% endif %}
+
+[zookeepers:children]
+kafkas
+
+[invokers]
+invoker0 ansible_host=144.76.63.94 ansible_connection=ssh
+invoker1 ansible_host=5.9.20.92 ansible_connection=ssh
+
+; db group is only used if db.provider is CouchDB
+[db]
+144.76.27.183 ansible_host=144.76.27.183 ansible_connection=local
+
+[redis]
+144.76.27.183 ansible_host=144.76.27.183 ansible_connection=local
+
+[apigateway]
+144.76.27.183 ansible_host=144.76.27.183 ansible_connection=local
