This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new dc75cc5 Correctly retry ASF mirror listing task (#317)
dc75cc5 is described below
commit dc75cc5dce35e63102b34f36b9c6d5d157322ccd
Author: Arvind Shyamsundar <[email protected]>
AuthorDate: Thu Feb 6 12:06:24 2020 -0800
Correctly retry ASF mirror listing task (#317)
* Correctly retry ASF mirror listing task
The previous addition of retry was ineffective because it had a
failed_when condition - which needs to be an 'until' condition to
correctly detect success.
* Improve the fetching of ASF mirror
Consolidates shared code and uses jq to fetch preferred mirror from
the JSON listing of mirrors.
* Switch to using closer.cgi
Evidently, https://www.apache.org/dyn/closer.cgi?as_json is what we are
supposed to use as opposed to the mirrors.cgi script.
---
ansible/roles/accumulo/tasks/download.yml | 10 +---------
ansible/roles/fluo/tasks/download.yml | 10 +---------
ansible/roles/fluo_yarn/tasks/download.yml | 10 +---------
ansible/roles/proxy/tasks/download.yml | 10 +---------
ansible/roles/proxy/tasks/get-asf-mirror.yml | 9 +++++++++
ansible/roles/spark/tasks/download.yml | 10 +---------
ansible/scripts/install_ansible.sh | 3 +++
7 files changed, 17 insertions(+), 45 deletions(-)
diff --git a/ansible/roles/accumulo/tasks/download.yml
b/ansible/roles/accumulo/tasks/download.yml
index d29a8c5..e46ca1a 100644
--- a/ansible/roles/accumulo/tasks/download.yml
+++ b/ansible/roles/accumulo/tasks/download.yml
@@ -15,15 +15,7 @@
# limitations under the License.
#
-- name: "determine best apache mirror to use"
- shell: curl -sk https://apache.org/mirrors.cgi?as_json | grep preferred |
cut -d \" -f 4
- args:
- warn: no
- retries: 10
- delay: 10
- register: apache_mirror
- failed_when: "'http' not in apache_mirror.stdout"
- changed_when: False
+- import_tasks: ../../proxy/tasks/get-asf-mirror.yml
- name: "check if Accumulo tarball was uploaded to proxy"
stat: path={{ tarballs_dir }}/{{ accumulo_tarball }}
register: accumulo
diff --git a/ansible/roles/fluo/tasks/download.yml
b/ansible/roles/fluo/tasks/download.yml
index 52f3fb8..5dd2296 100644
--- a/ansible/roles/fluo/tasks/download.yml
+++ b/ansible/roles/fluo/tasks/download.yml
@@ -15,15 +15,7 @@
# limitations under the License.
#
-- name: "determine best apache mirror to use"
- shell: curl -sk https://apache.org/mirrors.cgi?as_json | grep preferred |
cut -d \" -f 4
- args:
- warn: no
- retries: 10
- delay: 10
- register: apache_mirror
- failed_when: "'http' not in apache_mirror.stdout"
- changed_when: False
+- import_tasks: ../../proxy/tasks/get-asf-mirror.yml
- name: "check if Fluo tarball was uploaded to proxy"
stat: path={{ tarballs_dir }}/{{ fluo_tarball }}
register: fluo
diff --git a/ansible/roles/fluo_yarn/tasks/download.yml
b/ansible/roles/fluo_yarn/tasks/download.yml
index 21aa848..5b995e3 100644
--- a/ansible/roles/fluo_yarn/tasks/download.yml
+++ b/ansible/roles/fluo_yarn/tasks/download.yml
@@ -15,15 +15,7 @@
# limitations under the License.
#
-- name: "determine best apache mirror to use"
- shell: curl -sk https://apache.org/mirrors.cgi?as_json | grep preferred |
cut -d \" -f 4
- args:
- warn: no
- retries: 10
- delay: 10
- register: apache_mirror
- failed_when: "'http' not in apache_mirror.stdout"
- changed_when: False
+- import_tasks: ../../proxy/tasks/get-asf-mirror.yml
- name: "check if Fluo YARN tarball was uploaded to proxy"
stat: path={{ tarballs_dir }}/{{ fluo_yarn_tarball }}
register: fluo_yarn
diff --git a/ansible/roles/proxy/tasks/download.yml
b/ansible/roles/proxy/tasks/download.yml
index 06d0af3..cd0d9dc 100644
--- a/ansible/roles/proxy/tasks/download.yml
+++ b/ansible/roles/proxy/tasks/download.yml
@@ -15,15 +15,7 @@
# limitations under the License.
#
-- name: "determine best apache mirror to use"
- shell: curl -sk https://apache.org/mirrors.cgi?as_json | grep preferred |
cut -d \" -f 4
- args:
- warn: no
- retries: 10
- delay: 10
- register: apache_mirror
- failed_when: "'http' not in apache_mirror.stdout"
- changed_when: False
+- import_tasks: get-asf-mirror.yml
- name: "download common tarballs to proxy"
get_url: url={{ item.urlp }}/{{ item.fn }} dest={{ tarballs_dir }}/{{
item.fn }} checksum="{{ item.sum }}" force=no
register: gresult
diff --git a/ansible/roles/proxy/tasks/get-asf-mirror.yml
b/ansible/roles/proxy/tasks/get-asf-mirror.yml
new file mode 100644
index 0000000..9a2a220
--- /dev/null
+++ b/ansible/roles/proxy/tasks/get-asf-mirror.yml
@@ -0,0 +1,9 @@
+- name: "determine best apache mirror to use"
+ shell: curl -sk https://www.apache.org/dyn/closer.cgi?as_json | jq -r
.preferred
+ args:
+ warn: no
+ register: apache_mirror
+ retries: 10
+ delay: 10
+ until: "apache_mirror.stdout | length > 0"
+ changed_when: False
diff --git a/ansible/roles/spark/tasks/download.yml
b/ansible/roles/spark/tasks/download.yml
index 6093969..65afc5e 100644
--- a/ansible/roles/spark/tasks/download.yml
+++ b/ansible/roles/spark/tasks/download.yml
@@ -15,15 +15,7 @@
# limitations under the License.
#
-- name: "determine best apache mirror to use"
- shell: curl -sk https://apache.org/mirrors.cgi?as_json | grep preferred |
cut -d \" -f 4
- args:
- warn: no
- retries: 10
- delay: 10
- register: apache_mirror
- failed_when: "'http' not in apache_mirror.stdout"
- changed_when: False
+- import_tasks: ../../proxy/tasks/get-asf-mirror.yml
- name: "check if Spark tarball was uploaded to proxy"
stat: path={{ tarballs_dir }}/{{ spark_tarball }}
register: spark
diff --git a/ansible/scripts/install_ansible.sh
b/ansible/scripts/install_ansible.sh
index 1dbb7fb..bfb7c67 100755
--- a/ansible/scripts/install_ansible.sh
+++ b/ansible/scripts/install_ansible.sh
@@ -48,3 +48,6 @@ fi
# install lxml as it is a dependency for the maven_artifact Ansible module
sudo yum install -q -y python-lxml
+
+# install jq to ease JSON parsing on the proxy
+sudo yum install -y jq
