This is an automated email from the ASF dual-hosted git repository. rhoughton pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/geode.git
commit 1e46eeda20b1a28ac85d28791108f5c7d80bbdf3 Author: Robert Houghton <rhough...@pivotal.io> AuthorDate: Fri May 3 13:02:14 2019 -0700 GEODE-6739: Create Concourse heavy-lift workers in the current zone GCP costs are incurred on data egress between zones, even the same region. Track our `create-instance` attempts, to put the heavy lifter in the same zone on the first attempt, and roulette only on failure Co-authored-by: Robert Houghton <rhough...@pivotal.io> Co-authored-by: Helena Bales <hba...@pivotal.io> --- ci/pipelines/geode-build/jinja.template.yml | 6 ++++++ ci/pipelines/pull-request/jinja.template.yml | 6 ++++++ ci/pipelines/shared/shared_jinja.yml | 10 ++++++++++ ci/scripts/create_instance.sh | 18 ++++++++++++++---- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml index 4ea7643..c5f16b5 100644 --- a/ci/pipelines/geode-build/jinja.template.yml +++ b/ci/pipelines/geode-build/jinja.template.yml @@ -18,6 +18,7 @@ {% from 'shared_jinja.yml' import alpine_tools_config with context %} {% from 'shared_jinja.yml' import pipeline_prefix with context %} {% from 'shared_jinja.yml' import github_access with context %} +{% from 'shared_jinja.yml' import init_retry with context %} --- @@ -224,6 +225,7 @@ jobs: pre: ((semver-prerelease-token)) - do: - put: concourse-metadata-resource + {{ init_retry()|indent(6) }} - task: create_instance {{- alpine_tools_config()|indent(8) }} params: @@ -234,9 +236,13 @@ jobs: run: path: geode-ci/ci/scripts/create_instance.sh inputs: + - name: attempts-log + path: old - name: concourse-metadata-resource - name: geode-ci outputs: + - name: attempts-log + path: new - name: instance-data timeout: 15m attempts: 10 diff --git a/ci/pipelines/pull-request/jinja.template.yml b/ci/pipelines/pull-request/jinja.template.yml index bcec2a1..1d2e50e 100644 --- a/ci/pipelines/pull-request/jinja.template.yml +++ b/ci/pipelines/pull-request/jinja.template.yml @@ -17,6 +17,7 @@ {% from 'shared_jinja.yml' import alpine_tools_config with context %} {% from 'shared_jinja.yml' import pipeline_prefix with context %} +{% from 'shared_jinja.yml' import init_retry with context %} groups: - name: main @@ -87,6 +88,7 @@ jobs: get_params: {skip_download: true} - do: - put: concourse-metadata-resource + {{ init_retry()|indent(8) }} - task: create_instance {{- alpine_tools_config()|indent(10) }} params: @@ -103,8 +105,12 @@ jobs: - name: concourse-metadata-resource - name: geode - name: geode-ci + - name: attempts-log + path: old outputs: - name: instance-data + - name: attempts-log + path: new timeout: 15m attempts: 100 - task: rsync_code_up diff --git a/ci/pipelines/shared/shared_jinja.yml b/ci/pipelines/shared/shared_jinja.yml index 5c2d80e..080e411 100644 --- a/ci/pipelines/shared/shared_jinja.yml +++ b/ci/pipelines/shared/shared_jinja.yml @@ -38,3 +38,13 @@ username: ((github-username)) password: ((github-password)) {%- endif %} {%- endmacro %} + +{%- macro init_retry() -%} +- task: initial-output + {{- docker_config()|indent(2) }} + outputs: + - name: attempts-log + run: + path: touch + args: ['attempts-log/attempts'] +{%- endmacro %} \ No newline at end of file diff --git a/ci/scripts/create_instance.sh b/ci/scripts/create_instance.sh index 27692ff..02cc769 100755 --- a/ci/scripts/create_instance.sh +++ b/ci/scripts/create_instance.sh @@ -59,10 +59,6 @@ if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ [Ww]indows ]]; then WINDOWS_PREFIX="windows-" fi -PERMITTED_ZONES=(us-central1-a us-central1-b us-central1-c us-central1-f) -ZONE=${PERMITTED_ZONES[$((${RANDOM} % 4))]} -echo "Deploying to zone ${ZONE}" - INSTANCE_NAME_STRING="${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-build${JAVA_BUILD_VERSION}-test${JAVA_TEST_VERSION}-job#${BUILD_NAME}" INSTANCE_NAME="heavy-lifter-$(uuidgen -n @dns -s -N "${INSTANCE_NAME_STRING}")" @@ -77,6 +73,20 @@ GCP_NETWORK=${GCP_NETWORK##*/} GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork') GCP_SUBNETWORK=${GCP_SUBNETWORK##*/} +# Determine and store our attempt number +cp old/attempts new/ +echo attempt >> new/attempts +attempts=$(cat new/attempts | wc -l) +echo $attempts > /tmp/retry_number + +if [ $attempts -eq 1 ]; then + ZONE=${MY_ZONE} +else + PERMITTED_ZONES=(us-central1-a us-central1-b us-central1-c us-central1-f) + ZONE=${PERMITTED_ZONES[$((${RANDOM} % 4))]} +fi +echo "Deploying to zone ${ZONE}" + #in a retry loop we intentionally generate the same instance name, so make sure prior attempt is cleaned up gcloud compute instances delete ${INSTANCE_NAME} --zone=${ZONE} --quiet &>/dev/null || true