This is an automated email from the ASF dual-hosted git repository. mhanson pushed a commit to branch release/1.11.0 in repository https://gitbox.apache.org/repos/asf/geode.git
commit 3621f68a69ee01b79483783e4aac05daa1bfe3d8 Author: Robert Houghton <[email protected]> AuthorDate: Wed Dec 4 12:12:13 2019 -0800 Separate benchmark tasks into jobs. (#4404) * Separate benchmark tasks into jobs. To enable more granular re-run in Concourse, split the Benchmarks into independently triggerable jobs in the main pipeline. Extract variables from the template into the jinja.variables file. Make better use of YML anchors. Co-authored-by: Robert Houghton <[email protected]> Co-authored-by: Helena Bales <[email protected]> * empty to trigger pr (cherry picked from commit 1fcadf987706af7515a64869e088e1573695dae0) --- ci/pipelines/geode-build/jinja.template.yml | 92 ++++++++++++++++------------- ci/pipelines/shared/jinja.variables.yml | 10 ++++ 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml index a400a4a..b33fb8e 100644 --- a/ci/pipelines/geode-build/jinja.template.yml +++ b/ci/pipelines/geode-build/jinja.template.yml @@ -100,7 +100,9 @@ groups: {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %} - PublishArtifacts {%- endif %} - - Benchmark + {%- for flavor in (benchmarks.flavors) %} + - Benchmark{{flavor.title}} + {%- endfor %} - name: complete jobs: - {{ build_test.name }} @@ -113,7 +115,9 @@ groups: {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %} - PublishArtifacts {%- endif %} - - Benchmark + {%- for flavor in (benchmarks.flavors) %} + - Benchmark{{flavor.title}} + {%- endfor %} - name: linux jobs: - {{ build_test.name }} @@ -122,7 +126,9 @@ groups: - {{test.name}}Test{{java_test_version.name}} {%- endfor -%} {%- endfor %} - - Benchmark + {%- for flavor in (benchmarks.flavors) %} + - Benchmark{{flavor.title}} + {%- endfor %} - name: windows jobs: - {{ build_test.name }} @@ -362,7 +368,9 @@ jobs: {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %} - PublishArtifacts {% else %} - - Benchmark + {% for flavor in (benchmarks.flavors) %} + - Benchmark{{flavor.title}} + {% endfor %} {% endif %} trigger: true - get: geode-build-version @@ -381,7 +389,7 @@ jobs: path: bash args: - -cx - - |- + - | pushd geode GEODE_SHA=$(git rev-parse HEAD) popd @@ -405,15 +413,16 @@ jobs: params: file: geode-passing-tokens/passing-build-tokens.json -- name: Benchmark +{% for run_var in (benchmarks.flavors) %} +- name: Benchmark{{ run_var.title }} public: true max_in_flight: 3 plan: - get: geode-ci passed: {{ all_gating_jobs() | indent(4) }} + - get: alpine-tools-image - aggregate: - - get: alpine-tools-image - get: geode passed: {{ all_gating_jobs() | indent(6) }} @@ -423,8 +432,7 @@ jobs: passed: {{ all_gating_jobs() | indent(6) }} - put: concourse-metadata-resource - - aggregate: - {% for run_var in [{'title': '', 'flag':'', 'options': ''}, {'title': '_with_ssl', 'flag': '-PwithSsl', 'options': '--tests=*GetBenchmark --tests=*PutBenchmark'}, {'title': '_with_security_manager', 'flag': '-PwithSecurityManager', 'options': '--tests=Partitioned*'}] %} + - do: - task: run_benchmarks{{ run_var.title }} image: alpine-tools-image config: @@ -476,50 +484,50 @@ jobs: - name: geode-benchmarks - name: concourse-metadata-resource - name: results - {% endfor %} +{% endfor %} {% if repository.upstream_fork != "apache" or repository.branch == "develop" %} - name: PublishArtifacts public: true plan: - aggregate: - get: geode-ci - passed: - - Benchmark + passed: &publish-passed-inputs + {% for flavor in (benchmarks.flavors) %} + - Benchmark{{flavor.title}} + {% endfor %} - get: alpine-tools-image - get: geode - passed: - - Benchmark + passed: *publish-passed-inputs trigger: true - aggregate: - get: geode-build-version - passed: - - Benchmark - - do: - - put: concourse-metadata-resource - {{ init_retry()|indent(6) }} - - task: create_instance - image: alpine-tools-image - config: - platform: linux - params: - {{ common_instance_params(publish_artifacts) | indent(12) }} - GEODE_BRANCH: {{repository.branch}} - GEODE_FORK: {{repository.fork}} - JAVA_BUILD_VERSION: {{ java_build_version.version }} - run: - path: geode-ci/ci/scripts/create_instance.sh - inputs: - - name: concourse-metadata-resource - - name: geode-ci - - name: geode - - name: attempts-log - path: old - outputs: - - name: instance-data - - name: attempts-log - path: new - timeout: 15m - attempts: 10 + trigger: true + passed: *publish-passed-inputs + - put: concourse-metadata-resource + {{ init_retry()|indent(4) }} + - task: create_instance + image: alpine-tools-image + config: + platform: linux + params: + {{ common_instance_params(publish_artifacts) | indent(8) }} + GEODE_BRANCH: {{repository.branch}} + GEODE_FORK: {{repository.fork}} + JAVA_BUILD_VERSION: {{ java_build_version.version }} + run: + path: geode-ci/ci/scripts/create_instance.sh + inputs: + - name: concourse-metadata-resource + - name: geode-ci + - name: geode + - name: attempts-log + path: old + outputs: + - name: instance-data + - name: attempts-log + path: new + timeout: 15m + attempts: 10 - task: rsync_code_up image: alpine-tools-image config: diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml index 4b956b7..24fbcf3 100644 --- a/ci/pipelines/shared/jinja.variables.yml +++ b/ci/pipelines/shared/jinja.variables.yml @@ -19,6 +19,16 @@ benchmarks: baseline_branch: '' baseline_version: '1.10.0' benchmark_branch: develop + flavors: + - title: '_base' + flag: '' + options: '' + - title: '_with_ssl' + flag: '-PwithSsl' + options: '--tests=*GetBenchmark --tests=*PutBenchmark' + - title: '_with_security_manager' + flag: '-PwithSecurityManager' + options: '--tests=Partitioned*' build_test: ARTIFACT_SLUG: build
