This is an automated email from the ASF dual-hosted git repository. upthewaterspout pushed a commit to branch concourse-staging in repository https://gitbox.apache.org/repos/asf/geode.git
commit 13abf1e21d0bf5048d055a3a33d49b1425b4d2f7 Author: Dan Smith <[email protected]> AuthorDate: Thu Sep 6 15:13:03 2018 -0700 Parameterizing the number of CPUs for each job Adding a DUNIT_CPU_QUOTA parameter, and adding different dunit jobs for different experiments. --- ci/pipelines/geode-build/jinja.template.yml | 1 + ci/pipelines/shared/jinja.variables.yml | 57 +++++++++++++++++++++++++++++ ci/scripts/execute_tests.sh | 4 ++ gradle.properties | 1 - 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml index 183a595..aee7371 100644 --- a/ci/pipelines/geode-build/jinja.template.yml +++ b/ci/pipelines/geode-build/jinja.template.yml @@ -33,6 +33,7 @@ {% macro common_test_params(test) %} CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}} DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}} +DUNIT_CPU_QUOTA: {{test.DUNIT_CPU_QUOTA}} MAINTENANCE_VERSION: {{repository.branch}} PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}} PARALLEL_GRADLE: {{test.PARALLEL_GRADLE}} diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml index e902678..baac65e 100644 --- a/ci/pipelines/shared/jinja.variables.yml +++ b/ci/pipelines/shared/jinja.variables.yml @@ -32,6 +32,7 @@ build_test: CALL_STACK_TIMEOUT: "1800" PARALLEL_DUNIT: "false" DUNIT_PARALLEL_FORKS: "4" + DUNIT_CPU_QUOTA: "-1" # Actual gradle task is "build install" but is hard-coded in execute_build.sh # It's consumption is more like a "TEST_RESULTS_DESTINATIONS_DIRECTORY_STUB_THING. GRADLE_TASK: build @@ -44,16 +45,68 @@ tests: ARTIFACT_SLUG: acceptancetestfiles CALL_STACK_TIMEOUT: "1800" DUNIT_PARALLEL_FORKS: "0" + DUNIT_CPU_QUOTA: "-1" GRADLE_TASK: acceptanceTest execute_test_timeout: 45m PARALLEL_DUNIT: "false" PARALLEL_GRADLE: "false" +- name: "Distributed-24-4" + CPUS: "96" + RAM: "180" + ARTIFACT_SLUG: distributedtestfiles + CALL_STACK_TIMEOUT: "7200" + DUNIT_PARALLEL_FORKS: "24" + DUNIT_CPU_QUOTA: "4" + GRADLE_TASK: distributedTest + execute_test_timeout: 2h15m + PARALLEL_DUNIT: "true" +- name: "Distributed-24-8" + CPUS: "96" + RAM: "180" + ARTIFACT_SLUG: distributedtestfiles + CALL_STACK_TIMEOUT: "7200" + DUNIT_PARALLEL_FORKS: "24" + DUNIT_CPU_QUOTA: "8" + GRADLE_TASK: distributedTest + execute_test_timeout: 2h15m + PARALLEL_DUNIT: "true" +- name: "Distributed-48-2" + CPUS: "96" + RAM: "240" + ARTIFACT_SLUG: distributedtestfiles + CALL_STACK_TIMEOUT: "7200" + DUNIT_PARALLEL_FORKS: "48" + DUNIT_CPU_QUOTA: "2" + GRADLE_TASK: distributedTest + execute_test_timeout: 2h15m + PARALLEL_DUNIT: "true" +- name: "Distributed-48-4" + CPUS: "96" + RAM: "240" + ARTIFACT_SLUG: distributedtestfiles + CALL_STACK_TIMEOUT: "7200" + DUNIT_PARALLEL_FORKS: "48" + DUNIT_CPU_QUOTA: "4" + GRADLE_TASK: distributedTest + execute_test_timeout: 2h15m + PARALLEL_DUNIT: "true" +- name: "Distributed-96-1" + CPUS: "96" + RAM: "360" + ARTIFACT_SLUG: distributedtestfiles + CALL_STACK_TIMEOUT: "7200" + DUNIT_PARALLEL_FORKS: "96" + DUNIT_CPU_QUOTA: "1" + GRADLE_TASK: distributedTest + execute_test_timeout: 2h15m + PARALLEL_DUNIT: "true" - name: "Distributed" CPUS: "96" RAM: "180" ARTIFACT_SLUG: distributedtestfiles CALL_STACK_TIMEOUT: "7200" DUNIT_PARALLEL_FORKS: "24" + DUNIT_CPU_QUOTA: "-1" GRADLE_TASK: distributedTest execute_test_timeout: 2h15m PARALLEL_DUNIT: "true" @@ -63,6 +116,7 @@ tests: ARTIFACT_SLUG: integrationtestfiles CALL_STACK_TIMEOUT: "1500" DUNIT_PARALLEL_FORKS: "48" + DUNIT_CPU_QUOTA: "-1" GRADLE_TASK: integrationTest execute_test_timeout: 40m PARALLEL_DUNIT: "true" @@ -72,6 +126,7 @@ tests: ARTIFACT_SLUG: upgradetestfiles CALL_STACK_TIMEOUT: "3000" DUNIT_PARALLEL_FORKS: "48" + DUNIT_CPU_QUOTA: "-1" GRADLE_TASK: upgradeTest execute_test_timeout: 1h PARALLEL_DUNIT: "true" @@ -81,6 +136,7 @@ tests: ARTIFACT_SLUG: stressnewtestfiles CALL_STACK_TIMEOUT: "7200" DUNIT_PARALLEL_FORKS: "24" + DUNIT_CPU_QUOTA: "-1" GRADLE_TASK: repeatTest execute_test_timeout: 2h15m PARALLEL_DUNIT: "true" @@ -90,6 +146,7 @@ windowstests: tags: "acceptance" CALL_STACK_TIMEOUT: "1800" DUNIT_PARALLEL_FORKS: "0" + DUNIT_CPU_QUOTA: "-1" PARALLEL_DUNIT: "false" target_arg: :geode-assembly:acceptanceTest dir_arg: windows-acceptancetestfiles diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh index d36f85b..cb596f6 100755 --- a/ci/scripts/execute_tests.sh +++ b/ci/scripts/execute_tests.sh @@ -62,6 +62,10 @@ if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then PARALLEL_DUNIT="-PparallelDunit -PdunitDockerUser=geode" if [ -n "${DUNIT_PARALLEL_FORKS}" ]; then DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}" + if (( $DUNIT_CPU_QUOTA > 0 )); then + DUNIT_PARALLEL_FORKS="${DUNIT_PARALLEL_FORKS} -PdunitCpuQuota=${DUNIT_CPU_QUOTA}" + fi + fi else PARALLEL_DUNIT="" diff --git a/gradle.properties b/gradle.properties index 3edf13d..9852423 100755 --- a/gradle.properties +++ b/gradle.properties @@ -67,4 +67,3 @@ repeat = 100 org.gradle.caching = true org.gradle.parallel = false org.gradle.configureondemand = false -
