This is an automated email from the ASF dual-hosted git repository. jensdeppe pushed a commit to branch windows-heavy-lifter in repository https://gitbox.apache.org/repos/asf/geode.git
commit 62d84b790c7021cf8f98166678538005d4f61d5a Author: Jens Deppe <[email protected]> AuthorDate: Wed Aug 22 11:58:01 2018 -0700 GEODE-5668: Switch Windows build jobs to 'heavy-lifter' model --- ci/pipelines/geode-build/jinja.template.yml | 71 +++-------------------------- ci/pipelines/shared/jinja.variables.yml | 49 +++++++++----------- ci/scripts/execute_tests.sh | 2 +- ci/scripts/rsync_code_down.sh | 4 +- ci/scripts/start_instance.sh | 34 +++++++++++--- gradle.properties | 2 +- 6 files changed, 59 insertions(+), 103 deletions(-) diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml index e94c99d..e72b2ca 100644 --- a/ci/pipelines/geode-build/jinja.template.yml +++ b/ci/pipelines/geode-build/jinja.template.yml @@ -31,7 +31,12 @@ {% endmacro %} {% macro common_test_params(test) %} + {% if test.CALL_STACK_TIMEOUT %} CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}} + {% endif %} + {% if test.GRADLE_TASK_OPTIONS %} +GRADLE_TASK_OPTIONS: {{test.GRADLE_TASK_OPTIONS}} + {% endif %} DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}} MAINTENANCE_VERSION: {{repository.branch}} PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}} @@ -44,7 +49,7 @@ groups: - name: main jobs: - Build - {% for test in (tests + windowstests) if not test.name=="StressNew" -%} + {% for test in tests if not test.name=="StressNew" -%} - {{test.name}}Test {% endfor -%} - UpdatePassingRef @@ -321,67 +326,3 @@ jobs: - name: instance-data timeout: 1h {% endfor -%} -{% for test in windowstests %} -- name: {{test.name}}Test - public: true - serial: true - plan: - {{ plan_resource_gets() |indent(2) }} - - task: cleanup-java-processes - config: - platform: windows - run: - path: powershell - args: - - -command - - | - gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list - kill -name java -force - exit 0 - tags: - - windows-{{test.tags}}-tests - - task: execute_tests - config: - platform: windows - params: - JAVA_HOME: C:\progra~1\java\jdk1.8.0_181 - SOURCE_REPOSITORY: {{repository.branch}} - {{ common_test_params(test) | indent(8) }} - run: - path: bash - args: - - geode-ci/ci/scripts/windows/test-run.sh - - {{test.target_arg}} - - {{test.dir_arg}} - - {{test.package_arg}} - inputs: - - name: geode-ci - - name: geode - outputs: - - name: built-geode - tags: - - windows-{{test.tags}}-tests - ensure: - aggregate: - - task: archive_results - config: - platform: windows - params: - MAINTENANCE_VERSION: {{repository.branch}} - PUBLIC_BUCKET: ((!public-bucket)) - SERVICE_ACCOUNT: ((!concourse-gcp-account)) - run: - path: bash - args: - - geode-ci/ci/scripts/windows/test-archive.sh - - {{test.target_arg}} - - {{test.dir_arg}} - inputs: - - name: geode-ci - - name: geode-build-version - - name: built-geode - tags: - - windows-{{test.tags}}-tests - timeout: 1h - timeout: 6h -{% endfor %} diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml index 85d495a..e37c7ee 100644 --- a/ci/pipelines/shared/jinja.variables.yml +++ b/ci/pipelines/shared/jinja.variables.yml @@ -18,12 +18,6 @@ repository: project: geode -platforms: -- name: "Linux" - prefix: "" -- name: "Windows" - prefix: "Windows" - tests: - name: "Acceptance" CPUS: "8" @@ -71,37 +65,36 @@ tests: GRADLE_TASK: repeatTest execute_test_timeout: 2h15m PARALLEL_DUNIT: "true" - -windowstests: - name: "WindowsAcceptance" - tags: "acceptance" - CALL_STACK_TIMEOUT: "1800" + CPUS: "16" + RAM: "64" + ARTIFACT_SLUG: windows-acceptancetestfiles DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: :geode-assembly:acceptanceTest + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: :geode-assembly:acceptanceTest - dir_arg: windows-acceptancetestfiles - package_arg: "" - name: "WindowsGfshDistributed" - tags: "distributed" - CALL_STACK_TIMEOUT: "1800" + CPUS: "16" + RAM: "64" + ARTIFACT_SLUG: windows-gfshdistributedtest DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: distributedTest + GRADLE_TASK_OPTIONS: "-PtestCategory=org.apache.geode.test.junit.categories.GfshTest" + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: distributedTest - dir_arg: windows-gfshdistributedtest - package_arg: org.apache.geode.test.junit.categories.GfshTest - name: "WindowsIntegration" - tags: "integration" - CALL_STACK_TIMEOUT: "1800" + CPUS: "16" + RAM: "64" + ARTIFACT_SLUG: windows-integrationtestfiles DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: integrationTest + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: integrationTest - dir_arg: windows-integrationtestfiles - package_arg: "" - name: "WindowsUnit" - tags: "unit" - CALL_STACK_TIMEOUT: "1800" + CPUS: "16" + RAM: "64" + ARTIFACT_SLUG: windows-unittestfiles DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: test + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: test - dir_arg: windows-unittestfiles - package_arg: "" diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh index d36f85b..c4bd456 100755 --- a/ci/scripts/execute_tests.sh +++ b/ci/scripts/execute_tests.sh @@ -84,4 +84,4 @@ GRADLE_COMMAND="./gradlew \ ${SCM_PROPS}" echo "${GRADLE_COMMAND}" -ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode && ${GRADLE_COMMAND}" +ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'mkdir -p tmp; cd geode; ${GRADLE_COMMAND}'" diff --git a/ci/scripts/rsync_code_down.sh b/ci/scripts/rsync_code_down.sh index ce12bcc..a94bb90 100755 --- a/ci/scripts/rsync_code_down.sh +++ b/ci/scripts/rsync_code_down.sh @@ -40,7 +40,7 @@ echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config OUTPUT_DIR=${BASE_DIR}/geode-results -ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "cd geode && ./gradlew combineReports" +ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'cd geode; ./gradlew --no-daemon combineReports'" -time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:. ${OUTPUT_DIR}/. +time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:geode ${OUTPUT_DIR}/ set +x diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh index 3bee7f9..cd4d191 100755 --- a/ci/scripts/start_instance.sh +++ b/ci/scripts/start_instance.sh @@ -18,6 +18,7 @@ # limitations under the License. set -e +set -x BASE_DIR=$(pwd) @@ -54,11 +55,16 @@ SANITIZED_BUILD_PIPELINE_NAME=$(echo ${BUILD_PIPELINE_NAME} | tr "/" "-" | tr '[ SANITIZED_BUILD_JOB_NAME=$(echo ${BUILD_JOB_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]') SANITIZED_BUILD_NAME=$(echo ${BUILD_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]') IMAGE_FAMILY_PREFIX="" +WINDOWS_PREFIX="" if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-" fi +if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ [Ww]indows ]]; then + WINDOWS_PREFIX="windows-" +fi + INSTANCE_NAME="$(echo "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr '[:upper:]' '[:lower:]')" PROJECT=apachegeode-ci ZONE=us-central1-f @@ -78,9 +84,9 @@ while true; do --zone=${ZONE} \ --machine-type=custom-${CPUS}-${RAM_MEGABYTES} \ --min-cpu-platform=Intel\ Skylake \ - --network="heavy-lifters" \ - --subnet="heavy-lifters" \ - --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \ + --network="heavy-lifters-2" \ + --subnet="heavy-lifters-2" \ + --image-family="${IMAGE_FAMILY_PREFIX}${WINDOWS_PREFIX}geode-builder" \ --image-project=${PROJECT} \ --boot-disk-size=100GB \ --boot-disk-type=pd-ssd \ @@ -103,6 +109,22 @@ echo "${INSTANCE_INFORMATION}" > instance-data/instance-information INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP') echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address" -while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do - echo -n . -done +if [[ -z "${WINDOWS_PREFIX}" ]]; then + while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do + echo -n . + done +else + # Set up ssh access for Windows systems + while [[ -z "${PASSWORD}" ]]; do + PASSWORD=$( yes | gcloud beta compute reset-windows-password ${INSTANCE_NAME} --user=geode --zone=${ZONE} --format json | jq -r .password ) + sleep 5 + done + + ssh-keygen -N "" -f ${SSHKEY_FILE} + + KEY=$( cat ${SSHKEY_FILE}.pub ) + + winrm -hostname ${INSTANCE_IP_ADDRESS} -username geode -password "${PASSWORD}" \ + -https -insecure -port 5986 \ + "powershell -command \"&{ mkdir c:\users\geode\.ssh -force; set-content -path c:\users\geode\.ssh\authorized_keys -encoding utf8 -value '${KEY}' }\"" +fi diff --git a/gradle.properties b/gradle.properties index 3edf13d..e9c97ec 100755 --- a/gradle.properties +++ b/gradle.properties @@ -42,7 +42,7 @@ productName = Apache Geode productOrg = Apache Software Foundation (ASF) org.gradle.daemon = true -org.gradle.jvmargs = -Xmx3g +org.gradle.jvmargs = -Xmx4g minimumGradleVersion = 4.10 # Set this on the command line with -P or in ~/.gradle/gradle.properties
