This is an automated email from the ASF dual-hosted git repository. jensdeppe pushed a commit to branch windows-heavy-lifter in repository https://gitbox.apache.org/repos/asf/geode.git
commit 97a7795da3edf4cfb2460582b63c85332fb0a929 Author: Jens Deppe <jde...@pivotal.io> AuthorDate: Wed Aug 22 11:58:01 2018 -0700 initial commit for windows heavy lifting --- ci/pipelines/geode-build/jinja.template.yml | 71 +++-------------------------- ci/pipelines/shared/jinja.variables.yml | 53 ++++++++++----------- ci/scripts/execute_tests.sh | 2 +- ci/scripts/start_instance.sh | 29 ++++++++++-- 4 files changed, 56 insertions(+), 99 deletions(-) diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml index 39acf0a..7762377 100644 --- a/ci/pipelines/geode-build/jinja.template.yml +++ b/ci/pipelines/geode-build/jinja.template.yml @@ -31,7 +31,12 @@ {% endmacro %} {% macro common_test_params(test) %} + {%- if test.CALL_STACK_TIMEOUT -%} CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}} + {%- endif -%} + {%- if test.GRADLE_TASK_OPTIONS -%} +GRADLE_TASK_OPTIONS: {{test.GRADLE_TASK_OPTIONS}} + {%- endif -%} DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}} MAINTENANCE_VERSION: {{repository.branch}} PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}} @@ -43,7 +48,7 @@ groups: - name: main jobs: - Build - {% for test in (tests + windowstests) if not test.name=="StressNew" -%} + {% for test in tests if not test.name=="StressNew" -%} - {{test.name}}Test {% endfor -%} - UpdatePassingRef @@ -283,67 +288,3 @@ jobs: - name: instance-data timeout: 1h {% endfor -%} -{% for test in windowstests %} -- name: {{test.name}}Test - public: true - serial: true - plan: - {{ plan_resource_gets() |indent(2) }} - - task: cleanup-java-processes - config: - platform: windows - run: - path: powershell - args: - - -command - - | - gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list - kill -name java -force - exit 0 - tags: - - windows-{{test.tags}}-tests - - task: execute_tests - config: - platform: windows - params: - JAVA_HOME: C:\progra~1\java\jdk1.8.0_181 - SOURCE_REPOSITORY: {{repository.branch}} - {{ common_test_params(test) | indent(8) }} - run: - path: bash - args: - - geode-ci/ci/scripts/windows/test-run.sh - - {{test.target_arg}} - - {{test.dir_arg}} - - {{test.package_arg}} - inputs: - - name: geode-ci - - name: geode - outputs: - - name: built-geode - tags: - - windows-{{test.tags}}-tests - ensure: - aggregate: - - task: archive_results - config: - platform: windows - params: - MAINTENANCE_VERSION: {{repository.branch}} - PUBLIC_BUCKET: ((!public-bucket)) - SERVICE_ACCOUNT: ((!concourse-gcp-account)) - run: - path: bash - args: - - geode-ci/ci/scripts/windows/test-archive.sh - - {{test.target_arg}} - - {{test.dir_arg}} - inputs: - - name: geode-ci - - name: geode-build-version - - name: built-geode - tags: - - windows-{{test.tags}}-tests - timeout: 1h - timeout: 6h -{% endfor %} diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml index fd4789e..d0a28f4 100644 --- a/ci/pipelines/shared/jinja.variables.yml +++ b/ci/pipelines/shared/jinja.variables.yml @@ -18,12 +18,6 @@ repository: project: geode -platforms: -- name: "Linux" - prefix: "" -- name: "Windows" - prefix: "Windows" - tests: - name: "Acceptance" CPUS: "8" @@ -70,37 +64,40 @@ tests: GRADLE_TASK: repeatTest execute_test_timeout: 2h15m PARALLEL_DUNIT: "true" - -windowstests: - name: "WindowsAcceptance" - tags: "acceptance" - CALL_STACK_TIMEOUT: "1800" + tags: "windows" + CPUS: "16" + RAM: "32" + ARTIFACT_SLUG: windows-acceptancetestfiles DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: :geode-assembly:acceptanceTest + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: :geode-assembly:acceptanceTest - dir_arg: windows-acceptancetestfiles - package_arg: "" - name: "WindowsGfshDistributed" - tags: "distributed" - CALL_STACK_TIMEOUT: "1800" + tags: "windows" + CPUS: "16" + RAM: "32" + ARTIFACT_SLUG: windows-gfshdistributedtest DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: distributedTest + GRADLE_TASK_OPTIONS: "-PtestCategory=org.apache.geode.test.junit.categories.GfshTest" + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: distributedTest - dir_arg: windows-gfshdistributedtest - package_arg: org.apache.geode.test.junit.categories.GfshTest - name: "WindowsIntegration" - tags: "integration" - CALL_STACK_TIMEOUT: "1800" + tags: "windows" + CPUS: "16" + RAM: "32" + ARTIFACT_SLUG: windows-integrationtestfiles DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: integrationTest + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: integrationTest - dir_arg: windows-integrationtestfiles - package_arg: "" - name: "WindowsUnit" - tags: "unit" - CALL_STACK_TIMEOUT: "1800" + tags: "windows" + CPUS: "16" + RAM: "32" + ARTIFACT_SLUG: windows-unittestfiles DUNIT_PARALLEL_FORKS: "0" + GRADLE_TASK: test + execute_test_timeout: 6h PARALLEL_DUNIT: "false" - target_arg: test - dir_arg: windows-unittestfiles - package_arg: "" diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh index e270994..00db410 100755 --- a/ci/scripts/execute_tests.sh +++ b/ci/scripts/execute_tests.sh @@ -79,4 +79,4 @@ GRADLE_COMMAND="./gradlew \ ${SCM_PROPS}" echo "${GRADLE_COMMAND}" -ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode && ${GRADLE_COMMAND}" +ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'mkdir -p tmp; cd geode; ${GRADLE_COMMAND}'" diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh index c9a54f2..aa85924 100755 --- a/ci/scripts/start_instance.sh +++ b/ci/scripts/start_instance.sh @@ -50,11 +50,16 @@ SANITIZED_BUILD_PIPELINE_NAME=$(echo ${BUILD_PIPELINE_NAME} | tr "/" "-" | tr '[ SANITIZED_BUILD_JOB_NAME=$(echo ${BUILD_JOB_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]') SANITIZED_BUILD_NAME=$(echo ${BUILD_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]') IMAGE_FAMILY_PREFIX="" +WINDOWS_PREFIX="" if [[ "${GEODE_FORK}" != "apache" ]]; then IMAGE_FAMILY_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-" fi +if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ Windows ]]; then + WINDOWS_PREFIX="windows-" +fi + INSTANCE_NAME="$(echo "build-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr '[:upper:]' '[:lower:]')" PROJECT=apachegeode-ci ZONE=us-central1-f @@ -76,7 +81,7 @@ while true; do --min-cpu-platform=Intel\ Skylake \ --network="heavy-lifters" \ --subnet="heavy-lifters" \ - --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \ + --image-family="${IMAGE_FAMILY_PREFIX}${WINDOWS_PREFIX}geode-builder" \ --image-project=${PROJECT} \ --boot-disk-size=100GB \ --boot-disk-type=pd-ssd \ @@ -96,9 +101,23 @@ done echo "${INSTANCE_INFORMATION}" > instance-data/instance-information -INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP') +# This extracts the internal IP address for subsequent use +INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].networkIP') echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address" -while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do - echo -n . -done +if [[ -z "${WINDOWS_PREFIX}" ]]; then + while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do + echo -n . + done +else + # Set up ssh access for Windows systems + while [[ -z "${PASSWORD}" ]]; do + PASSWORD=$( yes | gcloud beta compute reset-windows-password ${INSTANCE_NAME} --user=geode --zone=${ZONE} --format json | jq -r .password ) + sleep 5 + done + + KEY=$( cat ${SSHKEY_FILE} ) + + winrm -hostname ${INSTANCE_IP_ADDRESS} -username geode -password "${PASSWORD}" \ + "powershell -command \"&{ mkdir c:\users\geode\.ssh -force; set-content -path c:\users\geode\.ssh\authorized_keys -encoding utf8 -value '${KEY}' }\"" +fi