Abacn commented on code in PR #27760: URL: https://github.com/apache/beam/pull/27760#discussion_r1281073417
########## .github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml: ########## @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Dataflow ARM + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + - '.github/workflows/beam_PostCommit_Java_Examples__Dataflow_ARM.yml' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.sender.login }}-${{ github.event.schedule }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + TAG: $(date +'%Y%m%d-%H%M%S%N') + +jobs: + beam_PostCommit_Java_Examples__Dataflow_ARM: + name: beam_PostCommit_Java_Examples__Dataflow_ARM (Run Java_Examples_Dataflow_ARM PostCommit) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + fail-fast: false + if: | + github.event_name == 'push' || Review Comment: Currently our postcommit only has schedule (cron) and comment (phrase) trigger. Should postcommit have push trigger (that will trigger on every push to master) @damccorm ########## runners/google-cloud-dataflow-java/arm/build.gradle: ########## @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import groovy.json.JsonOutput + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.runners.dataflow', + classesTriggerCheckerBugs: [ + 'PrimitiveParDoSingleFactory': 'https://github.com/typetools/checker-framework/issues/3791', + // TODO(https://github.com/apache/beam/issues/21068): This currently crashes with checkerframework 3.10.0 + // when compiling :runners:google-cloud-dataflow-java:compileJava with: + // message: class file for com.google.api.services.bigquery.model.TableRow not found + // ; The Checker Framework crashed. Please report the crash. + // Compilation unit: /usr/local/google/home/lcwik/git/beam/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java + // Last visited tree at line 57 column 1: + // @AutoService(CoderCloudObjectTranslatorRegistrar.class) + // Exception: com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.api.services.bigquery.model.TableRow not found; com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.api.services.bigquery.model.TableRow not found + 'DefaultCoderCloudObjectTranslatorRegistrar': 'TODO(https://github.com/apache/beam/issues/21068): Report the crash if still occurring on newest version', + ], +) + +description = "Apache Beam :: Runners :: Google Cloud Dataflow" + +/* + * We need to rely on manually specifying these evaluationDependsOn to ensure that + * the following projects are evaluated before we evaluate this project. This is because + * we are attempting to reference parameters such as "sourceSets.test.output" directly. + */ +evaluationDependsOn(":sdks:java:io:google-cloud-platform") +evaluationDependsOn(":sdks:java:core") +evaluationDependsOn(":examples:java") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") +evaluationDependsOn(":sdks:java:container:java8") +evaluationDependsOn(":sdks:java:container:java11") + +processResources { + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + 'dataflow.fnapi_environment_major_version' : '8', + 'dataflow.fnapi_container_version' : 'beam-master-20230426', + 'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3', + ] +} + +// Exclude tests that need a runner +test { + systemProperty "beamTestPipelineOptions", "" + systemProperty "beamUseDummyRunner", "true" + useJUnit { + excludeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + } +} + +dependencies { + dockerDependency project(path: ":runners:google-cloud-dataflow-java", configuration: "examplesJavaIntegrationTest") +} + +def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing' +def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1' +def dataflowValidatesTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-validates-runner-tests' +def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb' +def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" +def DockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}java${javaVer}_sdk" +def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss') +ext.DockerJavaMultiarchImageName = "${DockerJavaMultiarchImageContainer}:${dockerTag}" + +def runnerV2PipelineOptionsARMARM = [ + "--runner=TestDataflowRunner", + "--project=${dataflowProject}", + "--region=${dataflowRegion}", + "--tempRoot=${dataflowValidatesTempRoot}", + "--sdkContainerImage=${DockerJavaMultiarchImageContainer}:${dockerTag}", + "--experiments=use_unified_worker,use_runner_v2", + "--firestoreDb=${firestoreDb}", + "--machine_type=t2a-standard-1", +] + +// Build and push multi-arch docker images to a container registry for use within tests. +// NB: Tasks which consume docker images from the registry should depend on this +// task directly ('dependsOn buildAndPushDockerJavaMultiarchContainer'). This ensures the correct +// task ordering such that the registry doesn't get cleaned up prior to task completion. +def buildAndPushDockerJavaMultiarchContainer = tasks.register("buildAndPushDockerJavaMultiarchContainer") { + def javaVer = "java8" + if(project.hasProperty('compileAndRunTestsWithJava17')) { + javaVer = "java17" + } else if(project.hasProperty('compileAndRunTestsWithJava11')) { + javaVer = "java11" + } + dependsOn ":sdks:java:container:${javaVer}:docker" +} + +// Clean up built Java images +def cleanUpDockerJavaMultiarchImages = tasks.register("cleanUpDockerJavaMultiarchImages") { + doLast { + exec { + commandLine "gcloud", "--quiet", "container", "images", "untag", "${DockerJavaMultiarchImageName}" + } + exec { + commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${DockerJavaMultiarchImageContainer}" + } + } +} + +afterEvaluate { + // Ensure all tasks which use published docker images run before they are cleaned up + tasks.each { t -> + if (t.dependsOn.contains(buildAndPushDockerJavaMultiarchContainer) && !t.name.equalsIgnoreCase('printrunnerV2PipelineOptionsARM')) { + t.finalizedBy cleanUpDockerJavaMultiarchImages + } + } +} + +task printrunnerV2PipelineOptionsARM { + dependsOn buildAndPushDockerJavaMultiarchContainer + + doLast { + println "To run a Dataflow job with runner V2 on ARM, add the following pipeline options to your command-line:" + println runnerV2PipelineOptionsARM.join(' ') + println "Please delete your image upon completion with the following command:" + println "gcloud container images delete --force-delete-tags ${DockerJavaMultiarchImageName}" + } +} + +task examplesJavaRunnerV2IntegrationTestARM(type: Test) { + group = "Verification" + dependsOn buildAndPushDockerJavaMultiarchContainer + + systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptionsARM) + + // The examples/java preCommit task already covers running WordCountIT/WindowedWordCountIT so + // this postCommit integration test excludes them. + include '**/*IT.class' + exclude '**/WordCountIT.class' Review Comment: These exclusions are because wordcount on dataflow already run as precommit, so example postcommit excluded them. Here may include WordCountIT, WindowedWordCountIT also -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
