apilloud commented on code in PR #23609:
URL: https://github.com/apache/beam/pull/23609#discussion_r995186170
##########
.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy:
##########
@@ -23,175 +23,16 @@ import CommonTestProperties.TriggeringContext
import NexmarkBuilder as Nexmark
import NoPhraseTriggeringPostCommitBuilder
import PhraseTriggeringPostCommitBuilder
-import InfluxDBCredentialsHelper
-
-import static NexmarkDatabaseProperties.nexmarkBigQueryArgs
-import static NexmarkDatabaseProperties.nexmarkInfluxDBArgs
// This job runs the suite of ValidatesRunner tests against the Spark runner.
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Spark',
'Spark Runner Nexmark Tests', this) {
+
description('Runs the Nexmark suite on the Spark runner.')
- // Set common parameters.
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master',
240, true, 'beam-perf')
- InfluxDBCredentialsHelper.useCredentials(delegate)
- // Gradle goals for this job.
- steps {
- shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 2 RUNNER ***"')
- gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:java:testing:nexmark:run')
- commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:spark:2"' +
- ' -Pnexmark.args="' +
- [
- commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
- commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
- '--runner=SparkRunner',
- '--streaming=false',
- '--suite=SMOKE',
- '--streamTimeout=60' ,
- '--manageResources=false',
- '--monitorJobs=true'
- ].join(' '))
- }
- shell('echo "*** RUN NEXMARK SQL IN BATCH MODE USING SPARK 2 RUNNER
***"')
- gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:java:testing:nexmark:run')
- commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:spark:2"' +
- ' -Pnexmark.args="' +
- [
- commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
- commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
- '--runner=SparkRunner',
- '--queryLanguage=sql',
- '--streaming=false',
- '--suite=SMOKE',
- '--streamTimeout=60' ,
- '--manageResources=false',
- '--monitorJobs=true'
- ].join(' '))
- }
- shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 3 RUNNER ***"')
- gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:java:testing:nexmark:run')
- commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:spark:3"' +
- ' -Pnexmark.args="' +
- [
- commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
- commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
- '--runner=SparkRunner',
- '--streaming=false',
- '--suite=SMOKE',
- '--streamTimeout=60' ,
- '--manageResources=false',
- '--monitorJobs=true'
- ].join(' '))
- }
- shell('echo "*** RUN NEXMARK SQL IN BATCH MODE USING SPARK 3 RUNNER
***"')
- gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:java:testing:nexmark:run')
- commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:spark:3"' +
- ' -Pnexmark.args="' +
- [
- commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
- commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
- '--runner=SparkRunner',
- '--queryLanguage=sql',
- '--streaming=false',
- '--suite=SMOKE',
- '--streamTimeout=60' ,
- '--manageResources=false',
- '--monitorJobs=true'
- ].join(' '))
- }
- shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 2 STRUCTURED
STREAMING RUNNER ***"')
- gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:java:testing:nexmark:run')
- commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:spark:2"' +
- ' -Pnexmark.args="' +
- [
- commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
- commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
- '--runner=SparkStructuredStreamingRunner',
- '--streaming=false',
- '--suite=SMOKE',
- // Skip query 3 (SparkStructuredStreamingRunner does not
support State/Timers yet)
- '--skipQueries=3',
- '--streamTimeout=60' ,
- '--manageResources=false',
- '--monitorJobs=true'
- ].join(' '))
- }
- shell('echo "*** RUN NEXMARK SQL IN BATCH MODE USING SPARK 2
STRUCTURED STREAMING RUNNER ***"')
- gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:java:testing:nexmark:run')
- commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:spark:2"' +
- ' -Pnexmark.args="' +
- [
- commonJobProperties.mapToArgString(nexmarkBigQueryArgs),
- commonJobProperties.mapToArgString(nexmarkInfluxDBArgs),
- '--runner=SparkStructuredStreamingRunner',
- '--queryLanguage=sql',
- '--streaming=false',
- '--suite=SMOKE',
- '--streamTimeout=60' ,
- '--manageResources=false',
- '--monitorJobs=true'
- ].join(' '))
- }
- shell('echo "*** RUN NEXMARK IN BATCH MODE USING SPARK 3 STRUCTURED
STREAMING RUNNER ***"')
Review Comment:
This is actually going to fail when it runs. I'll probably open a bug and
fix in a followup.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
