[
https://issues.apache.org/jira/browse/BEAM-3954?focusedWorklogId=123860&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-123860
]
ASF GitHub Bot logged work on BEAM-3954:
----------------------------------------
Author: ASF GitHub Bot
Created on: 16/Jul/18 23:29
Start Date: 16/Jul/18 23:29
Worklog Time Spent: 10m
Work Description: stale[bot] closed pull request #4960: [BEAM-3954] try
jenkins-pipeline with docker images - Test, Do Not Merge
URL: https://github.com/apache/beam/pull/4960
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/Dockerfile b/.test-infra/jenkins/Dockerfile
new file mode 100644
index 00000000000..cf02f478305
--- /dev/null
+++ b/.test-infra/jenkins/Dockerfile
@@ -0,0 +1,69 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM openjdk:8-jdk
+MAINTAINER Beam [email protected]
+
+# Install dependencies for Jenkins Python tests
+RUN apt-get update && \
+ apt-get install -y \
+ python-pip \
+ python-virtualenv \
+ python-dev \
+ python-tox \
+ maven \
+ rsync \
+ time \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN wget
https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-189.0.0-linux-x86_64.tar.gz
-O gcloud.tar.gz && \
+tar xf gcloud.tar.gz && \
+./google-cloud-sdk/install.sh --quiet
+ENV PATH="/google-cloud-sdk/bin:${PATH}"
+RUN gcloud components update --quiet || echo 'gcloud components update failed'
&& \
+ rm gcloud.tar.gz
+
+# Add the entrypoint script which install gcloud sdk
+#COPY docker-entrypoint.sh /usr/local/bin/
+#RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh
+#ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+
+# Create beam user to validate the build on user space
+ENV USER=user \
+ UID=1001 \
+ GID=1002 \
+ HOME=/home/user
+RUN groupadd --system --gid=$GID $USER; \
+ useradd --system --uid=$UID --gid $USER $USER;
+RUN mkdir -p $HOME; \
+ chown -R $USER:$USER $HOME; \
+ chmod 777 $HOME;
+USER $USER
+WORKDIR $HOME
+COPY my-first-project-190318.json $HOME/credentials.json
+RUN gcloud auth activate-service-account --key-file=credentials.json && \
+ gcloud config set project my-first-project-190318 && \
+ rm credentials.json
+
+
+#ARG URL=https://github.com/apache/beam
+#
+#RUN git clone $URL beam; \
+# cd beam; \
+# git config --local --add remote.origin.fetch
'+refs/pull/*/head:refs/remotes/origin/pr/*'; \
+# git fetch --quiet --all;
diff --git a/.test-infra/jenkins/docker-entrypoint.sh
b/.test-infra/jenkins/docker-entrypoint.sh
new file mode 100644
index 00000000000..b4b2009a845
--- /dev/null
+++ b/.test-infra/jenkins/docker-entrypoint.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apk add --update openssl
+
+wget
https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-189.0.0-linux-x86_64.tar.gz
-O gcloud.tar.gz
+tar xf gcloud.tar.gz
+./google-cloud-sdk/install.sh --quiet
+. ./google-cloud-sdk/path.bash.inc
+gcloud components update --quiet || echo 'gcloud components update failed'
+RUN gcloud auth activate-service-account --key-file=credentials.json && \
+ gcloud config set project my-first-project-190318 && \
+ rm credentials.json
+exec "$@"
diff --git a/.test-infra/jenkins/job_00_seed.groovy
b/.test-infra/jenkins/job_00_seed.groovy
deleted file mode 100644
index 9fcd9d632ba..00000000000
--- a/.test-infra/jenkins/job_00_seed.groovy
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Defines the seed job, which creates or updates all other Jenkins projects.
-job('beam_SeedJob') {
- description('Automatically configures all Apache Beam Jenkins projects
based' +
- ' on Jenkins DSL groovy files checked into the code repository.')
-
- properties {
- githubProjectUrl('https://github.com/apache/beam/')
- }
-
- // Restrict to only run on Jenkins executors labeled 'beam'
- label('beam')
-
- logRotator {
- daysToKeep(14)
- }
-
- scm {
- git {
- remote {
- github('apache/beam')
-
- // ${ghprbPullId} is not interpolated by groovy, but passed through to
Jenkins where it
- // refers to the environment variable
- refspec(['+refs/heads/*:refs/remotes/origin/*',
-
'+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*']
- .join(' '))
-
- // The variable ${sha1} is not interpolated by groovy, but a parameter
of the Jenkins job
- branch('${sha1}')
-
- extensions {
- cleanAfterCheckout()
- }
- }
- }
- }
-
- parameters {
- // Setup for running this job from a pull request
- stringParam(
- 'sha1',
- 'master',
- 'Commit id or refname (eg: origin/pr/4001/head) you want to build
against.')
- }
-
- wrappers {
- timeout {
- absolute(60)
- abortBuild()
- }
- }
-
- triggers {
- // Run once per day
- cron('0 */6 * * *')
-
- githubPullRequest {
- admins(['asfbot'])
- useGitHubHooks()
- orgWhitelist(['apache'])
- allowMembersOfWhitelistedOrgsAsAdmin()
- permitAll()
-
- // Also run when manually kicked on a pull request
- triggerPhrase('Run Seed Job')
- onlyTriggerPhrase()
-
- extensions {
- commitStatus {
- context("Jenkins: Seed Job")
- }
-
- buildStatus {
- completedStatus('SUCCESS', '--none--')
- completedStatus('FAILURE', '--none--')
- completedStatus('ERROR', '--none--')
- }
- }
- }
- }
-
- // If anything goes wrong, mail the main dev list, because it is a big deal
- publishers {
- mailer('[email protected]', false, true)
- }
-
- steps {
- dsl {
- // A list or a glob of other groovy files to process.
- external('.test-infra/jenkins/job_*.groovy')
-
- // If a job is removed from the script, disable it (rather than
deleting).
- removeAction('DISABLE')
- }
- }
-}
diff --git a/.test-infra/jenkins/job_beam_Inventory.groovy
b/.test-infra/jenkins/job_beam_Inventory.groovy
deleted file mode 100644
index 60b18d3d59c..00000000000
--- a/.test-infra/jenkins/job_beam_Inventory.groovy
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// These jobs list details about each beam runner, to clarify what software
-// is on each machine.
-def nums = 9..24
-nums.each {
- def machine = "beam${it}"
- job("beam_Inventory_${machine}") {
- description("Run inventory on ${machine}")
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Sets that this is a cron job.
- common_job_properties.setCronJob(delegate, '45 18 * * *')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Machine Inventory',
- "Run Inventory ${machine}")
-
- parameters {
- nodeParam('TEST_HOST') {
- description("Select test host ${machine}")
- defaultNodes([machine])
- allowedNodes([machine])
- trigger('multiSelectionDisallowed')
- eligibility('IgnoreOfflineNodeEligibility')
- }
- }
-
- steps {
- shell('ls /home/jenkins/tools')
- shell('ls /home/jenkins/tools/*')
- shell('python --version || echo "python not found"')
- shell('python3 --version || echo "python3 not found"')
- shell('/home/jenkins/tools/maven/latest/mvn -v || echo "mvn not found"')
- shell('/home/jenkins/tools/gradle4.3/gradle -v || echo "gradle not
found"')
- shell('gcloud -v || echo "gcloud not found"')
- shell('kubectl version -c || echo "kubectl not found"')
- shell('virtualenv -p python2.7 test2 && . ./test2/bin/activate && python
--version && deactivate || echo "python 2.7 not found"')
- shell('virtualenv -p python3 test3 && . ./test3/bin/activate && python
--version && deactivate || echo "python 3 not found"')
- shell('echo "Maven home $MAVEN_HOME"')
- shell('env')
- }
- }
-}
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
deleted file mode 100644
index 2c8f7ef182b..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Beam performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_Dataflow'){
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]',
- false)
-
- def argMap = [
- benchmarks: 'dpb_wordcount_benchmark',
- dpb_dataflow_staging_location:
'gs://temp-storage-for-perf-tests/staging',
- dpb_wordcount_input: 'dataflow-samples/shakespeare/kinglear.txt',
- config_override:
'dpb_wordcount_benchmark.dpb_service.service_type=dataflow'
- ]
-
- common_job_properties.buildPerformanceTest(delegate, argMap)
-
- // [BEAM-2141] Perf tests do not pass.
- disabled()
-}
diff --git
a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy
deleted file mode 100644
index dc382c1f9bd..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-def testsConfigurations = [
- [
- jobName : 'beam_PerformanceTests_TextIOIT',
- jobDescription : 'Runs PerfKit tests for TextIOIT',
- itClass : 'org.apache.beam.sdk.io.text.TextIOIT',
- bqTable : 'beam_performance.textioit_pkb_results',
- prCommitStatusName: 'Java TextIO Performance Test',
- prTriggerPhase : 'Run Java TextIO Performance Test',
- extraPipelineArgs: [
- numberOfRecords: '1000000'
- ]
-
- ],
- [
- jobName :
'beam_PerformanceTests_Compressed_TextIOIT',
- jobDescription : 'Runs PerfKit tests for TextIOIT with
GZIP compression',
- itClass : 'org.apache.beam.sdk.io.text.TextIOIT',
- bqTable :
'beam_performance.compressed_textioit_pkb_results',
- prCommitStatusName : 'Java CompressedTextIO Performance Test',
- prTriggerPhase : 'Run Java CompressedTextIO Performance
Test',
- extraPipelineArgs: [
- numberOfRecords: '1000000',
- compressionType: 'GZIP'
- ]
- ],
- [
- jobName : 'beam_PerformanceTests_AvroIOIT',
- jobDescription : 'Runs PerfKit tests for AvroIOIT',
- itClass : 'org.apache.beam.sdk.io.avro.AvroIOIT',
- bqTable : 'beam_performance.avroioit_pkb_results',
- prCommitStatusName: 'Java AvroIO Performance Test',
- prTriggerPhase : 'Run Java AvroIO Performance Test',
- extraPipelineArgs: [
- numberOfRecords: '1000000'
- ]
- ],
- [
- jobName : 'beam_PerformanceTests_TFRecordIOIT',
- jobDescription : 'Runs PerfKit tests for
beam_PerformanceTests_TFRecordIOIT',
- itClass :
'org.apache.beam.sdk.io.tfrecord.TFRecordIOIT',
- bqTable :
'beam_performance.tfrecordioit_pkb_results',
- prCommitStatusName: 'Java TFRecordIO Performance Test',
- prTriggerPhase : 'Run Java TFRecordIO Performance Test',
- extraPipelineArgs: [
- numberOfRecords: '1000000'
- ]
- ],
- [
- jobName : 'beam_PerformanceTests_XmlIOIT',
- jobDescription : 'Runs PerfKit tests for
beam_PerformanceTests_XmlIOIT',
- itClass : 'org.apache.beam.sdk.io.xml.XmlIOIT',
- bqTable : 'beam_performance.xmlioit_pkb_results',
- prCommitStatusName: 'Java XmlIOPerformance Test',
- prTriggerPhase : 'Run Java XmlIO Performance Test',
- extraPipelineArgs: [
- numberOfRecords: '100000000',
- charset: 'UTF-8'
- ]
- ]
-]
-
-for (testConfiguration in testsConfigurations) {
- create_filebasedio_performance_test_job(testConfiguration)
-}
-
-
-private void create_filebasedio_performance_test_job(testConfiguration) {
-
- // This job runs the file-based IOs performance tests on PerfKit
Benchmarker.
- job(testConfiguration.jobName) {
- description(testConfiguration.jobDescription)
-
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- testConfiguration.prCommitStatusName,
- testConfiguration.prTriggerPhase)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]',
- false)
-
- def pipelineOptions = [
- project : 'apache-beam-testing',
- tempRoot : 'gs://temp-storage-for-perf-tests',
- filenamePrefix :
"gs://temp-storage-for-perf-tests/${testConfiguration.jobName}/\${BUILD_ID}/",
- ]
- if (testConfiguration.containsKey('extraPipelineArgs')) {
- pipelineOptions << testConfiguration.extraPipelineArgs
- }
-
- def argMap = [
- benchmarks : 'beam_integration_benchmark',
- beam_it_timeout : '1200',
- beam_it_profile : 'io-it',
- beam_prebuilt : 'false',
- beam_sdk : 'java',
- beam_it_module : 'sdks/java/io/file-based-io-tests',
- beam_it_class : testConfiguration.itClass,
- beam_it_options :
common_job_properties.joinPipelineOptions(pipelineOptions),
- beam_extra_mvn_properties: '["filesystem=gcs"]',
- bigquery_table : testConfiguration.bqTable,
- ]
- common_job_properties.buildPerformanceTest(delegate, argMap)
- }
-}
\ No newline at end of file
diff --git
a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT_HDFS.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT_HDFS.groovy
deleted file mode 100644
index 7aa5c3251c0..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT_HDFS.groovy
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-def testsConfigurations = [
- [
- jobName : 'beam_PerformanceTests_TextIOIT_HDFS',
- jobDescription : 'Runs PerfKit tests for TextIOIT on HDFS',
- itClass : 'org.apache.beam.sdk.io.text.TextIOIT',
- bqTable :
'beam_performance.textioit_hdfs_pkb_results',
- prCommitStatusName: 'Java TextIO Performance Test on HDFS',
- prTriggerPhase : 'Run Java TextIO Performance Test HDFS',
- extraPipelineArgs: [
- numberOfRecords: '1000000'
- ]
-
- ],
- [
- jobName :
'beam_PerformanceTests_Compressed_TextIOIT_HDFS',
- jobDescription : 'Runs PerfKit tests for TextIOIT with
GZIP compression on HDFS',
- itClass : 'org.apache.beam.sdk.io.text.TextIOIT',
- bqTable :
'beam_performance.compressed_textioit_hdfs_pkb_results',
- prCommitStatusName : 'Java CompressedTextIO Performance Test
on HDFS',
- prTriggerPhase : 'Run Java CompressedTextIO Performance
Test HDFS',
- extraPipelineArgs: [
- numberOfRecords: '1000000',
- compressionType: 'GZIP'
- ]
- ],
- [
- jobName : 'beam_PerformanceTests_AvroIOIT_HDFS',
- jobDescription : 'Runs PerfKit tests for AvroIOIT on HDFS',
- itClass : 'org.apache.beam.sdk.io.avro.AvroIOIT',
- bqTable :
'beam_performance.avroioit_hdfs_pkb_results',
- prCommitStatusName: 'Java AvroIO Performance Test on HDFS',
- prTriggerPhase : 'Run Java AvroIO Performance Test HDFS',
- extraPipelineArgs: [
- numberOfRecords: '1000000'
- ]
- ],
-// TODO(BEAM-3945) TFRecord performance test is failing only when running on
hdfs.
-// We need to fix this before enabling this job on jenkins.
-// [
-// jobName :
'beam_PerformanceTests_TFRecordIOIT_HDFS',
-// jobDescription : 'Runs PerfKit tests for
beam_PerformanceTests_TFRecordIOIT on HDFS',
-// itClass :
'org.apache.beam.sdk.io.tfrecord.TFRecordIOIT',
-// bqTable :
'beam_performance.tfrecordioit_hdfs_pkb_results',
-// prCommitStatusName: 'Java TFRecordIO Performance Test on
HDFS',
-// prTriggerPhase : 'Run Java TFRecordIO Performance Test
HDFS',
-// extraPipelineArgs: [
-// numberOfRecords: '1000000'
-// ]
-// ],
- [
- jobName : 'beam_PerformanceTests_XmlIOIT_HDFS',
- jobDescription : 'Runs PerfKit tests for
beam_PerformanceTests_XmlIOIT on HDFS',
- itClass : 'org.apache.beam.sdk.io.xml.XmlIOIT',
- bqTable :
'beam_performance.xmlioit_hdfs_pkb_results',
- prCommitStatusName: 'Java XmlIOPerformance Test on HDFS',
- prTriggerPhase : 'Run Java XmlIO Performance Test HDFS',
- extraPipelineArgs: [
- numberOfRecords: '100000',
- charset: 'UTF-8'
- ]
- ]
-]
-
-for (testConfiguration in testsConfigurations) {
- create_filebasedio_performance_test_job(testConfiguration)
-}
-
-
-private void create_filebasedio_performance_test_job(testConfiguration) {
-
- // This job runs the file-based IOs performance tests on PerfKit
Benchmarker.
- job(testConfiguration.jobName) {
- description(testConfiguration.jobDescription)
-
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- testConfiguration.prCommitStatusName,
- testConfiguration.prTriggerPhase)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]',
- false)
-
- def pipelineArgs = [
- project : 'apache-beam-testing',
- tempRoot : 'gs://temp-storage-for-perf-tests',
- ]
- if (testConfiguration.containsKey('extraPipelineArgs')) {
- pipelineArgs << testConfiguration.extraPipelineArgs
- }
-
- def pipelineArgList = []
- pipelineArgs.each({
- key, value -> pipelineArgList.add("\"--$key=$value\"")
- })
- def pipelineArgsJoined = "[" + pipelineArgList.join(',') + "]"
-
- String namespace =
common_job_properties.getKubernetesNamespace('filebasedioithdfs')
- String kubeconfig =
common_job_properties.getKubeconfigLocationForNamespace(namespace)
-
- def argMap = [
- kubeconfig : kubeconfig,
- benchmarks : 'beam_integration_benchmark',
- beam_it_timeout : '1200',
- beam_it_profile : 'io-it',
- beam_prebuilt : 'false',
- beam_sdk : 'java',
- beam_it_module : 'sdks/java/io/file-based-io-tests',
- beam_it_class : testConfiguration.itClass,
- beam_it_options : pipelineArgsJoined,
- beam_extra_mvn_properties: '["filesystem=hdfs"]',
- bigquery_table : testConfiguration.bqTable,
- beam_options_config_file : makePathAbsolute('pkb-config.yml'),
- beam_kubernetes_scripts :
makePathAbsolute('hdfs-single-datanode-cluster.yml') + ',' +
makePathAbsolute('hdfs-single-datanode-cluster-for-local-dev.yml')
- ]
- common_job_properties.setupKubernetes(delegate, namespace, kubeconfig)
- common_job_properties.buildPerformanceTest(delegate, argMap)
- common_job_properties.cleanupKubernetes(delegate, namespace,
kubeconfig)
- }
-}
-
-static def makePathAbsolute(String path) {
- return '"$WORKSPACE/src/.test-infra/kubernetes/hadoop/SmallITCluster/' +
path + '"'
-}
\ No newline at end of file
diff --git
a/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy
deleted file mode 100644
index a4182858f8e..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-job('beam_PerformanceTests_HadoopInputFormat') {
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]',
- false)
-
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Java HadoopInputFormatIO Performance Test',
- 'Run Java HadoopInputFormatIO Performance Test')
-
- def pipelineOptions = [
- tempRoot : 'gs://temp-storage-for-perf-tests',
- project : 'apache-beam-testing',
- postgresPort : '5432',
- numberOfRecords: '600000'
- ]
-
- String namespace =
common_job_properties.getKubernetesNamespace('hadoopinputformatioit')
- String kubeconfig =
common_job_properties.getKubeconfigLocationForNamespace(namespace)
-
- def testArgs = [
- kubeconfig : kubeconfig,
- beam_it_timeout : '1200',
- benchmarks : 'beam_integration_benchmark',
- beam_it_profile : 'io-it',
- beam_prebuilt : 'false',
- beam_sdk : 'java',
- beam_it_module : 'sdks/java/io/hadoop-input-format',
- beam_it_class :
'org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT',
- beam_it_options :
common_job_properties.joinPipelineOptions(pipelineOptions),
- beam_kubernetes_scripts :
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
- beam_options_config_file:
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
- bigquery_table :
'beam_performance.hadoopinputformatioit_pkb_results'
- ]
-
- common_job_properties.setupKubernetes(delegate, namespace, kubeconfig)
- common_job_properties.buildPerformanceTest(delegate, testArgs)
- common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig)
-}
-
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
deleted file mode 100644
index 30964a0db0f..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-job('beam_PerformanceTests_JDBC') {
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]',
- false)
-
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Java JdbcIO Performance Test',
- 'Run Java JdbcIO Performance Test')
-
- def pipelineOptions = [
- tempRoot : 'gs://temp-storage-for-perf-tests',
- project : 'apache-beam-testing',
- postgresPort : '5432',
- numberOfRecords: '5000000'
- ]
-
- String namespace = common_job_properties.getKubernetesNamespace('jdbcioit')
- String kubeconfig =
common_job_properties.getKubeconfigLocationForNamespace(namespace)
-
- def testArgs = [
- kubeconfig : kubeconfig,
- beam_it_timeout : '1800',
- benchmarks : 'beam_integration_benchmark',
- beam_it_profile : 'io-it',
- beam_prebuilt : 'false',
- beam_sdk : 'java',
- beam_it_module : 'sdks/java/io/jdbc',
- beam_it_class : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
- beam_it_options :
common_job_properties.joinPipelineOptions(pipelineOptions),
- beam_kubernetes_scripts :
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
- beam_options_config_file:
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
- bigquery_table : 'beam_performance.jdbcioit_pkb_results'
- ]
-
- common_job_properties.setupKubernetes(delegate, namespace, kubeconfig)
- common_job_properties.buildPerformanceTest(delegate, testArgs)
- common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig)
-}
-
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_MongoDBIO_IT.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_MongoDBIO_IT.groovy
deleted file mode 100644
index 31cfc5138a2..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_MongoDBIO_IT.groovy
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-job('beam_PerformanceTests_MongoDBIO_IT') {
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]',
- false)
-
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Java MongoDBIO Performance Test',
- 'Run Java MongoDBIO Performance Test')
-
- def pipelineOptions = [
- tempRoot : 'gs://temp-storage-for-perf-tests',
- project : 'apache-beam-testing',
- numberOfRecords: '10000000'
- ]
-
- String namespace =
common_job_properties.getKubernetesNamespace('mongodbioit')
- String kubeconfig =
common_job_properties.getKubeconfigLocationForNamespace(namespace)
-
- def testArgs = [
- kubeconfig : kubeconfig,
- beam_it_timeout : '1800',
- benchmarks : 'beam_integration_benchmark',
- beam_it_profile : 'io-it',
- beam_prebuilt : 'false',
- beam_sdk : 'java',
- beam_it_module : 'sdks/java/io/mongodb',
- beam_it_class :
'org.apache.beam.sdk.io.mongodb.MongoDBIOIT',
- beam_it_options :
common_job_properties.joinPipelineOptions(pipelineOptions),
- beam_kubernetes_scripts :
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml'),
- beam_options_config_file:
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/mongodb/load-balancer/pkb-config.yml'),
- bigquery_table :
'beam_performance.mongodbioit_pkb_results'
- ]
-
- common_job_properties.setupKubernetes(delegate, namespace, kubeconfig)
- common_job_properties.buildPerformanceTest(delegate, testArgs)
- common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig)
-}
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Python.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_Python.groovy
deleted file mode 100644
index 6a71bdaa51d..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_Python.groovy
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Beam Python performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_Python'){
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Python SDK Performance Test',
- 'Run Python Performance Test')
-
- def pipelineArgs = [
- project: 'apache-beam-testing',
- staging_location: 'gs://temp-storage-for-end-to-end-tests/staging-it',
- temp_location: 'gs://temp-storage-for-end-to-end-tests/temp-it',
- output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output'
- ]
- def pipelineArgList = []
- pipelineArgs.each({
- key, value -> pipelineArgList.add("--$key=$value")
- })
- def pipelineArgsJoined = pipelineArgList.join(',')
-
- def argMap = [
- beam_sdk : 'python',
- benchmarks: 'beam_integration_benchmark',
- beam_it_args: pipelineArgsJoined
- ]
-
- common_job_properties.buildPerformanceTest(delegate, argMap)
-}
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy
deleted file mode 100644
index 2e62d0b2509..00000000000
--- a/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Beam performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_Spark'){
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Spark Performance Test',
- 'Run Spark Performance Test')
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- '[email protected]',
- false)
-
- def argMap = [
- benchmarks: 'dpb_wordcount_benchmark',
- // There are currently problems uploading to Dataproc, so we use a file
- // already present on the machines as input.
- dpb_wordcount_input: '/etc/hosts',
- config_override:
'dpb_wordcount_benchmark.dpb_service.service_type=dataproc',
- bigquery_table: 'beam_performance.spark_pkp_results'
- ]
-
- common_job_properties.buildPerformanceTest(delegate, argMap)
-}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Go_GradleBuild.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Go_GradleBuild.groovy
deleted file mode 100644
index 0e667b4c109..00000000000
--- a/.test-infra/jenkins/job_beam_PostCommit_Go_GradleBuild.groovy
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Go postcommit which runs a gradle build, and the current set
-// of postcommit tests.
-job('beam_PostCommit_Go_GradleBuild') {
- description('Runs Go PostCommit tests against master.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'master',
- 150)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate, '15 */6 * * *', false)
-
- def gradle_command_line = './gradlew ' +
common_job_properties.gradle_switches.join(' ') + ' :goPostCommit'
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- gradle_command_line,
- 'Run Go PostCommit')
-
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':goPostCommit')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_GradleBuild.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_GradleBuild.groovy
deleted file mode 100644
index 7456ea514d4..00000000000
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_GradleBuild.groovy
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-job('beam_PostCommit_Java_GradleBuild') {
- description('Runs PostCommit tests on the Java SDK.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
- // Publish all test results to Jenkins
- publishers {
- archiveJunit('**/build/test-results/**/*.xml')
- }
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Java SDK Post Commit Tests',
- 'Run Java PostCommit')
-
- // Gradle goals for this job.
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':javaPostCommit')
- common_job_properties.setGradleSwitches(delegate)
- // Specify maven home on Jenkins, needed by Maven archetype integration
tests.
- switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.2')
- }
- }
-}
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
deleted file mode 100644
index a76260ab9e2..00000000000
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Apex runner.
-job('beam_PostCommit_Java_ValidatesRunner_Apex_Gradle') {
- description('Runs the ValidatesRunner suite on the Apex runner.')
- previousNames('beam_PostCommit_Java_ValidatesRunner_Apex')
- previousNames('beam_PostCommit_Java_RunnableOnService_Apex')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Publish all test results to Jenkins
- publishers {
- archiveJunit('**/build/test-results/**/*.xml')
- }
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Apex Runner ValidatesRunner Tests',
- 'Run Apex ValidatesRunner')
-
- // Gradle goals for this job.
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':beam-runners-apex:validatesRunner')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
deleted file mode 100644
index 6a1ea96ae9a..00000000000
---
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow
-// runner.
-job('beam_PostCommit_Java_ValidatesRunner_Dataflow_Gradle') {
- description('Runs the ValidatesRunner suite on the Dataflow runner.')
- previousNames('beam_PostCommit_Java_ValidatesRunner_Dataflow')
- previousNames('beam_PostCommit_Java_RunnableOnService_Dataflow')
-
- // Set common parameters. Sets a long (5 hour) timeout due to timeouts in
[BEAM-3775].
- common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 300)
-
- // Publish all test results to Jenkins
- publishers {
- archiveJunit('**/build/test-results/**/*.xml')
- }
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Google Cloud Dataflow Runner ValidatesRunner Tests',
- 'Run Dataflow ValidatesRunner')
-
- // Gradle goals for this job.
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':beam-runners-google-cloud-dataflow-java:validatesRunner')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
deleted file mode 100644
index 07327ff5548..00000000000
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Flink runner.
-job('beam_PostCommit_Java_ValidatesRunner_Flink_Gradle') {
- description('Runs the ValidatesRunner suite on the Flink runner.')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Publish all test results to Jenkins
- publishers {
- archiveJunit('**/build/test-results/**/*.xml')
- }
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Flink Runner ValidatesRunner Tests',
- 'Run Flink ValidatesRunner')
-
- // Gradle goals for this job.
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':beam-runners-flink_2.11:validatesRunner')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
deleted file mode 100644
index c3e5c9a6a4b..00000000000
---
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Gearpump
-// runner.
-job('beam_PostCommit_Java_ValidatesRunner_Gearpump_Gradle') {
- description('Runs the ValidatesRunner suite on the Gearpump runner.')
- previousNames('beam_PostCommit_Java_ValidatesRunner_Gearpump')
- previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'gearpump-runner')
-
- // Publish all test results to Jenkins
- publishers {
- archiveJunit('**/build/test-results/**/*.xml')
- }
-
- // Sets that this is a PostCommit job.
- // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
- // In post-commit this job triggers only on SCM changes.
- common_job_properties.setPostCommit(delegate, '0 5 31 2 *')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Gearpump Runner ValidatesRunner Tests',
- 'Run Gearpump ValidatesRunner')
-
- // Gradle goals for this job.
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':beam-runners-gearpump:validatesRunner')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
deleted file mode 100644
index 1d4179c0317..00000000000
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Spark runner.
-job('beam_PostCommit_Java_ValidatesRunner_Spark_Gradle') {
- description('Runs the ValidatesRunner suite on the Spark runner.')
- previousNames('beam_PostCommit_Java_ValidatesRunner_Spark')
- previousNames('beam_PostCommit_Java_RunnableOnService_Spark')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
- // Publish all test results to Jenkins
- publishers {
- archiveJunit('**/build/test-results/**/*.xml')
- }
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Spark Runner ValidatesRunner Tests',
- 'Run Spark ValidatesRunner')
-
- // Gradle goals for this job.
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':beam-runners-spark:validatesRunner')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesContainer_Dataflow.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesContainer_Dataflow.groovy
deleted file mode 100644
index 90344107b13..00000000000
---
a/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesContainer_Dataflow.groovy
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of Python ValidatesContainer tests against the
-// Dataflow runner.
-job('beam_PostCommit_Python_ValidatesContainer_Dataflow') {
- description('Runs Python ValidatesContainer suite on the Dataflow runner.')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate, '30 3 * * *', false)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Google Cloud Dataflow Runner Python ValidatesContainer Tests',
- 'Run Python Dataflow ValidatesContainer')
-
- // Execute shell command to test Python SDK.
- steps {
- shell('cd ' + common_job_properties.checkoutDir + ' && bash
sdks/python/container/run_validatescontainer.sh')
- }
-}
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy
deleted file mode 100644
index 392a93598c7..00000000000
---
a/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of Python ValidatesRunner tests against the
-// Dataflow runner.
-job('beam_PostCommit_Python_ValidatesRunner_Dataflow') {
- description('Runs Python ValidatesRunner suite on the Dataflow runner.')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Google Cloud Dataflow Runner Python ValidatesRunner Tests',
- 'Run Python Dataflow ValidatesRunner')
-
- // Execute shell command to test Python SDK.
- steps {
- shell('cd ' + common_job_properties.checkoutDir + ' && bash
sdks/python/run_validatesrunner.sh')
- }
-}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy
deleted file mode 100644
index 73a918e13a6..00000000000
--- a/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job defines the Python postcommit tests.
-job('beam_PostCommit_Python_Verify') {
- description('Runs postcommit tests on the Python SDK.')
-
- previousNames('beam_PostCommit_PythonVerify')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Python SDK PostCommit Tests',
- 'Run Python PostCommit')
-
- // Execute shell command to test Python SDK.
- steps {
- shell('cd ' + common_job_properties.checkoutDir + ' && bash
sdks/python/run_postcommit.sh')
- }
-}
diff --git a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
index 29c7ae11eff..4b37407c685 100644
--- a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
@@ -1,63 +1,63 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This verifies the nightly snapshot build.
-// From https://repository.apache.org/content/groups/snapshots/org/apache/beam.
-job('beam_PostRelease_NightlySnapshot') {
- description('Runs post release verification of the nightly snapshot.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- parameters {
- stringParam('snapshot_version',
- '',
- 'Version of the repository snapshot to install')
- stringParam('snapshot_url',
- '',
- 'Repository URL to install from')
- }
-
- // This is a post-commit job that runs once per day, not for every push.
- common_job_properties.setPostCommit(
- delegate,
- '0 11 * * *',
- false)
-
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- './gradlew :release:runJavaExamplesValidationTask',
- 'Run Dataflow PostRelease')
-
- steps {
- // Run a quickstart from
https://beam.apache.org/get-started/quickstart-java
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':release:runJavaExamplesValidationTask')
- common_job_properties.setGradleSwitches(delegate)
- switches('-Pver=$snapshot_version -Prepourl=$snapshot_url')
- }
- }
-}
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//import common_job_properties
+//
+//// This verifies the nightly snapshot build.
+//// From
https://repository.apache.org/content/groups/snapshots/org/apache/beam.
+//job('beam_PostRelease_NightlySnapshot') {
+// description('Runs post release verification of the nightly snapshot.')
+//
+// // Execute concurrent builds if necessary.
+// concurrentBuild()
+//
+// // Set common parameters.
+// common_job_properties.setTopLevelMainJobProperties(delegate)
+//
+// parameters {
+// stringParam('snapshot_version',
+// '',
+// 'Version of the repository snapshot to install')
+// stringParam('snapshot_url',
+// '',
+// 'Repository URL to install from')
+// }
+//
+// // This is a post-commit job that runs once per day, not for every push.
+// common_job_properties.setPostCommit(
+// delegate,
+// '0 11 * * *',
+// false)
+//
+//
+// // Allows triggering this build against pull requests.
+// common_job_properties.enablePhraseTriggeringFromPullRequest(
+// delegate,
+// './gradlew :release:runQuickstartsJava',
+// 'Run Dataflow PostRelease')
+//
+// steps {
+// // Run a quickstart from
https://beam.apache.org/get-started/quickstart-java
+// gradle {
+// rootBuildScriptDir(common_job_properties.checkoutDir)
+// tasks(':release:runQuickstartsJava')
+// common_job_properties.setGradleSwitches(delegate)
+// switches('-Pver=$snapshot_version -Prepourl=$snapshot_url')
+// }
+// }
+//}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Go_GradleBuild.groovy
b/.test-infra/jenkins/job_beam_PreCommit_Go_GradleBuild.groovy
index 2aff95751a7..9cedc1a90ee 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Go_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Go_GradleBuild.groovy
@@ -1,45 +1,45 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Go precommit which runs a gradle build, and the current set
-// of precommit tests.
-job('beam_PreCommit_Go_GradleBuild') {
- description('Runs Go PreCommit tests for the current GitHub Pull Request.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'master',
- 150)
-
- def gradle_command_line = './gradlew ' +
common_job_properties.gradle_switches.join(' ') + ' :goPreCommit'
- // Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run Go
PreCommit')
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':goPreCommit')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//import common_job_properties
+//
+//// This is the Go precommit which runs a gradle build, and the current set
+//// of precommit tests.
+//job('beam_PreCommit_Go_GradleBuild') {
+// description('Runs Go PreCommit tests for the current GitHub Pull Request.')
+//
+// // Execute concurrent builds if necessary.
+// concurrentBuild()
+//
+// // Set common parameters.
+// common_job_properties.setTopLevelMainJobProperties(
+// delegate,
+// 'master',
+// 150)
+//
+// def gradle_command_line = './gradlew ' +
common_job_properties.gradle_switches.join(' ') + ' :goPreCommit'
+// // Sets that this is a PreCommit job.
+// common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run Go
PreCommit')
+// steps {
+// gradle {
+// rootBuildScriptDir(common_job_properties.checkoutDir)
+// tasks(':goPreCommit')
+// common_job_properties.setGradleSwitches(delegate)
+// }
+// }
+//}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
index 56af6da64a4..1069afcc5c4 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
@@ -16,34 +16,39 @@
* limitations under the License.
*/
-import common_job_properties
+//import common_job_properties
// This is the Java precommit which runs a Gradle build, and the current set
// of precommit tests.
job('beam_PreCommit_Java_GradleBuild') {
- description('Runs Java PreCommit tests for the current GitHub Pull Request.')
+// description('Runs Java PreCommit tests for the current GitHub Pull
Request.')
+//
+// // Execute concurrent builds if necessary.
+// concurrentBuild()
+//
+// // Set common parameters.
+// common_job_properties.setTopLevelMainJobProperties(
+// delegate,
+// 'master',
+// 90)
+//
+// // Publish all test results to Jenkins
+// publishers {
+// archiveJunit('**/build/test-results/**/*.xml')
+// }
+//
+// // Sets that this is a PreCommit job.
+// common_job_properties.setPreCommit(delegate, './gradlew :javaPreCommit',
'Run Java PreCommit')
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'master',
- 90)
-
- // Publish all test results to Jenkins
- publishers {
- archiveJunit('**/build/test-results/**/*.xml')
- }
+ wrappers {
+ buildInDocker {
+ dockerfile('.test-infra/jenkins/', 'Dockerfile')
+ }
+ }
- // Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, './gradlew :javaPreCommit',
'Run Java PreCommit')
steps {
gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
tasks(':javaPreCommit')
- common_job_properties.setGradleSwitches(delegate)
// Specify maven home on Jenkins, needed by Maven archetype integration
tests.
switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.2')
}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_GradleBuild.groovy
b/.test-infra/jenkins/job_beam_PreCommit_Python_GradleBuild.groovy
index cc58dc005ed..4bbbc273e6d 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Python_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Python_GradleBuild.groovy
@@ -16,36 +16,44 @@
* limitations under the License.
*/
-import common_job_properties
+//import common_job_properties
// This is the Python precommit which runs a Gradle build, and the current set
// of precommit tests.
job('beam_PreCommit_Python_GradleBuild') {
- description('Runs Python PreCommit tests for the current GitHub Pull
Request.')
+// description('Runs Python PreCommit tests for the current GitHub Pull
Request.')
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'master',
- 90)
-
- // Publish all test results to Jenkins. Note that Nose documentation
- // specifically mentions that it produces JUnit compatible test results.
+// // Execute concurrent builds if necessary.
+// concurrentBuild()
+//
+// // Set common parameters.
+// common_job_properties.setTopLevelMainJobProperties(
+// delegate,
+// 'master',
+// 90)
+//
+// // Publish all test results to Jenkins. Note that Nose documentation
+// // specifically mentions that it produces JUnit compatible test results.
publishers {
archiveJunit('**/nosetests.xml')
}
-
- def gradle_command_line = './gradlew ' +
common_job_properties.gradle_switches.join(' ') + ' :pythonPreCommit'
+//
+// def gradle_command_line = './gradlew ' +
common_job_properties.gradle_switches.join(' ') + ' :pythonPreCommit'
// Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run
Python PreCommit')
+// common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run
Python PreCommit')
+
+
+ wrappers {
+ buildInDocker {
+ dockerfile('.test-infra/jenkins/', 'Dockerfile')
+ }
+ }
+
steps {
gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
+// rootBuildScriptDir(common_job_properties.checkoutDir)
tasks(':pythonPreCommit')
- common_job_properties.setGradleSwitches(delegate)
+// common_job_properties.setGradleSwitches(delegate)
}
}
}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Merge.groovy
b/.test-infra/jenkins/job_beam_PreCommit_Website_Merge.groovy
deleted file mode 100644
index 71fa13eb728..00000000000
--- a/.test-infra/jenkins/job_beam_PreCommit_Website_Merge.groovy
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// Defines a job.
-job('beam_PreCommit_Website_Merge') {
- description('Runs website tests for mergebot.')
-
- // Set common parameters.
- common_job_properties.setTopLevelWebsiteJobProperties(delegate, 'mergebot')
-
- triggers {
- githubPush()
- }
-
- steps {
- // Run the following shell script as a build step.
- shell '''
- # Install RVM per instructions at https://rvm.io/rvm/install.
- gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
- 409B6B1796C275462A1703113804BB82D39DC0E3 \\
- 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-
- \\curl -sSL https://get.rvm.io | bash
- source /home/jenkins/.rvm/scripts/rvm
-
- # Install Ruby.
- RUBY_VERSION_NUM=2.3.0
- rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
-
- # Install Bundler gem
- PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
- GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
- gem install bundler --user-install
-
- # Enter the git clone for remaining commands
- cd src
-
- # Install all needed gems.
- bundle install --path ~/.gem/
-
- # Build the new site and test it.
- rm -fr ./content/
- bundle exec rake test
- '''.stripIndent().trim()
- }
-}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
b/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
deleted file mode 100644
index d1a79dac15c..00000000000
--- a/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// Defines a job.
-job('beam_PreCommit_Website_Stage') {
- description('Stages the pull requests proposed for the Apache Beam ' +
- 'website to a temporary location to ease reviews.')
-
- // Set common parameters.
- common_job_properties.setTopLevelWebsiteJobProperties(delegate)
-
- // Set pull request build trigger.
- common_job_properties.setPreCommit(
- delegate,
- 'Automatic staging of pull requests',
- '\nJenkins built the site at commit id ${ghprbActualCommit} with ' +
- 'Jekyll and staged it [here](http://apache-beam-website-pull-' +
- 'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' +
- 'Happy reviewing.\n\nNote that any previous site has been deleted. ' +
- 'This staged site will be automatically deleted after its TTL ' +
- 'expires. Push any commit to the pull request branch or re-trigger ' +
- 'the build to get it staged again.')
-
- steps {
- // Run the following shell script as a build step.
- shell '''
- # Install RVM.
- gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
- 409B6B1796C275462A1703113804BB82D39DC0E3 \\
- 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-
- \\curl -sSL https://get.rvm.io | bash
- source /home/jenkins/.rvm/scripts/rvm
-
- # Install Ruby.
- RUBY_VERSION_NUM=2.3.0
- rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
-
- # Install Bundler gem
- PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
- GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
- gem install bundler --user-install
-
- # Enter the git clone for remaining commands
- cd src
-
- # Install all needed gems.
- bundle install --path ~/.gem/
-
- # Remove current site if it exists.
- GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/"
- gsutil -m rm -r -f ${GCS_PATH} || true
-
- # Build the new site with the baseurl specified.
- rm -fr ./content/
- bundle exec jekyll build --baseurl=/${ghprbPullId}
-
- # Install BeautifulSoup HTML Parser for python.
- pip install --user beautifulsoup4
-
- # Fix links on staged website.
- python .jenkins/append_index_html_to_internal_links.py
-
- # Upload the new site.
- gsutil -m cp -R ./content/* ${GCS_PATH}
- '''.stripIndent().trim()
- }
-}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
b/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
deleted file mode 100644
index dd11871ef02..00000000000
--- a/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// Defines a job.
-job('beam_PreCommit_Website_Test') {
- description('Runs tests on the pull requests proposed for the Apache Beam ' +
- 'website.')
-
- // Set common parameters.
- common_job_properties.setTopLevelWebsiteJobProperties(delegate)
-
- // Execute concurrent builds. Multiple builds of this project may be executed
- // in parallel. This is safe because this build does not require exclusive
- // access to any shared resources.
- concurrentBuild()
-
- // Set pull request build trigger.
- common_job_properties.setPreCommit(
- delegate,
- 'Test website (dead links, etc.)')
-
- steps {
- // Run the following shell script as a build step.
- shell '''
- # Install RVM.
- gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
- 409B6B1796C275462A1703113804BB82D39DC0E3 \\
- 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-
- \\curl -sSL https://get.rvm.io | bash
- source /home/jenkins/.rvm/scripts/rvm
-
- # Install Ruby.
- RUBY_VERSION_NUM=2.3.0
- rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
-
- # Install Bundler gem
- PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
- GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
- gem install bundler --user-install
-
- # Enter the git clone for remaining commands
- cd src
-
- # Install all needed gems.
- bundle install --path ~/.gem/
-
- # Build the new site and test it.
- rm -fr ./content/
- bundle exec rake test
- '''.stripIndent().trim()
- }
-}
diff --git a/.test-infra/jenkins/job_beam_ReleaseCandidate_Python.groovy
b/.test-infra/jenkins/job_beam_ReleaseCandidate_Python.groovy
index 22ed5790645..978f1b771a7 100644
--- a/.test-infra/jenkins/job_beam_ReleaseCandidate_Python.groovy
+++ b/.test-infra/jenkins/job_beam_ReleaseCandidate_Python.groovy
@@ -16,27 +16,17 @@
* limitations under the License.
*/
-import common_job_properties
-
job('beam_PostRelease_Python_Candidate') {
- description('Runs verification of the Python release candidate.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Python SDK Release Candidates Validation',
- 'Run Python ReleaseCandidate')
+ wrappers {
+ buildInDocker {
+ dockerfile('.test-infra/jenkins/', 'Dockerfile')
+ }
+ }
// Execute shell command to test Python SDK.
+ // ./release/src/main/groovy/run_release_candidate_python_quickstart.sh
steps {
- shell('cd ' + common_job_properties.checkoutDir +
- ' && bash
release/src/main/groovy/run_release_candidate_python_quickstart.sh' +
- ' && bash
release/src/main/groovy/run_release_candidate_python_mobile_gaming.sh')
+ shell('./gradlew :pythonPreCommit --info --continue')
}
+
}
diff --git a/.test-infra/jenkins/job_beam_Release_Gradle_NightlySnapshot.groovy
b/.test-infra/jenkins/job_beam_Release_Gradle_NightlySnapshot.groovy
deleted file mode 100644
index 9bb68c59c8a..00000000000
--- a/.test-infra/jenkins/job_beam_Release_Gradle_NightlySnapshot.groovy
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This creates the nightly snapshot build.
-// Into https://repository.apache.org/content/groups/snapshots/org/apache/beam.
-job('beam_Release_Gradle_NightlySnapshot') {
- description('Publish a nightly snapshot.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // This is a post-commit job that runs once per day, not for every push.
- common_job_properties.setPostCommit(
- delegate,
- '0 7 * * *',
- false,
- '[email protected]')
-
-
- def gradle_switches = [
- ]
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- './gradlew publish',
- 'Run Gradle Publish')
-
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks('publish')
- common_job_properties.setGradleSwitches(delegate)
- // Publish a snapshot build.
- switches("-Ppublishing")
- // Don't run tasks in parallel, currently the maven-publish/signing
plugins
- // cause build failures when run in parallel with messages like 'error
snapshotting'
- switches('--no-parallel')
- }
- }
-}
-
diff --git a/.test-infra/jenkins/my-first-project-190318.json
b/.test-infra/jenkins/my-first-project-190318.json
new file mode 100644
index 00000000000..9b8c2970b3e
--- /dev/null
+++ b/.test-infra/jenkins/my-first-project-190318.json
@@ -0,0 +1,12 @@
+{
+ "type": "service_account",
+ "project_id": "my-first-project-190318",
+ "private_key_id": "a6df5ff7016396154e83fd2ccd9a8f60208303ec",
+ "private_key": "-----BEGIN PRIVATE
KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDm6pz5uQYbDQjm\njnyMYAO8TTuuUfVhKQM6el5lNrOAHgX/gF8U++8CT9p4s3H/9ikR9zAQKqJqy4T/\nRc/LkQ9UZOQSRBjze9VnTUO1CLMHcIQmc5ewhoCBIizycxICM1QgwTa0oVKdq4pN\nO+yYuwWzvdmsBP/QCdtmGho3dMdPJ7ZfWKVimLZ301WGoUnULmegzsWacqllpM+J\nDJPXf4GCPsVARbdMME4Sw7wW9MNK8V21HWtuL+07NyNTayLROpGKyvw52jqrEOv9\nkGkjIA5tnbzDoxZ5VzoHS0CeTc3czRBHkbLWRwGGxSU3m+yHIoEQYHWmIATBOIt9\nnoaw+BeLAgMBAAECggEAGCylpGsfXYOe0+xE07lVisgXIGfKJZLSmLOo4DW60wXR\noO8ChULewcI0YACBRceygwP6zyZ3fHC0Crl3SJt2HGK3kcX5SrxDGh53oSM5AuVf\noFrnnX5nQ52dUQCE+SAAhbFUldJxwLntr6o/Wbnynd+ZQy3+zfvKMUU4S2f5Ware\nvjqu8k/9H0TF8QzEvWANocah7/5XQQlcT0y0oV5xZOYfVjxt7oH1tKjSOP6+Uq95\n5cYwklUjuqDWLbRytsVGmH9mMQSeRKeiFnxcMbAD6+pFYFPzjdNCmt5gRLWx6/pk\nl/fHLjVLFQUxeKm6BiixHtT09mLxttRAveUd84xAkQKBgQD2eLCzpXq6Jwn5ewfW\nDZJU0ZG1E9nsU2TcNxoSZEgQqq1YGnDEbcmuHNfV0AmdBeWI2kq8WvesSGdXevUz\nFArh94V9tGVVcY0s9zhhfxIMphaUwICqqgc6P3tmbzOF2GgE/tHVbqgxk4mLjkAB\n4BMzqtxDUF6eBiyO0Q2BBbW2kwKBgQDv1/nzRMmIE/ip7DsFxg7Fd6gWSSuNq6e9\nkNFph7LEVZcDsJFeqp2KaIZ9LM0Z50AfImKClEs+TtcUUr9G5Hn/lNa5uG6w3YGN\n6tP0wk6lgjHGV+7lriYU0eMQImr24ADgXIxn1YZcpK4X7pDpbPAPvKzWYnH1pThn\nmxKAcGj+KQKBgQCg3UjHF5wwX0Z9VDVbGeSGA3oMK3aybgOIT1Zu5pC6gAs/U5KC\nQf297Ala/bsohy620XKx2NWJ1n/Mcu4SdmRb84dm8hSpXcF8taZQy/Tl2IkocTQJ\nL4wkEkbiJnYER6gsinw9x0lzgg3lzeQs1SjzMfMs5Am9Ld5rIDqmGSnujQKBgDn7\n2O27H43KZj+OYOaQb1nGB9j7b5Acg70MmdQNfmaA2rWFpl9gmB2yD3GLtzof8hyU\nz4QKHfa+ULkVGDEnY2cqsI7Ny9TueDe7Ier2XtUcmLP+5j6Yz5QQ+fgYmgt9NDkJ\nptgjgww6s+wNPPNx5senMZHS0wI0I2LI1rw+cOOhAoGBAMny1ub/f7KgNaBH7F9i\nxl00Wk/gHzpU2oiejUjl8PbBKfFdJbq4g64gzvcVG96vTn9uJdLBNL82qhq0CdyQ\n9x3visG71SLQsoareuBVZURCBaeIfGKPJPgJD42lVbCCS+hw8PSz3B0ed8F/W1zC\ndy+gU7Rx6YcYLAdgH1Hwe7A0\n-----END
PRIVATE KEY-----\n",
+ "client_email":
"beam-jenkins-doc...@my-first-project-190318.iam.gserviceaccount.com",
+ "client_id": "113020059043207705938",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth";,
+ "token_uri": "https://accounts.google.com/o/oauth2/token";,
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs";,
+ "client_x509_cert_url":
"https://www.googleapis.com/robot/v1/metadata/x509/beam-jenkins-docker%40my-first-project-190318.iam.gserviceaccount.com";
+}
diff --git a/build.gradle b/build.gradle
index 772003b967b..5aefdfacbf9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -109,6 +109,10 @@ rat {
// VCF test files
"**/apache_beam/testing/data/vcf/*",
+
+ // Gcloud service account Json files
+ ".test-infra/jenkins/*.json",
+
]
// Add .gitignore excludes to the Apache Rat exclusion list. We re-create
the behavior
diff --git a/release/src/main/groovy/python_release_automation_utils.sh
b/release/src/main/groovy/python_release_automation_utils.sh
index 554f3aa5693..02533e6d18a 100644
--- a/release/src/main/groovy/python_release_automation_utils.sh
+++ b/release/src/main/groovy/python_release_automation_utils.sh
@@ -110,14 +110,14 @@ function cleanup_pubsub() {
# Python RC configurations
-VERSION=$(get_version)
+VERSION=2.4.0
CANDIDATE_URL="https://dist.apache.org/repos/dist/dev/beam/$VERSION/";
BEAM_PYTHON_SDK="apache-beam-$VERSION-python.zip"
# Cloud Configurations
-PROJECT_ID='apache-beam-testing'
-BUCKET_NAME='temp-storage-for-release-validation-tests/nightly-snapshot-validation'
-TEMP_DIR='/tmp'
+PROJECT_ID='my-first-project-190318'
+BUCKET_NAME='my-first-project-190318'
+TEMP_DIR='/temp'
DATASET='beam_postrelease_mobile_gaming'
NUM_WORKERS=1
diff --git a/release/src/main/groovy/run_in_docker.sh
b/release/src/main/groovy/run_in_docker.sh
new file mode 100644
index 00000000000..eedc9ed9966
--- /dev/null
+++ b/release/src/main/groovy/run_in_docker.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pwd
+ls
+
+echo 'I am running within a docker image'
\ No newline at end of file
diff --git a/release/src/main/groovy/run_release_candidate_python_quickstart.sh
b/release/src/main/groovy/run_release_candidate_python_quickstart.sh
index 914c2f24a31..09ae7ab61cd 100755
--- a/release/src/main/groovy/run_release_candidate_python_quickstart.sh
+++ b/release/src/main/groovy/run_release_candidate_python_quickstart.sh
@@ -91,7 +91,6 @@ pushd $TMPDIR
# 1. Download files from RC staging location
#
-wget $CANDIDATE_URL$SHA1_FILE_NAME
wget $CANDIDATE_URL$ASC_FILE_NAME
wget $CANDIDATE_URL$BEAM_PYTHON_SDK
@@ -99,19 +98,10 @@ wget $CANDIDATE_URL$BEAM_PYTHON_SDK
# 2. Verify sha1, md5 hashes and gpg signature
#
-print_separator "Checking sha1 and md5 hashes"
-hash_check=$(sha1sum -c $SHA1_FILE_NAME | head -1 |awk '{print $2}')
-if [[ "$hash_check" != "OK" ]]
-then
- echo "ERROR: The sha1 hash doesn't match."
- complete "The sha1 hash doesn't match."
- exit 1
-fi
-echo "SUCCEED: Hashes verification completed."
-wget https://dist.apache.org/repos/dist/dev/beam/KEYS
-gpg --import KEYS
-gpg --verify $ASC_FILE_NAME $BEAM_PYTHON_SDK
+#wget https://dist.apache.org/repos/dist/dev/beam/KEYS
+#gpg --import KEYS
+#gpg --verify $ASC_FILE_NAME $BEAM_PYTHON_SDK
#
@@ -121,10 +111,10 @@ gpg --verify $ASC_FILE_NAME $BEAM_PYTHON_SDK
print_separator "Creating new virtualenv and installing the SDK"
virtualenv temp_virtualenv
. temp_virtualenv/bin/activate
-gcloud_version=$(gcloud --version | head -1 | awk '{print $4}')
-if [[ "$gcloud_version" < "189" ]]; then
- update_gcloud
-fi
+#gcloud_version=$(gcloud --version | head -1 | awk '{print $4}')
+#if [[ "$gcloud_version" < "189" ]]; then
+# update_gcloud
+#fi
pip install $BEAM_PYTHON_SDK[gcp]
diff --git
a/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh
b/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh
index 589e6baffc4..4360b2ae71e 100755
---
a/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh
+++
b/sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh
@@ -16,9 +16,11 @@
# limitations under the License.
#
+echo "-----------------start--------------"
wget --no-verbose -O $SRC_FILE $URL
unzip -q $SRC_FILE
rm $SRC_FILE
ln -s $HOME/$SRC_DIR $HOME/beam
cd $HOME/beam
+echo "lalalla $@"
exec "$@"
diff --git a/sdks/java/maven-archetypes/starter/pom.xml
b/sdks/java/maven-archetypes/starter/pom.xml
index 6237b6bdf65..afca8cac615 100644
--- a/sdks/java/maven-archetypes/starter/pom.xml
+++ b/sdks/java/maven-archetypes/starter/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
- <parent>
+ <!--<parent>-->
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>
+ <!--<artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>-->
<version>2.5.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
+ <!--<relativePath>../pom.xml</relativePath>-->
+ <!--</parent>-->
<artifactId>beam-sdks-java-maven-archetypes-starter</artifactId>
<name>Apache Beam :: SDKs :: Java :: Maven Archetypes :: Starter</name>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 123860)
Time Spent: 1h 10m (was: 1h)
> Get Jenkins agents dockerized
> ------------------------------
>
> Key: BEAM-3954
> URL: https://issues.apache.org/jira/browse/BEAM-3954
> Project: Beam
> Issue Type: Sub-task
> Components: build-system
> Reporter: yifan zou
> Assignee: yifan zou
> Priority: Major
> Time Spent: 1h 10m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
