[
https://issues.apache.org/jira/browse/BEAM-4496?focusedWorklogId=148429&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-148429
]
ASF GitHub Bot logged work on BEAM-4496:
----------------------------------------
Author: ASF GitHub Bot
Created on: 26/Sep/18 22:09
Start Date: 26/Sep/18 22:09
Worklog Time Spent: 10m
Work Description: swegner closed pull request #6431: [BEAM-4496] Website
publish via jenkins to beam asf-site branch
URL: https://github.com/apache/beam/pull/6431
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/CommonJobProperties.groovy
b/.test-infra/jenkins/CommonJobProperties.groovy
index 428ac0de449..40725c42c2b 100644
--- a/.test-infra/jenkins/CommonJobProperties.groovy
+++ b/.test-infra/jenkins/CommonJobProperties.groovy
@@ -63,14 +63,14 @@ class CommonJobProperties {
String branch = 'master',
int timeout = 100,
boolean allowRemotePoll = true,
- boolean localPerfTest = false) {
+ String jenkinsExecutorLabel =
'beam') {
setTopLevelJobProperties(
context,
'beam',
branch,
timeout,
allowRemotePoll,
- localPerfTest)
+ jenkinsExecutorLabel)
}
// Sets common top-level job properties. Accessed through one of the above
@@ -80,12 +80,7 @@ class CommonJobProperties {
String defaultBranch,
int defaultTimeout,
boolean allowRemotePoll = true,
- boolean localPerfTest = false) {
- def jenkinsExecutorLabel = 'beam'
- if (localPerfTest) {
- jenkinsExecutorLabel = 'beam-perf'
- }
-
+ String jenkinsExecutorLabel =
'beam') {
// GitHub project.
context.properties {
githubProjectUrl('https://github.com/apache/' + repositoryName + '/')
diff --git a/.test-infra/jenkins/PostcommitJobBuilder.groovy
b/.test-infra/jenkins/PostcommitJobBuilder.groovy
index 45b841e64c4..460555fb368 100644
--- a/.test-infra/jenkins/PostcommitJobBuilder.groovy
+++ b/.test-infra/jenkins/PostcommitJobBuilder.groovy
@@ -35,6 +35,15 @@ class PostcommitJobBuilder {
this.job = null
}
+ /**
+ * Set the job details.
+ *
+ * @param nameBase Job name for the postcommit job, a _PR suffix added if
the trigger is set.
+ * @param triggerPhrase Phrase to trigger jobs, empty to not have a trigger.
+ * @param githubUiHint Short description in the github UI.
+ * @param scope Delegate for the job.
+ * @param jobDefinition Closure for the job.
+ */
static void postCommitJob(nameBase,
triggerPhrase,
githubUiHint,
@@ -42,7 +51,9 @@ class PostcommitJobBuilder {
jobDefinition = {}) {
PostcommitJobBuilder jb = new PostcommitJobBuilder(scope, jobDefinition)
jb.defineAutoPostCommitJob(nameBase)
- jb.defineGhprbTriggeredJob(nameBase + "_PR", triggerPhrase, githubUiHint,
false)
+ if (triggerPhrase) {
+ jb.defineGhprbTriggeredJob(nameBase + "_PR", triggerPhrase,
githubUiHint, false)
+ }
}
void defineAutoPostCommitJob(name) {
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
index a2dc54c83d9..80dff3cd118 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
@@ -26,7 +26,7 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
description('Runs the Nexmark suite on the Direct runner.')
// Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, true)
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, 'beam-perf')
// Gradle goals for this job.
steps {
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
index e21f14d0968..05434afabaf 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
@@ -26,7 +26,7 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
description('Runs the Nexmark suite on the Flink runner.')
// Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, true)
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, 'beam-perf')
// Gradle goals for this job.
steps {
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
index c6690f75c97..dbf2333c312 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
@@ -26,7 +26,7 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
description('Runs the Nexmark suite on the Spark runner.')
// Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, true)
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, 'beam-perf')
// Gradle goals for this job.
steps {
diff --git a/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy
b/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy
new file mode 100644
index 00000000000..669a84abcfe
--- /dev/null
+++ b/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
+
+
+// This job builds and publishes the website into the asf-site branch of the
beam repo.
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Website_Publish', '',
+ 'Website Publish', this) {
+
+ description('Publish generated website content into asf-site branch for
hosting.')
+
+ // Set common parameters.
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 30,
true, 'git-websites')
+
+ // Gradle goals for this job.
+ steps {
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-website:clean')
+ tasks(':beam-website:publishWebsite')
+ commonJobProperties.setGradleSwitches(delegate)
+ }
+ }
+}
diff --git a/build.gradle b/build.gradle
index 00415791724..63ee9d4d044 100644
--- a/build.gradle
+++ b/build.gradle
@@ -63,6 +63,7 @@ buildscript {
classpath
"gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.3"
// Enable creating an offline repository
classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.0.13"
// Enable errorprone Java static analysis
classpath "com.github.ben-manes:gradle-versions-plugin:0.17.0"
// Enable dependency checks
+ classpath "org.ajoberstar.grgit:grgit-gradle:3.0.0-beta.1"
// Enable website git publish to asf-site branch
// Plugins which require online access should not be enabled when running
in offline mode.
if (!gradle.startParameter.isOffline()) {
diff --git a/website/Rakefile b/website/Rakefile
index 6addceafb09..40283e8ecac 100644
--- a/website/Rakefile
+++ b/website/Rakefile
@@ -3,7 +3,7 @@ require 'html-proofer'
require 'etc'
task :test do
- HTMLProofer.check_directory("./content", {
+ HTMLProofer.check_directory("./generated-content", {
:typhoeus => {
:timeout => 60,
:connecttimeout => 40 },
diff --git a/website/_config.yml b/website/_config.yml
index 5b1444acab5..4a421474a92 100644
--- a/website/_config.yml
+++ b/website/_config.yml
@@ -33,7 +33,7 @@ baseurl: ""
# input and output directory
source: src
-destination: content
+destination: generated-content
# the base hostname & protocol for your site
url: "https://beam.apache.org";
diff --git a/website/build.gradle b/website/build.gradle
index 7f5892600c2..7d22492e988 100644
--- a/website/build.gradle
+++ b/website/build.gradle
@@ -19,9 +19,15 @@
// Define common lifecycle tasks and artifact types
apply plugin: "base"
+// Publish website to asf-git branch.
+apply plugin: 'org.ajoberstar.grgit'
+
def dockerImageTag = 'beam-website'
def dockerWorkDir = "/repo"
-def buildDir = "$project.rootDir/build/website"
+def buildDir = "${project.rootDir}/build/website"
+def buildContentDir = "${project.rootDir}/build/website/generated-content"
+def repoContentDir = "${project.rootDir}/website/generated-content"
+def commitedChanges = false
task buildDockerImage(type: Exec) {
inputs.files 'Gemfile', 'Gemfile.lock'
@@ -70,7 +76,7 @@ task buildWebsite(type: Exec) {
inputs.files 'Gemfile.lock', '_config.yml'
inputs.dir 'src'
outputs.dir "$buildDir/.sass-cache"
- outputs.dir "$buildDir/content"
+ outputs.dir buildContentDir
commandLine 'docker', 'exec',
"${->startDockerContainer.containerId()}", '/bin/bash', '-c',
"""cd $dockerWorkDir/build/website && \
@@ -87,20 +93,97 @@ task testWebsite(type: Exec) {
finalizedBy stopAndRemoveDockerContainer
inputs.files "$buildDir/Rakefile"
- inputs.dir "$buildDir/content"
+ inputs.dir buildContentDir
commandLine 'docker', 'exec',
"${->startDockerContainer.containerId()}", '/bin/bash', '-c',
"""cd $dockerWorkDir/build/website && \
bundle exec rake test"""
}
-/**
- * Removed testWebsite from the Beam build dependency because it is broken and
obsolete.
- * See https://issues.apache.org/jira/browse/BEAM-5367 for more info.
- */
-
-// check.dependsOn testWebsite
+check.dependsOn testWebsite
task preCommit {
dependsOn testWebsite
}
+
+// Creates a new commit on asf-site branch
+task commitWebsite << {
+ assert file("${buildContentDir}/index.html").exists()
+ // Generated javadoc and pydoc content is not built or stored in this repo.
+ assert !file("${buildContentDir}/documentation/sdks/javadoc").exists()
+ assert !file("${buildContentDir}/documentation/sdks/pydoc").exists()
+
+ def git = grgit.open()
+ // get the latest commit on master
+ def latestCommit = grgit.log(maxCommits: 1)[0].abbreviatedId
+ git.checkout(branch: 'asf-site')
+
+ // Delete the previous content.
+ git.remove(patterns: [ 'website/generated-content' ])
+ assert !file("${repoContentDir}/index.html").exists()
+ delete repoContentDir
+
+ // Copy the built content and add it.
+ copy {
+ from buildContentDir
+ into repoContentDir
+ }
+ assert file("${repoContentDir}/index.html").exists()
+ git.add(patterns: ['website/generated-content'])
+
+ def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss')
+ String message = "Publishing website ${currentDate} at commit
${latestCommit}"
+ if (git.status().isClean()) {
+ println 'No changes to commit'
+ } else {
+ println 'Creating commit for changes'
+ commitedChanges = true
+ git.commit(message: message)
+ }
+}
+
+
+/*
+ * Pushes the asf-site branch commits.
+ *
+ * This requires write access to the asf-site branch and can be run on
+ * Jenkins executors with the git-websites label.
+ *
+ * For more details on publishing, see:
+ * https://www.apache.org/dev/project-site.html
+ *
https://github.com/apache/infrastructure-puppet/blob/deployment/modules/gitwcsub/files/config/gitwcsub.cfg
+ *
+ * You can test this locally with a forked repository by manually adding the
+ * website-publish remote pointing to your forked repository, for example:
+ * git remote add website-publish [email protected]:${GITUSER}/beam.git
+ * because the remote is only added if it doesn't exist. The remote needs
+ * to be added before every execution of the publishing.
+ */
+task publishWebsite << {
+ def git = grgit.open()
+ git.checkout(branch: 'asf-site')
+ if (!commitedChanges) {
+ println 'No changes to push'
+ return
+ }
+ if (!git.remote.list().find { it.name == 'website-publish' }) {
+ println "Adding website-publish remote"
+ // Cannot authenticate to the default github uri, so specify gitbox.
+ git.remote.add(name: 'website-publish',
+ url: 'https://gitbox.apache.org/repos/asf/beam.git',
+ pushRefSpecs: ['refs/heads/asf-site'])
+ }
+ // Because git.push() fails to authenticate, run git push directly.
+ exec {
+ executable 'sh'
+ args '-c', "git push website-publish asf-site"
+ }
+ // Remove the remote. grgit does not have a remote.remove method.
+ exec {
+ executable 'sh'
+ args '-c', "git remote remove website-publish"
+ }
+}
+
+commitWebsite.dependsOn testWebsite
+publishWebsite.dependsOn commitWebsite
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 148429)
Time Spent: 19h 20m (was: 19h 10m)
> Create Jenkins job to push generated HTML to asf-site branch
> ------------------------------------------------------------
>
> Key: BEAM-4496
> URL: https://issues.apache.org/jira/browse/BEAM-4496
> Project: Beam
> Issue Type: Sub-task
> Components: build-system, website
> Reporter: Scott Wegner
> Assignee: Alan Myrvold
> Priority: Major
> Labels: beam-site-automation-reliability
> Time Spent: 19h 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
