[
https://issues.apache.org/jira/browse/BEAM-4496?focusedWorklogId=147929&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-147929
]
ASF GitHub Bot logged work on BEAM-4496:
----------------------------------------
Author: ASF GitHub Bot
Created on: 26/Sep/18 03:58
Start Date: 26/Sep/18 03:58
Worklog Time Spent: 10m
Work Description: alanmyrvold closed pull request #6431: [BEAM-4496]
Website publish via jenkins to beam asf-site branch
URL: https://github.com/apache/beam/pull/6431
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/CommonJobProperties.groovy
b/.test-infra/jenkins/CommonJobProperties.groovy
index 428ac0de449..40725c42c2b 100644
--- a/.test-infra/jenkins/CommonJobProperties.groovy
+++ b/.test-infra/jenkins/CommonJobProperties.groovy
@@ -63,14 +63,14 @@ class CommonJobProperties {
String branch = 'master',
int timeout = 100,
boolean allowRemotePoll = true,
- boolean localPerfTest = false) {
+ String jenkinsExecutorLabel =
'beam') {
setTopLevelJobProperties(
context,
'beam',
branch,
timeout,
allowRemotePoll,
- localPerfTest)
+ jenkinsExecutorLabel)
}
// Sets common top-level job properties. Accessed through one of the above
@@ -80,12 +80,7 @@ class CommonJobProperties {
String defaultBranch,
int defaultTimeout,
boolean allowRemotePoll = true,
- boolean localPerfTest = false) {
- def jenkinsExecutorLabel = 'beam'
- if (localPerfTest) {
- jenkinsExecutorLabel = 'beam-perf'
- }
-
+ String jenkinsExecutorLabel =
'beam') {
// GitHub project.
context.properties {
githubProjectUrl('https://github.com/apache/' + repositoryName + '/')
diff --git a/.test-infra/jenkins/PostcommitJobBuilder.groovy
b/.test-infra/jenkins/PostcommitJobBuilder.groovy
index 45b841e64c4..394c8200c45 100644
--- a/.test-infra/jenkins/PostcommitJobBuilder.groovy
+++ b/.test-infra/jenkins/PostcommitJobBuilder.groovy
@@ -42,7 +42,9 @@ class PostcommitJobBuilder {
jobDefinition = {}) {
PostcommitJobBuilder jb = new PostcommitJobBuilder(scope, jobDefinition)
jb.defineAutoPostCommitJob(nameBase)
- jb.defineGhprbTriggeredJob(nameBase + "_PR", triggerPhrase, githubUiHint,
false)
+ if (triggerPhrase) {
+ jb.defineGhprbTriggeredJob(nameBase + "_PR", triggerPhrase,
githubUiHint, false)
+ }
}
void defineAutoPostCommitJob(name) {
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
index a2dc54c83d9..80dff3cd118 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
@@ -26,7 +26,7 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
description('Runs the Nexmark suite on the Direct runner.')
// Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, true)
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, 'beam-perf')
// Gradle goals for this job.
steps {
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
index e21f14d0968..05434afabaf 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
@@ -26,7 +26,7 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
description('Runs the Nexmark suite on the Flink runner.')
// Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, true)
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, 'beam-perf')
// Gradle goals for this job.
steps {
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
index c6690f75c97..dbf2333c312 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
@@ -26,7 +26,7 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
description('Runs the Nexmark suite on the Spark runner.')
// Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, true)
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240,
true, 'beam-perf')
// Gradle goals for this job.
steps {
diff --git a/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy
b/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy
new file mode 100644
index 00000000000..041619a2e43
--- /dev/null
+++ b/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
+//import NoPhraseTriggeringPostCommitBuilder
+
+
+// This job builds and publishes the website into the asf-site branch of the
beam repo.
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Website_Publish', 'Run
Website Publish',
+ 'Website Publish', this) {
+
+ description('Publish generated website content into asf-site branch for
hosting.')
+
+ // Set common parameters.
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 30,
true, 'git-websites')
+
+ // Gradle goals for this job.
+ steps {
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-website:clean')
+ tasks(':beam-website:publishWebsite')
+ commonJobProperties.setGradleSwitches(delegate)
+ }
+ }
+}
diff --git a/build.gradle b/build.gradle
index 00415791724..63ee9d4d044 100644
--- a/build.gradle
+++ b/build.gradle
@@ -63,6 +63,7 @@ buildscript {
classpath
"gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.3"
// Enable creating an offline repository
classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.0.13"
// Enable errorprone Java static analysis
classpath "com.github.ben-manes:gradle-versions-plugin:0.17.0"
// Enable dependency checks
+ classpath "org.ajoberstar.grgit:grgit-gradle:3.0.0-beta.1"
// Enable website git publish to asf-site branch
// Plugins which require online access should not be enabled when running
in offline mode.
if (!gradle.startParameter.isOffline()) {
diff --git a/content b/content
new file mode 120000
index 00000000000..6cb4abeb973
--- /dev/null
+++ b/content
@@ -0,0 +1 @@
+website/generated-content
\ No newline at end of file
diff --git a/website/Rakefile b/website/Rakefile
index 6addceafb09..40283e8ecac 100644
--- a/website/Rakefile
+++ b/website/Rakefile
@@ -3,7 +3,7 @@ require 'html-proofer'
require 'etc'
task :test do
- HTMLProofer.check_directory("./content", {
+ HTMLProofer.check_directory("./generated-content", {
:typhoeus => {
:timeout => 60,
:connecttimeout => 40 },
diff --git a/website/_config.yml b/website/_config.yml
index 5b1444acab5..4a421474a92 100644
--- a/website/_config.yml
+++ b/website/_config.yml
@@ -33,7 +33,7 @@ baseurl: ""
# input and output directory
source: src
-destination: content
+destination: generated-content
# the base hostname & protocol for your site
url: "https://beam.apache.org";
diff --git a/website/build.gradle b/website/build.gradle
index 7f5892600c2..f120d52ffaf 100644
--- a/website/build.gradle
+++ b/website/build.gradle
@@ -19,9 +19,14 @@
// Define common lifecycle tasks and artifact types
apply plugin: "base"
+// Publish website to asf-git branch.
+apply plugin: 'org.ajoberstar.grgit'
+
def dockerImageTag = 'beam-website'
def dockerWorkDir = "/repo"
-def buildDir = "$project.rootDir/build/website"
+def buildDir = "${project.rootDir}/build/website"
+def buildContentDir = "${project.rootDir}/build/website/generated-content"
+def repoContentDir = "${project.rootDir}/website/generated-content"
task buildDockerImage(type: Exec) {
inputs.files 'Gemfile', 'Gemfile.lock'
@@ -70,7 +75,7 @@ task buildWebsite(type: Exec) {
inputs.files 'Gemfile.lock', '_config.yml'
inputs.dir 'src'
outputs.dir "$buildDir/.sass-cache"
- outputs.dir "$buildDir/content"
+ outputs.dir buildContentDir
commandLine 'docker', 'exec',
"${->startDockerContainer.containerId()}", '/bin/bash', '-c',
"""cd $dockerWorkDir/build/website && \
@@ -87,20 +92,63 @@ task testWebsite(type: Exec) {
finalizedBy stopAndRemoveDockerContainer
inputs.files "$buildDir/Rakefile"
- inputs.dir "$buildDir/content"
+ inputs.dir buildContentDir
commandLine 'docker', 'exec',
"${->startDockerContainer.containerId()}", '/bin/bash', '-c',
"""cd $dockerWorkDir/build/website && \
bundle exec rake test"""
}
-/**
- * Removed testWebsite from the Beam build dependency because it is broken and
obsolete.
- * See https://issues.apache.org/jira/browse/BEAM-5367 for more info.
- */
-
-// check.dependsOn testWebsite
+check.dependsOn testWebsite
task preCommit {
dependsOn testWebsite
}
+
+// Creates a new commit on asf-site branch
+task commitWebsite << {
+ assert file("${buildContentDir}/index.html").exists()
+ // Generated javadoc and pydoc content is noa built or stored in this repo.
+ assert !file("${buildContentDir}/documentation/sdks/javadoc").exists()
+ assert !file("${buildContentDir}/documentation/sdks/pydoc").exists()
+
+ def git = grgit.open()
+ git.checkout(branch: 'asf-site')
+
+ // Delete the previous content.
+ git.remove(patterns: [ 'website/generated-content' ])
+ assert !file("${repoContentDir}/index.html").exists()
+ delete repoContentDir
+
+ // Copy the built content it and add it.
+ copy {
+ from buildContentDir
+ into repoContentDir
+ }
+ assert file("${repoContentDir}/index.html").exists()
+ git.add(patterns: ['website/generated-content'])
+
+ def latestCommit = grgit.log(maxCommits: 1)[0].abbreviatedId
+ def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss')
+ String message = "Publishing website ${currentDate} at commit
${latestCommit}"
+ git.commit(message: message)
+}
+
+// Pushes the asf-site branch commits.
+task publishWebsite << {
+ def git = grgit.open()
+ git.checkout(branch: 'asf-site')
+ // Cannot authenticate to the default github uri, so specify gitbox.
+ exec {
+ executable 'sh'
+ args '-c', "git remote set-url origin
https://gitbox.apache.org/repos/asf/beam.git";
+ }
+ // git.push() fails to authenticate, so run git push directly.
+ exec {
+ executable 'sh'
+ args '-c', "git push origin asf-site"
+ }
+}
+
+commitWebsite.dependsOn testWebsite
+publishWebsite.dependsOn commitWebsite
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 147929)
Time Spent: 16h (was: 15h 50m)
> Create Jenkins job to push generated HTML to asf-site branch
> ------------------------------------------------------------
>
> Key: BEAM-4496
> URL: https://issues.apache.org/jira/browse/BEAM-4496
> Project: Beam
> Issue Type: Sub-task
> Components: build-system, website
> Reporter: Scott Wegner
> Assignee: Alan Myrvold
> Priority: Major
> Labels: beam-site-automation-reliability
> Time Spent: 16h
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
