[
https://issues.apache.org/jira/browse/BEAM-4495?focusedWorklogId=140194&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-140194
]
ASF GitHub Bot logged work on BEAM-4495:
----------------------------------------
Author: ASF GitHub Bot
Created on: 31/Aug/18 17:33
Start Date: 31/Aug/18 17:33
Worklog Time Spent: 10m
Work Description: pabloem closed pull request #6282: [BEAM-4495] Website
pre-commit job
URL: https://github.com/apache/beam/pull/6282
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/WebsiteShared.groovy
b/.test-infra/jenkins/WebsiteShared.groovy
index ff2b733b5bb..986bc757a09 100755
--- a/.test-infra/jenkins/WebsiteShared.groovy
+++ b/.test-infra/jenkins/WebsiteShared.groovy
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+// TODO(BEAM-4505): Remove this file once apache/beam-site jobs stop using it.
@groovy.transform.Field static final String install_ruby_and_gems_bash = '''
maxKeyFetchAttempts=5
diff --git a/.test-infra/jenkins/job_PreCommit_Website.groovy
b/.test-infra/jenkins/job_PreCommit_Website.groovy
new file mode 100644
index 00000000000..e5f1f4c020c
--- /dev/null
+++ b/.test-infra/jenkins/job_PreCommit_Website.groovy
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PrecommitJobBuilder
+
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+ scope: this,
+ nameBase: 'Website',
+ gradleTask: ':websitePreCommit',
+ triggerPathPatterns: ['^website/.*$']
+)
+builder.build()
+
diff --git a/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy
b/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy
index d65fdb89fff..4fffd364724 100644
--- a/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy
@@ -19,6 +19,9 @@
import CommonJobProperties as commonJobProperties
import WebsiteShared as websiteShared
+// TODO(BEAM-4505): This job is for the apache/beam-site repository and
+// should be removed once website sources are migrated to apache/beam.
+
// Defines a job.
job('beam_PreCommit_Website_Merge') {
description('Runs website tests for mergebot.')
diff --git a/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy
b/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy
index 4203db5e2d1..ef683ac957a 100644
--- a/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy
@@ -19,6 +19,9 @@
import CommonJobProperties as commonJobProperties
import WebsiteShared as websiteShared
+// TODO(BEAM-4505): This job is for the apache/beam-site repository and
+// should be removed once website sources are migrated to apache/beam.
+
// Defines a job.
job('beam_PreCommit_Website_Stage') {
description('Stages the pull requests proposed for the Apache Beam ' +
diff --git a/.test-infra/jenkins/job_PreCommit_Website_Test.groovy
b/.test-infra/jenkins/job_PreCommit_Website_Test.groovy
index 5f639fedabd..bc7e2724dad 100644
--- a/.test-infra/jenkins/job_PreCommit_Website_Test.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Website_Test.groovy
@@ -19,6 +19,9 @@
import CommonJobProperties as commonJobProperties
import WebsiteShared as websiteShared
+// TODO(BEAM-4505): This job is for the apache/beam-site repository and
+// should be removed once website sources are migrated to apache/beam.
+
// Defines a job.
job('beam_PreCommit_Website_Test') {
description('Runs tests on the pull requests proposed for the Apache Beam ' +
diff --git a/build.gradle b/build.gradle
index f79e56e1564..e4b4f6e5095 100644
--- a/build.gradle
+++ b/build.gradle
@@ -141,8 +141,6 @@ rat {
// Ruby build files
"**/Gemfile.lock",
"**/Rakefile",
- "**/_config.yml",
- "**/_config_test.yml",
"**/.htaccess",
"website/src/_sass/_bootstrap.scss",
"website/src/_sass/bootstrap/**/*",
@@ -239,6 +237,12 @@ task pythonPostCommit() {
dependsOn ":beam-sdks-python:postCommit"
}
+task websitePreCommit() {
+ dependsOn ":rat"
+ dependsOn ":beam-website:preCommit"
+}
+
+
task runBeamDependencyCheck() {
dependsOn ":dependencyUpdates"
dependsOn ":beam-sdks-python:dependencyUpdates"
diff --git a/settings.gradle b/settings.gradle
index a606b17cf80..89e68951303 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -172,3 +172,5 @@ include "beam-sdks-python-container"
project(":beam-sdks-python-container").dir = file("sdks/python/container")
include "beam-vendor-sdks-java-extensions-protobuf"
project(":beam-vendor-sdks-java-extensions-protobuf").dir =
file("vendor/sdks-java-extensions-protobuf")
+include "beam-website"
+project(":beam-website").dir = file("website")
diff --git a/website/.gitignore b/website/.gitignore
index 818a7304531..072e28bc055 100644
--- a/website/.gitignore
+++ b/website/.gitignore
@@ -3,7 +3,7 @@ _site
.jekyll-metadata
vendor/
.bundle/
-.testcontent/
+content/
# Ignore IntelliJ files.
.idea/
diff --git a/website/Dockerfile b/website/Dockerfile
new file mode 100644
index 00000000000..8a597437075
--- /dev/null
+++ b/website/Dockerfile
@@ -0,0 +1,33 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+# This image contains Ruby and dependencies required to build and test the Beam
+# website. It is used by tasks in build.gradle.
+
+FROM ruby:2.5
+
+WORKDIR /ruby
+RUN gem install bundler
+# Update buildDockerImage's inputs.files if you change this list.
+ADD Gemfile Gemfile.lock /ruby/
+RUN bundle install --deployment --path $GEM_HOME
+
+# Required for website testing using HTMLProofer.
+ENV LC_ALL C.UTF-8
+
+CMD sleep 3600
diff --git a/website/Rakefile b/website/Rakefile
index c2e7069c667..6addceafb09 100644
--- a/website/Rakefile
+++ b/website/Rakefile
@@ -3,9 +3,7 @@ require 'html-proofer'
require 'etc'
task :test do
- FileUtils.rm_rf('./.testcontent')
- sh "bundle exec jekyll build --config _config.yml,_config_test.yml"
- HTMLProofer.check_directory("./.testcontent", {
+ HTMLProofer.check_directory("./content", {
:typhoeus => {
:timeout => 60,
:connecttimeout => 40 },
@@ -13,6 +11,10 @@ task :test do
:check_html => true,
:file_ignore => [/javadoc/, /v2/, /pydoc/],
:url_ignore => [
+ # Javadocs and Pydocs are only available on asf-site branch
+ /documentation\/sdks\/javadoc/,
+ /documentation\/sdks\/pydoc/,
+
/jstorm.io/,
/datatorrent.com/,
/ai.google/ # https://issues.apache.org/jira/browse/INFRA-16527
diff --git a/website/_config_test.yml b/website/_config_test.yml
deleted file mode 100644
index 551099de89b..00000000000
--- a/website/_config_test.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# Welcome to Jekyll!
-
-# This config file contains options that override the default config for
testing.
-
-destination: .testcontent
diff --git a/website/build.gradle b/website/build.gradle
new file mode 100644
index 00000000000..0a64e73ed50
--- /dev/null
+++ b/website/build.gradle
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Define common lifecycle tasks and artifact types
+apply plugin: "base"
+
+def dockerImageTag = 'beam-website'
+def dockerWorkDir = "/repo"
+def buildDir = "$project.rootDir/build/website"
+
+task buildDockerImage(type: Exec) {
+ inputs.files 'Gemfile', 'Gemfile.lock'
+ commandLine 'docker', 'build', '-t', dockerImageTag, '.'
+}
+
+task createDockerContainer(type: Exec) {
+ dependsOn buildDockerImage
+ standardOutput = new ByteArrayOutputStream()
+ ext.containerId = {
+ return standardOutput.toString().trim()
+ }
+ commandLine '/bin/bash', '-c',
+ "docker create -v $project.rootDir:$dockerWorkDir -u \$(id -u):\$(id -g)
$dockerImageTag"
+}
+
+task startDockerContainer(type: Exec) {
+ dependsOn createDockerContainer
+ ext.containerId = {
+ return createDockerContainer.containerId()
+ }
+ commandLine 'docker', 'start',
+ "${->createDockerContainer.containerId()}" // Lazily evaluate containerId.
+}
+
+task stopAndRemoveDockerContainer(type: Exec) {
+ commandLine 'docker', 'rm', '-f', "${->createDockerContainer.containerId()}"
+}
+
+task setupBuildDir(type: Copy) {
+ from('.') {
+ include 'Gemfile*'
+ include 'Rakefile'
+ }
+ into buildDir
+}
+
+task cleanWebsite(type: Delete) {
+ delete buildDir
+}
+clean.dependsOn cleanWebsite
+
+task buildWebsite(type: Exec) {
+ dependsOn startDockerContainer, setupBuildDir
+ finalizedBy stopAndRemoveDockerContainer
+ inputs.files 'Gemfile.lock', '_config.yml'
+ inputs.dir 'src'
+ outputs.dir "$buildDir/.sass-cache"
+ outputs.dir "$buildDir/content"
+ commandLine 'docker', 'exec',
+ "${->startDockerContainer.containerId()}", '/bin/bash', '-c',
+ """cd $dockerWorkDir/build/website && \
+ bundle exec jekyll build \
+ --config $dockerWorkDir/website/_config.yml \
+ --incremental \
+ --source $dockerWorkDir/website/src
+ """
+}
+build.dependsOn buildWebsite
+
+task testWebsite(type: Exec) {
+ dependsOn startDockerContainer, buildWebsite
+ finalizedBy stopAndRemoveDockerContainer
+
+ inputs.files "$buildDir/Rakefile"
+ inputs.dir "$buildDir/content"
+ commandLine 'docker', 'exec',
+ "${->startDockerContainer.containerId()}", '/bin/bash', '-c',
+ """cd $dockerWorkDir/build/website && \
+ bundle exec rake test"""
+}
+
+check.dependsOn testWebsite
+
+task preCommit {
+ dependsOn testWebsite
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 140194)
Time Spent: 8.5h (was: 8h 20m)
> Create website pre-commits for apache/beam repository
> -----------------------------------------------------
>
> Key: BEAM-4495
> URL: https://issues.apache.org/jira/browse/BEAM-4495
> Project: Beam
> Issue Type: Sub-task
> Components: testing, website
> Reporter: Scott Wegner
> Assignee: Udi Meiri
> Priority: Major
> Labels: beam-site-automation-reliability
> Time Spent: 8.5h
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
