[
https://issues.apache.org/jira/browse/BEAM-3249?focusedWorklogId=90022&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90022
]
ASF GitHub Bot logged work on BEAM-3249:
----------------------------------------
Author: ASF GitHub Bot
Created on: 11/Apr/18 16:10
Start Date: 11/Apr/18 16:10
Worklog Time Spent: 10m
Work Description: lukecwik closed pull request #4992: [BEAM-3249] Disable
all Jenkins jobs that rely on Maven.
URL: https://github.com/apache/beam/pull/4992
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/build_rules.gradle b/build_rules.gradle
index 08b578ec724..1d6ceeca23d 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -31,6 +31,32 @@
println "Applying build_rules.gradle to $project.name"
+// Define the set of repositories and dependencies required to
+// fetch and enable plugins.
+buildscript {
+ repositories {
+ maven { url file(offlineRepositoryRoot) }
+
+ // To run gradle in offline mode, one must first invoke
+ // 'updateOfflineRepository' to create an offline repo
+ // inside the root project directory. See the application
+ // of the offline repo plugin within build_rules.gradle
+ // for further details.
+ if (gradle.startParameter.isOffline()) {
+ return
+ }
+
+ mavenLocal()
+ mavenCentral()
+ jcenter()
+ maven { url "https://plugins.gradle.org/m2/"; }
+ maven { url "http://repo.spring.io/plugins-release"; }
+ }
+ dependencies {
+ classpath "com.github.jengelman.gradle.plugins:shadow:2.0.1"
// Enable shading Java dependencies
+ }
+}
+
/*************************************************************************************************/
// Apply common properties/repositories and tasks to all projects.
@@ -281,11 +307,34 @@ ext.getJavaRelocatedPath = { String suffix ->
+ suffix)
}
+ext.DEFAULT_SHADOW_CLOSURE = {
+ dependencies {
+ exclude(".*")
+ include(dependency(library.java.guava))
+ }
+ relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
+ // com.google.common is too generic, need to exclude guava-testlib
+ exclude "com.google.common.collect.testing.**"
+ exclude "com.google.common.escape.testing.**"
+ exclude "com.google.common.testing.**"
+ exclude "com.google.common.util.concurrent.testing.**"
+ }
+}
+
// A class defining the set of configurable properties accepted by
applyJavaNature
class JavaNatureConfiguration {
double javaVersion = 1.8 // Controls the JDK source language and
target compatibility
boolean enableFindbugs = true // Controls whether the findbugs plugin is
enabled and configured
boolean enableShadow = true // Controls whether the shadow plugin is
enabled and configured
+ // The shadowJar / shadowTestJar tasks execute the following closure to
configure themselves.
+ // Users can compose their closure with the default closure via:
+ // DEFAULT_SHADOW_CLOSURE << {
+ // dependencies {
+ // include(...)
+ // }
+ // relocate(...)
+ // }
+ Closure shadowClosure;
}
// Configures a project with a default set of plugins that should apply to all
Java projects.
@@ -337,6 +386,10 @@ ext.applyJavaNature = {
println "applyJavaNature with " + (it ? "$it" : "default configuration") + "
for project $project.name"
// Use the implicit it parameter of the closure to handle zero argument or
one argument map calls.
JavaNatureConfiguration configuration = it ? it as JavaNatureConfiguration :
new JavaNatureConfiguration()
+ if (!configuration.shadowClosure) {
+ configuration.shadowClosure = DEFAULT_SHADOW_CLOSURE
+ }
+
apply plugin: "java"
// Configure the Java compiler source language and target compatibility
levels. Also ensure that
@@ -446,23 +499,6 @@ ext.applyJavaNature = {
if (configuration.enableShadow) {
apply plugin: 'com.github.johnrengelman.shadow'
- // Shade guava in all our dependencies.
- shadowJar {
- classifier = "shaded"
- mergeServiceFiles()
- dependencies {
- exclude(".*")
- include(dependency(library.java.guava))
- }
- relocate("com.google.common", getJavaRelocatedPath("com.google.common"))
{
- // com.google.common is too generic, need to exclude guava-testlib
- exclude "com.google.common.collect.testing.**"
- exclude "com.google.common.escape.testing.**"
- exclude "com.google.common.testing.**"
- exclude "com.google.common.util.concurrent.testing.**"
- }
- }
-
// Create a new configuration 'shadowTest' like 'shadow' for the test scope
configurations {
shadow {
@@ -475,18 +511,27 @@ ext.applyJavaNature = {
}
testCompile.extendsFrom shadowTest
}
- // Ensure that shaded classes are part of the artifact set.
- artifacts.archives shadowJar
- // TODO: Figure out how to create ShadowJar task for ShadowTestJar here
- // that is extendable within each sub-project with any additional includes.
- // This could mirror the "shadowJar" configuration block.
- // Optionally, we could also copy the shading configuration from the main
- // source set and apply it here.
- //
- // Shading the test jar has a lot less need but can still be beneficial for
- // resolving class conflicts for tests during test execution or exposing
- // test libraries for users.
+ // Always configure the shadowJar classifier and merge service files.
+ shadowJar ({
+ classifier = "shaded"
+ mergeServiceFiles()
+
+ } << configuration.shadowClosure)
+
+ // Always configure the shadowTestJar classifier and merge service files.
+ tasks.create(name: 'shadowTestJar', type:
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar, {
+ classifier = "shaded-tests"
+ from sourceSets.test.output
+ configurations = [project.configurations.testRuntime]
+ } << configuration.shadowClosure)
+
+ // Ensure that shaded jar and test-jar are part of the their own
cconfiguration artifact sets
+ // and the archives configuration set.
+ artifacts.shadow shadowJar
+ artifacts.shadowTest shadowTestJar
+ artifacts.archives shadowJar
+ artifacts.archives shadowTestJar
}
task sourcesJar(type: Jar) {
diff --git a/examples/java/build.gradle b/examples/java/build.gradle
index d9f25fb2e67..76889cafa59 100644
--- a/examples/java/build.gradle
+++ b/examples/java/build.gradle
@@ -85,34 +85,15 @@ dependencies {
dataflowStreamingRunnerPreCommit project(path:
":beam-runners-google-cloud-dataflow-java", configuration: "shadow")
directRunnerPreCommit project(path: ":beam-runners-direct-java",
configuration: "shadow")
flinkRunnerPreCommit project(path: ":beam-runners-flink_2.11",
configuration: "shadow")
+ // TODO: Make the netty version used configurable, we add netty-all
4.1.17.Final so it appears on the classpath
+ // before 4.1.8.Final defined by Apache Beam
+ sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final"
sparkRunnerPreCommit project(path: ":beam-runners-spark", configuration:
"shadow")
sparkRunnerPreCommit project(path: ":beam-sdks-java-io-hadoop-file-system",
configuration: "shadow")
sparkRunnerPreCommit library.java.spark_streaming
sparkRunnerPreCommit library.java.spark_core
}
-// Create a shaded test jar.
-task shadowTestJar(type:
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
- classifier = "shaded-tests"
- from sourceSets.test.output
- configurations = [project.configurations.testRuntime]
- dependencies {
- exclude(".*")
- include(dependency(library.java.guava))
- }
- relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
- // com.google.common is too generic, need to exclude guava-testlib
- exclude "com.google.common.collect.testing.**"
- exclude "com.google.common.escape.testing.**"
- exclude "com.google.common.testing.**"
- exclude "com.google.common.util.concurrent.testing.**"
- }
-}
-
-artifacts {
- shadowTest shadowTestJar
-}
-
/*
* Create a ${runner}PreCommit task for each runner which runs a set
* of integration tests for WordCount and WindowedWordCount.
diff --git a/runners/core-construction-java/build.gradle
b/runners/core-construction-java/build.gradle
index 40e2599ee0b..44171011ee0 100644
--- a/runners/core-construction-java/build.gradle
+++ b/runners/core-construction-java/build.gradle
@@ -52,25 +52,3 @@ dependencies {
shadowTest library.java.junit
shadowTest library.java.mockito_core
}
-
-// Create a shaded test jar.
-task shadowTestJar(type:
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
- classifier = "shaded-tests"
- from sourceSets.test.output
- configurations = [project.configurations.testRuntime]
- dependencies {
- exclude(".*")
- include(dependency(library.java.guava))
- }
- relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
- // com.google.common is too generic, need to exclude guava-testlib
- exclude "com.google.common.collect.testing.**"
- exclude "com.google.common.escape.testing.**"
- exclude "com.google.common.testing.**"
- exclude "com.google.common.util.concurrent.testing.**"
- }
-}
-
-artifacts {
- shadowTest shadowTestJar
-}
diff --git a/runners/core-java/build.gradle b/runners/core-java/build.gradle
index 129618ee271..11d606372ea 100644
--- a/runners/core-java/build.gradle
+++ b/runners/core-java/build.gradle
@@ -49,25 +49,3 @@ dependencies {
shadowTest library.java.slf4j_jdk14
shadowTest library.java.jackson_dataformat_yaml
}
-
-// Create a shaded test jar.
-task shadowTestJar(type:
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
- classifier = "shaded-tests"
- from sourceSets.test.output
- configurations = [project.configurations.testRuntime]
- dependencies {
- exclude(".*")
- include(dependency(library.java.guava))
- }
- relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
- // com.google.common is too generic, need to exclude guava-testlib
- exclude "com.google.common.collect.testing.**"
- exclude "com.google.common.escape.testing.**"
- exclude "com.google.common.testing.**"
- exclude "com.google.common.util.concurrent.testing.**"
- }
-}
-
-artifacts {
- shadowTest shadowTestJar
-}
diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle
index 5779dffbb7b..fd99721378a 100644
--- a/runners/direct-java/build.gradle
+++ b/runners/direct-java/build.gradle
@@ -19,7 +19,20 @@
import groovy.json.JsonOutput
apply from: project(":").file("build_rules.gradle")
-applyJavaNature()
+applyJavaNature(shadowClosure: DEFAULT_SHADOW_CLOSURE << {
+ dependencies {
+ include(dependency(library.java.protobuf_java))
+ include(dependency(library.java.protobuf_java_util))
+ include(project(path: ":beam-model-pipeline", configuration: "shadow"))
+ include(project(path: ":beam-runners-core-construction-java",
configuration: "shadow"))
+ include(project(path: ":beam-runners-core-java", configuration: "shadow"))
+ include(project(path: ":beam-runners-local-java-core", configuration:
"shadow"))
+ }
+ relocate "org.apache.beam.runners.core", getJavaRelocatedPath("runners.core")
+ relocate "org.apache.beam.model", getJavaRelocatedPath("model")
+ relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
+ relocate "javax.annotation", getJavaRelocatedPath("javax.annotation")
+})
description = "Apache Beam :: Runners :: Direct Java"
@@ -84,52 +97,5 @@ task needsRunnerTests(type: Test) {
}
}
-shadowJar {
- dependencies {
- include(dependency(library.java.protobuf_java))
- include(dependency(library.java.protobuf_java_util))
- include(project(path: ":beam-model-pipeline", configuration: "shadow"))
- include(project(path: ":beam-runners-core-construction-java",
configuration: "shadow"))
- include(project(path: ":beam-runners-core-java", configuration: "shadow"))
- include(project(path: ":beam-runners-local-java-core", configuration:
"shadow"))
- }
- relocate "org.apache.beam.runners.core", getJavaRelocatedPath("runners.core")
- relocate "org.apache.beam.model", getJavaRelocatedPath("model")
- relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
- relocate "javax.annotation", getJavaRelocatedPath("javax.annotation")
-}
-
-// Create a shaded test jar.
-task shadowTestJar(type:
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
- classifier = "shaded-tests"
- from sourceSets.test.output
- configurations = [project.configurations.testRuntime]
- dependencies {
- exclude(".*")
- include(dependency(library.java.guava))
- include(dependency(library.java.protobuf_java))
- include(dependency(library.java.protobuf_java_util))
- include(project(path: ":beam-model-pipeline", configuration: "shadow"))
- include(project(path: ":beam-runners-core-construction-java",
configuration: "shadow"))
- include(project(path: ":beam-runners-core-java", configuration: "shadow"))
- include(project(path: ":beam-runners-local-java-core", configuration:
"shadow"))
- }
- relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
- // com.google.common is too generic, need to exclude guava-testlib
- exclude "com.google.common.collect.testing.**"
- exclude "com.google.common.escape.testing.**"
- exclude "com.google.common.testing.**"
- exclude "com.google.common.util.concurrent.testing.**"
- }
- relocate "org.apache.beam.runners.core", getJavaRelocatedPath("runners.core")
- relocate "org.apache.beam.model", getJavaRelocatedPath("model")
- relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
- relocate "javax.annotation", getJavaRelocatedPath("javax.annotation")
-}
-
-artifacts {
- shadowTest shadowTestJar
-}
-
// Generates :beam-runners-direct-java:runQuickstartJavaDirect
createJavaQuickstartValidationTask(name: 'Direct')
diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 53d62c7669e..6634be99c36 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -94,7 +94,6 @@ dependencies {
validatesRunner project(path: project.path, configuration: "shadowTest")
validatesRunner project(path: project.path, configuration: "shadow")
validatesRunner project(path: project.path, configuration: "provided")
- validatesRunner project.sourceSets.test.output
}
configurations.testRuntimeClasspath {
diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle
index 3ac9c233f92..267d9b23626 100644
--- a/sdks/java/core/build.gradle
+++ b/sdks/java/core/build.gradle
@@ -17,7 +17,17 @@
*/
apply from: project(":").file("build_rules.gradle")
-applyJavaNature()
+applyJavaNature(shadowClosure: DEFAULT_SHADOW_CLOSURE << {
+ dependencies {
+ include(dependency(library.java.protobuf_java))
+ include(dependency(library.java.byte_buddy))
+ include(dependency("org.apache.commons:.*"))
+ }
+ relocate "com.google.thirdparty",
getJavaRelocatedPath("com.google.thirdparty")
+ relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
+ relocate "net.bytebuddy", getJavaRelocatedPath("net.bytebuddy")
+ relocate "org.apache.commons", getJavaRelocatedPath("org.apache.commons")
+})
applyAvroNature()
description = "Apache Beam :: SDKs :: Java :: Core"
@@ -75,46 +85,3 @@ dependencies {
shadowTest "com.esotericsoftware.kryo:kryo:2.21"
shadowTest project(path: ":beam-model-fn-execution", configuration: "shadow")
}
-
-// Shade dependencies.
-shadowJar {
- dependencies {
- include(dependency(library.java.protobuf_java))
- include(dependency(library.java.byte_buddy))
- include(dependency("org.apache.commons:.*"))
- }
- relocate "com.google.thirdparty",
getJavaRelocatedPath("com.google.thirdparty")
- relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
- relocate "net.bytebuddy", getJavaRelocatedPath("net.bytebuddy")
- relocate "org.apache.commons", getJavaRelocatedPath("org.apache.commons")
-}
-
-// Create a shaded test jar.
-task shadowTestJar(type:
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
- classifier = "shaded-tests"
- from sourceSets.test.output
- configurations = [project.configurations.testRuntime]
- dependencies {
- exclude(".*")
- include(dependency(library.java.guava))
- include(dependency(library.java.protobuf_java))
- include(dependency(library.java.protobuf_java_util))
- include(dependency(library.java.byte_buddy))
- include(dependency("org.apache.commons:.*"))
- }
- relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
- // com.google.common is too generic, need to exclude guava-testlib
- exclude "com.google.common.collect.testing.**"
- exclude "com.google.common.escape.testing.**"
- exclude "com.google.common.testing.**"
- exclude "com.google.common.util.concurrent.testing.**"
- }
- relocate "com.google.thirdparty",
getJavaRelocatedPath("com.google.thirdparty")
- relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
- relocate "net.bytebuddy", getJavaRelocatedPath("net.bytebuddy")
- relocate "org.apache.commons", getJavaRelocatedPath("org.apache.commons")
-}
-
-artifacts {
- shadowTest shadowTestJar
-}
diff --git a/sdks/java/io/google-cloud-platform/build.gradle
b/sdks/java/io/google-cloud-platform/build.gradle
index f9df0965f68..b9b0a93abc7 100644
--- a/sdks/java/io/google-cloud-platform/build.gradle
+++ b/sdks/java/io/google-cloud-platform/build.gradle
@@ -17,7 +17,15 @@
*/
apply from: project(":").file("build_rules.gradle")
-applyJavaNature(enableFindbugs: false)
+applyJavaNature(
+ enableFindbugs: false,
+ // Override the default shading configuration to exclude everything since
+ // Bigtable needs to expose Guava types.
+ shadowClosure: {
+ dependencies {
+ exclude(".*")
+ }
+ })
description = "Apache Beam :: SDKs :: Java :: IO :: Google Cloud Platform"
ext.summary = "IO library to read and write Google Cloud Platform systems from
Beam."
@@ -32,10 +40,10 @@ ext.summary = "IO library to read and write Google Cloud
Platform systems from B
evaluationDependsOn(":beam-sdks-java-extensions-google-cloud-platform-core")
dependencies {
- compile library.java.guava
shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
shadow project(path:
":beam-sdks-java-extensions-google-cloud-platform-core", configuration:
"shadow")
shadow project(path: ":beam-sdks-java-extensions-protobuf", configuration:
"shadow")
+ shadow library.java.guava
shadow library.java.jackson_databind
shadow library.java.grpc_core
shadow library.java.google_api_services_bigquery
@@ -77,25 +85,3 @@ dependencies {
shadowTest library.java.junit
shadowTest library.java.slf4j_jdk14
}
-
-// Create a shaded test jar.
-task shadowTestJar(type:
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
- classifier = "shaded-tests"
- from sourceSets.test.output
- configurations = [project.configurations.testRuntime]
- dependencies {
- exclude(".*")
- include(dependency(library.java.guava))
- }
- relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
- // com.google.common is too generic, need to exclude guava-testlib
- exclude "com.google.common.collect.testing.**"
- exclude "com.google.common.escape.testing.**"
- exclude "com.google.common.testing.**"
- exclude "com.google.common.util.concurrent.testing.**"
- }
-}
-
-artifacts {
- shadowTest shadowTestJar
-}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 90022)
Time Spent: 11.5h (was: 11h 20m)
> Use Gradle to build/release project
> -----------------------------------
>
> Key: BEAM-3249
> URL: https://issues.apache.org/jira/browse/BEAM-3249
> Project: Beam
> Issue Type: Improvement
> Components: build-system, testing
> Reporter: Luke Cwik
> Assignee: Luke Cwik
> Priority: Major
> Time Spent: 11.5h
> Remaining Estimate: 0h
>
> I have collected data by running several builds against master using Gradle
> and Maven without using Gradle's support for incremental builds.
> Gradle (mins)
> min: 25.04
> max: 160.14
> median: 45.78
> average: 52.19
> stdev: 30.80
> Maven (mins)
> min: 56.86
> max: 216.55
> median: 87.93
> average: 109.10
> stdev: 48.01
> I excluded a few timeouts (240 mins) that happened during the Maven build
> from its numbers but we can see conclusively that Gradle is about twice as
> fast for the build when compared to Maven when run using Jenkins.
> Original dev@ thread:
> https://lists.apache.org/thread.html/225dddcfc78f39bbb296a0d2bbef1caf37e17677c7e5573f0b6fe253@%3Cdev.beam.apache.org%3E
> The data is available here
> https://docs.google.com/spreadsheets/d/1MHVjF-xoI49_NJqEQakUgnNIQ7Qbjzu8Y1q_h3dbF1M/edit?usp=sharing
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
