[ 
https://issues.apache.org/jira/browse/BEAM-3249?focusedWorklogId=90020&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90020
 ]

ASF GitHub Bot logged work on BEAM-3249:
----------------------------------------

                Author: ASF GitHub Bot
            Created on: 11/Apr/18 16:09
            Start Date: 11/Apr/18 16:09
    Worklog Time Spent: 10m 
      Work Description: lukecwik closed pull request #5087: [BEAM-3249, 
BEAM-4014] Automatically propagate shading configuration to test-jar
URL: https://github.com/apache/beam/pull/5087
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/build_rules.gradle b/build_rules.gradle
index 08b578ec724..1d6ceeca23d 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -31,6 +31,32 @@
 
 println "Applying build_rules.gradle to $project.name"
 
+// Define the set of repositories and dependencies required to
+// fetch and enable plugins.
+buildscript {
+  repositories {
+    maven { url file(offlineRepositoryRoot) }
+
+    // To run gradle in offline mode, one must first invoke
+    // 'updateOfflineRepository' to create an offline repo
+    // inside the root project directory. See the application
+    // of the offline repo plugin within build_rules.gradle
+    // for further details.
+    if (gradle.startParameter.isOffline()) {
+      return
+    }
+
+    mavenLocal()
+    mavenCentral()
+    jcenter()
+    maven { url "https://plugins.gradle.org/m2/"; }
+    maven { url "http://repo.spring.io/plugins-release"; }
+  }
+  dependencies {
+    classpath "com.github.jengelman.gradle.plugins:shadow:2.0.1"               
                         // Enable shading Java dependencies
+  }
+}
+
 
/*************************************************************************************************/
 // Apply common properties/repositories and tasks to all projects.
 
@@ -281,11 +307,34 @@ ext.getJavaRelocatedPath = { String suffix ->
           + suffix)
 }
 
+ext.DEFAULT_SHADOW_CLOSURE = {
+  dependencies {
+    exclude(".*")
+    include(dependency(library.java.guava))
+  }
+  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
+    // com.google.common is too generic, need to exclude guava-testlib
+    exclude "com.google.common.collect.testing.**"
+    exclude "com.google.common.escape.testing.**"
+    exclude "com.google.common.testing.**"
+    exclude "com.google.common.util.concurrent.testing.**"
+  }
+}
+
 // A class defining the set of configurable properties accepted by 
applyJavaNature
 class JavaNatureConfiguration {
   double javaVersion = 1.8        // Controls the JDK source language and 
target compatibility
   boolean enableFindbugs = true   // Controls whether the findbugs plugin is 
enabled and configured
   boolean enableShadow = true     // Controls whether the shadow plugin is 
enabled and configured
+  // The shadowJar / shadowTestJar tasks execute the following closure to 
configure themselves.
+  // Users can compose their closure with the default closure via:
+  // DEFAULT_SHADOW_CLOSURE << {
+  //    dependencies {
+  //      include(...)
+  //    }
+  //    relocate(...)
+  // }
+  Closure shadowClosure;
 }
 
 // Configures a project with a default set of plugins that should apply to all 
Java projects.
@@ -337,6 +386,10 @@ ext.applyJavaNature = {
   println "applyJavaNature with " + (it ? "$it" : "default configuration") + " 
for project $project.name"
   // Use the implicit it parameter of the closure to handle zero argument or 
one argument map calls.
   JavaNatureConfiguration configuration = it ? it as JavaNatureConfiguration : 
new JavaNatureConfiguration()
+  if (!configuration.shadowClosure) {
+    configuration.shadowClosure = DEFAULT_SHADOW_CLOSURE
+  }
+
   apply plugin: "java"
 
   // Configure the Java compiler source language and target compatibility 
levels. Also ensure that
@@ -446,23 +499,6 @@ ext.applyJavaNature = {
   if (configuration.enableShadow) {
     apply plugin: 'com.github.johnrengelman.shadow'
 
-    // Shade guava in all our dependencies.
-    shadowJar {
-      classifier = "shaded"
-      mergeServiceFiles()
-      dependencies {
-        exclude(".*")
-        include(dependency(library.java.guava))
-      }
-      relocate("com.google.common", getJavaRelocatedPath("com.google.common")) 
{
-        // com.google.common is too generic, need to exclude guava-testlib
-        exclude "com.google.common.collect.testing.**"
-        exclude "com.google.common.escape.testing.**"
-        exclude "com.google.common.testing.**"
-        exclude "com.google.common.util.concurrent.testing.**"
-      }
-    }
-
     // Create a new configuration 'shadowTest' like 'shadow' for the test scope
     configurations {
       shadow {
@@ -475,18 +511,27 @@ ext.applyJavaNature = {
       }
       testCompile.extendsFrom shadowTest
     }
-    // Ensure that shaded classes are part of the artifact set.
-    artifacts.archives shadowJar
 
-    // TODO: Figure out how to create ShadowJar task for ShadowTestJar here
-    // that is extendable within each sub-project with any additional includes.
-    // This could mirror the "shadowJar" configuration block.
-    // Optionally, we could also copy the shading configuration from the main
-    // source set and apply it here.
-    //
-    // Shading the test jar has a lot less need but can still be beneficial for
-    // resolving class conflicts for tests during test execution or exposing
-    // test libraries for users.
+    // Always configure the shadowJar classifier and merge service files.
+    shadowJar ({
+      classifier = "shaded"
+      mergeServiceFiles()
+
+    } << configuration.shadowClosure)
+
+    // Always configure the shadowTestJar classifier and merge service files.
+    tasks.create(name: 'shadowTestJar', type: 
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar, {
+      classifier = "shaded-tests"
+      from sourceSets.test.output
+      configurations = [project.configurations.testRuntime]
+    } << configuration.shadowClosure)
+
+    // Ensure that shaded jar and test-jar are part of the their own 
cconfiguration artifact sets
+    // and the archives configuration set.
+    artifacts.shadow shadowJar
+    artifacts.shadowTest shadowTestJar
+    artifacts.archives shadowJar
+    artifacts.archives shadowTestJar
   }
 
   task sourcesJar(type: Jar) {
diff --git a/examples/java/build.gradle b/examples/java/build.gradle
index d9f25fb2e67..76889cafa59 100644
--- a/examples/java/build.gradle
+++ b/examples/java/build.gradle
@@ -85,34 +85,15 @@ dependencies {
   dataflowStreamingRunnerPreCommit project(path: 
":beam-runners-google-cloud-dataflow-java", configuration: "shadow")
   directRunnerPreCommit project(path: ":beam-runners-direct-java", 
configuration: "shadow")
   flinkRunnerPreCommit project(path: ":beam-runners-flink_2.11", 
configuration: "shadow")
+  // TODO: Make the netty version used configurable, we add netty-all 
4.1.17.Final so it appears on the classpath
+  // before 4.1.8.Final defined by Apache Beam
+  sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final"
   sparkRunnerPreCommit project(path: ":beam-runners-spark", configuration: 
"shadow")
   sparkRunnerPreCommit project(path: ":beam-sdks-java-io-hadoop-file-system", 
configuration: "shadow")
   sparkRunnerPreCommit library.java.spark_streaming
   sparkRunnerPreCommit library.java.spark_core
 }
 
-// Create a shaded test jar.
-task shadowTestJar(type: 
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
-  classifier = "shaded-tests"
-  from sourceSets.test.output
-  configurations = [project.configurations.testRuntime]
-  dependencies {
-    exclude(".*")
-    include(dependency(library.java.guava))
-  }
-  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
-    // com.google.common is too generic, need to exclude guava-testlib
-    exclude "com.google.common.collect.testing.**"
-    exclude "com.google.common.escape.testing.**"
-    exclude "com.google.common.testing.**"
-    exclude "com.google.common.util.concurrent.testing.**"
-  }
-}
-
-artifacts {
-  shadowTest shadowTestJar
-}
-
 /*
  * Create a ${runner}PreCommit task for each runner which runs a set
  * of integration tests for WordCount and WindowedWordCount.
diff --git a/runners/core-construction-java/build.gradle 
b/runners/core-construction-java/build.gradle
index 40e2599ee0b..44171011ee0 100644
--- a/runners/core-construction-java/build.gradle
+++ b/runners/core-construction-java/build.gradle
@@ -52,25 +52,3 @@ dependencies {
   shadowTest library.java.junit
   shadowTest library.java.mockito_core
 }
-
-// Create a shaded test jar.
-task shadowTestJar(type: 
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
-  classifier = "shaded-tests"
-  from sourceSets.test.output
-  configurations = [project.configurations.testRuntime]
-  dependencies {
-    exclude(".*")
-    include(dependency(library.java.guava))
-  }
-  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
-    // com.google.common is too generic, need to exclude guava-testlib
-    exclude "com.google.common.collect.testing.**"
-    exclude "com.google.common.escape.testing.**"
-    exclude "com.google.common.testing.**"
-    exclude "com.google.common.util.concurrent.testing.**"
-  }
-}
-
-artifacts {
-  shadowTest shadowTestJar
-}
diff --git a/runners/core-java/build.gradle b/runners/core-java/build.gradle
index 129618ee271..11d606372ea 100644
--- a/runners/core-java/build.gradle
+++ b/runners/core-java/build.gradle
@@ -49,25 +49,3 @@ dependencies {
   shadowTest library.java.slf4j_jdk14
   shadowTest library.java.jackson_dataformat_yaml
 }
-
-// Create a shaded test jar.
-task shadowTestJar(type: 
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
-  classifier = "shaded-tests"
-  from sourceSets.test.output
-  configurations = [project.configurations.testRuntime]
-  dependencies {
-    exclude(".*")
-    include(dependency(library.java.guava))
-  }
-  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
-    // com.google.common is too generic, need to exclude guava-testlib
-    exclude "com.google.common.collect.testing.**"
-    exclude "com.google.common.escape.testing.**"
-    exclude "com.google.common.testing.**"
-    exclude "com.google.common.util.concurrent.testing.**"
-  }
-}
-
-artifacts {
-  shadowTest shadowTestJar
-}
diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle
index 5779dffbb7b..fd99721378a 100644
--- a/runners/direct-java/build.gradle
+++ b/runners/direct-java/build.gradle
@@ -19,7 +19,20 @@
 import groovy.json.JsonOutput
 
 apply from: project(":").file("build_rules.gradle")
-applyJavaNature()
+applyJavaNature(shadowClosure: DEFAULT_SHADOW_CLOSURE << {
+  dependencies {
+    include(dependency(library.java.protobuf_java))
+    include(dependency(library.java.protobuf_java_util))
+    include(project(path: ":beam-model-pipeline", configuration: "shadow"))
+    include(project(path: ":beam-runners-core-construction-java", 
configuration: "shadow"))
+    include(project(path: ":beam-runners-core-java", configuration: "shadow"))
+    include(project(path: ":beam-runners-local-java-core", configuration: 
"shadow"))
+  }
+  relocate "org.apache.beam.runners.core", getJavaRelocatedPath("runners.core")
+  relocate "org.apache.beam.model", getJavaRelocatedPath("model")
+  relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
+  relocate "javax.annotation", getJavaRelocatedPath("javax.annotation")
+})
 
 description = "Apache Beam :: Runners :: Direct Java"
 
@@ -84,52 +97,5 @@ task needsRunnerTests(type: Test) {
   }
 }
 
-shadowJar {
-  dependencies {
-    include(dependency(library.java.protobuf_java))
-    include(dependency(library.java.protobuf_java_util))
-    include(project(path: ":beam-model-pipeline", configuration: "shadow"))
-    include(project(path: ":beam-runners-core-construction-java", 
configuration: "shadow"))
-    include(project(path: ":beam-runners-core-java", configuration: "shadow"))
-    include(project(path: ":beam-runners-local-java-core", configuration: 
"shadow"))
-  }
-  relocate "org.apache.beam.runners.core", getJavaRelocatedPath("runners.core")
-  relocate "org.apache.beam.model", getJavaRelocatedPath("model")
-  relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
-  relocate "javax.annotation", getJavaRelocatedPath("javax.annotation")
-}
-
-// Create a shaded test jar.
-task shadowTestJar(type: 
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
-  classifier = "shaded-tests"
-  from sourceSets.test.output
-  configurations = [project.configurations.testRuntime]
-  dependencies {
-    exclude(".*")
-    include(dependency(library.java.guava))
-    include(dependency(library.java.protobuf_java))
-    include(dependency(library.java.protobuf_java_util))
-    include(project(path: ":beam-model-pipeline", configuration: "shadow"))
-    include(project(path: ":beam-runners-core-construction-java", 
configuration: "shadow"))
-    include(project(path: ":beam-runners-core-java", configuration: "shadow"))
-    include(project(path: ":beam-runners-local-java-core", configuration: 
"shadow"))
-  }
-  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
-    // com.google.common is too generic, need to exclude guava-testlib
-    exclude "com.google.common.collect.testing.**"
-    exclude "com.google.common.escape.testing.**"
-    exclude "com.google.common.testing.**"
-    exclude "com.google.common.util.concurrent.testing.**"
-  }
-  relocate "org.apache.beam.runners.core", getJavaRelocatedPath("runners.core")
-  relocate "org.apache.beam.model", getJavaRelocatedPath("model")
-  relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
-  relocate "javax.annotation", getJavaRelocatedPath("javax.annotation")
-}
-
-artifacts {
-  shadowTest shadowTestJar
-}
-
 // Generates :beam-runners-direct-java:runQuickstartJavaDirect
 createJavaQuickstartValidationTask(name: 'Direct')
diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 53d62c7669e..6634be99c36 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -94,7 +94,6 @@ dependencies {
   validatesRunner project(path: project.path, configuration: "shadowTest")
   validatesRunner project(path: project.path, configuration: "shadow")
   validatesRunner project(path: project.path, configuration: "provided")
-  validatesRunner project.sourceSets.test.output
 }
 
 configurations.testRuntimeClasspath {
diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle
index 3ac9c233f92..267d9b23626 100644
--- a/sdks/java/core/build.gradle
+++ b/sdks/java/core/build.gradle
@@ -17,7 +17,17 @@
  */
 
 apply from: project(":").file("build_rules.gradle")
-applyJavaNature()
+applyJavaNature(shadowClosure: DEFAULT_SHADOW_CLOSURE << {
+  dependencies {
+    include(dependency(library.java.protobuf_java))
+    include(dependency(library.java.byte_buddy))
+    include(dependency("org.apache.commons:.*"))
+  }
+  relocate "com.google.thirdparty", 
getJavaRelocatedPath("com.google.thirdparty")
+  relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
+  relocate "net.bytebuddy", getJavaRelocatedPath("net.bytebuddy")
+  relocate "org.apache.commons", getJavaRelocatedPath("org.apache.commons")
+})
 applyAvroNature()
 
 description = "Apache Beam :: SDKs :: Java :: Core"
@@ -75,46 +85,3 @@ dependencies {
   shadowTest "com.esotericsoftware.kryo:kryo:2.21"
   shadowTest project(path: ":beam-model-fn-execution", configuration: "shadow")
 }
-
-// Shade dependencies.
-shadowJar {
-  dependencies {
-    include(dependency(library.java.protobuf_java))
-    include(dependency(library.java.byte_buddy))
-    include(dependency("org.apache.commons:.*"))
-  }
-  relocate "com.google.thirdparty", 
getJavaRelocatedPath("com.google.thirdparty")
-  relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
-  relocate "net.bytebuddy", getJavaRelocatedPath("net.bytebuddy")
-  relocate "org.apache.commons", getJavaRelocatedPath("org.apache.commons")
-}
-
-// Create a shaded test jar.
-task shadowTestJar(type: 
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
-  classifier = "shaded-tests"
-  from sourceSets.test.output
-  configurations = [project.configurations.testRuntime]
-  dependencies {
-    exclude(".*")
-    include(dependency(library.java.guava))
-    include(dependency(library.java.protobuf_java))
-    include(dependency(library.java.protobuf_java_util))
-    include(dependency(library.java.byte_buddy))
-    include(dependency("org.apache.commons:.*"))
-  }
-  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
-    // com.google.common is too generic, need to exclude guava-testlib
-    exclude "com.google.common.collect.testing.**"
-    exclude "com.google.common.escape.testing.**"
-    exclude "com.google.common.testing.**"
-    exclude "com.google.common.util.concurrent.testing.**"
-  }
-  relocate "com.google.thirdparty", 
getJavaRelocatedPath("com.google.thirdparty")
-  relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
-  relocate "net.bytebuddy", getJavaRelocatedPath("net.bytebuddy")
-  relocate "org.apache.commons", getJavaRelocatedPath("org.apache.commons")
-}
-
-artifacts {
-  shadowTest shadowTestJar
-}
diff --git a/sdks/java/io/google-cloud-platform/build.gradle 
b/sdks/java/io/google-cloud-platform/build.gradle
index f9df0965f68..b9b0a93abc7 100644
--- a/sdks/java/io/google-cloud-platform/build.gradle
+++ b/sdks/java/io/google-cloud-platform/build.gradle
@@ -17,7 +17,15 @@
  */
 
 apply from: project(":").file("build_rules.gradle")
-applyJavaNature(enableFindbugs: false)
+applyJavaNature(
+  enableFindbugs: false,
+  // Override the default shading configuration to exclude everything since
+  // Bigtable needs to expose Guava types.
+  shadowClosure: {
+    dependencies {
+      exclude(".*")
+    }
+  })
 
 description = "Apache Beam :: SDKs :: Java :: IO :: Google Cloud Platform"
 ext.summary = "IO library to read and write Google Cloud Platform systems from 
Beam."
@@ -32,10 +40,10 @@ ext.summary = "IO library to read and write Google Cloud 
Platform systems from B
 evaluationDependsOn(":beam-sdks-java-extensions-google-cloud-platform-core")
 
 dependencies {
-  compile library.java.guava
   shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
   shadow project(path: 
":beam-sdks-java-extensions-google-cloud-platform-core", configuration: 
"shadow")
   shadow project(path: ":beam-sdks-java-extensions-protobuf", configuration: 
"shadow")
+  shadow library.java.guava
   shadow library.java.jackson_databind
   shadow library.java.grpc_core
   shadow library.java.google_api_services_bigquery
@@ -77,25 +85,3 @@ dependencies {
   shadowTest library.java.junit
   shadowTest library.java.slf4j_jdk14
 }
-
-// Create a shaded test jar.
-task shadowTestJar(type: 
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
-  classifier = "shaded-tests"
-  from sourceSets.test.output
-  configurations = [project.configurations.testRuntime]
-  dependencies {
-    exclude(".*")
-    include(dependency(library.java.guava))
-  }
-  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
-    // com.google.common is too generic, need to exclude guava-testlib
-    exclude "com.google.common.collect.testing.**"
-    exclude "com.google.common.escape.testing.**"
-    exclude "com.google.common.testing.**"
-    exclude "com.google.common.util.concurrent.testing.**"
-  }
-}
-
-artifacts {
-  shadowTest shadowTestJar
-}


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
-------------------

    Worklog Id:     (was: 90020)
    Time Spent: 11h 20m  (was: 11h 10m)

> Use Gradle to build/release project
> -----------------------------------
>
>                 Key: BEAM-3249
>                 URL: https://issues.apache.org/jira/browse/BEAM-3249
>             Project: Beam
>          Issue Type: Improvement
>          Components: build-system, testing
>            Reporter: Luke Cwik
>            Assignee: Luke Cwik
>            Priority: Major
>          Time Spent: 11h 20m
>  Remaining Estimate: 0h
>
> I have collected data by running several builds against master using Gradle 
> and Maven without using Gradle's support for incremental builds.
> Gradle (mins)
> min: 25.04
> max: 160.14
> median: 45.78
> average: 52.19
> stdev: 30.80
> Maven (mins)
> min: 56.86
> max: 216.55
> median: 87.93
> average: 109.10
> stdev: 48.01
> I excluded a few timeouts (240 mins) that happened during the Maven build 
> from its numbers but we can see conclusively that Gradle is about twice as 
> fast for the build when compared to Maven when run using Jenkins.
> Original dev@ thread: 
> https://lists.apache.org/thread.html/225dddcfc78f39bbb296a0d2bbef1caf37e17677c7e5573f0b6fe253@%3Cdev.beam.apache.org%3E
> The data is available here 
> https://docs.google.com/spreadsheets/d/1MHVjF-xoI49_NJqEQakUgnNIQ7Qbjzu8Y1q_h3dbF1M/edit?usp=sharing



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to