This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new abf9874 [BEAM-5780] Make fn-api-worker and legacy-worker point to the
different dir (#6715)
abf9874 is described below
commit abf987467493492e78c5aced806994423aa7c70d
Author: Boyuan Zhang <[email protected]>
AuthorDate: Wed Oct 17 14:57:06 2018 -0700
[BEAM-5780] Make fn-api-worker and legacy-worker point to the different dir
(#6715)
Make fn-api-worker and legacy-worker point to the different dir
---
.../google-cloud-dataflow-java/worker/build.gradle | 99 +---------
.../worker/legacy-worker/build.gradle | 217 +++++++++++++++++++++
settings.gradle | 2 +-
3 files changed, 219 insertions(+), 99 deletions(-)
diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle
b/runners/google-cloud-dataflow-java/worker/build.gradle
index 0a9a21f..2f6e090 100644
--- a/runners/google-cloud-dataflow-java/worker/build.gradle
+++ b/runners/google-cloud-dataflow-java/worker/build.gradle
@@ -16,32 +16,13 @@
* limitations under the License.
*/
-/******************************************************************************/
-// Apply BeamModulePlugin
-
-// Reuse project_root/buildSrc in this build.gradle file to reduce the
-// maintenance burden and simpily this file. See BeamModulePlugin for
-// documentation on default build tasks and properties that are enabled in
-// addition to natures that will be applied to worker.
apply plugin: org.apache.beam.gradle.BeamModulePlugin
-group = "org.apache.beam.runners.dataflow"
-
-/******************************************************************************/
-// Apply Java nature with customized configurations
-
// Set a specific version of 'com.google.apis:google-api-services-dataflow'
// by adding -Pdataflow.version=<version> in Gradle command. Otherwise,
// 'google_clients_version' defined in BeamModulePlugin will be used as
default.
def DATAFLOW_VERSION = "dataflow.version"
-// To build FnAPI or legacy worker.
-// Use -PisLegacyWorker in Gradle command if build legacy worker, otherwise,
-// FnAPI worker is considered as default.
-def is_legacy_worker = {
- return project.name.contains("dataflow-java-legacy-worker")
-}
-
// Get full dependency of 'com.google.apis:google-api-services-dataflow'
def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ?
"com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION)
: library.java.google_api_services_dataflow
@@ -73,36 +54,10 @@ def sdk_provided_project_dependencies = [
":beam-sdks-java-io-google-cloud-platform",
]
-// Exclude unneeded dependencies when building jar
-def excluded_dependencies = [
- "com.google.auto.service:auto-service", // Provided scope added from
applyJavaNature
- "com.google.auto.value:auto-value", // Provided scope added from
applyJavaNature
- "org.codehaus.jackson:jackson-core-asl", // Exclude an old version of
jackson-core-asl introduced by google-http-client-jackson
- "org.objenesis:objenesis", // Transitive dependency
introduced from Beam
- "org.tukaani:xz", // Transitive dependency
introduced from Beam
- library.java.commons_compress, // Transitive dependency
introduced from Beam
- library.java.error_prone_annotations, // Provided scope added in
worker
- library.java.hamcrest_core, // Test only
- library.java.hamcrest_library, // Test only
- library.java.junit, // Test only
-]
-
-applyJavaNature(validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE << {
+applyJavaNature(publish: false, enableFindbugs: false /* TODO(BEAM-5658):
enable findbugs */, validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE << {
dependencies {
include(project(path: ":beam-runners-google-cloud-dataflow-java-windmill",
configuration: "shadow"))
include(dependency(".*:.*"))
-
- if (is_legacy_worker()) {
- sdk_provided_dependencies.each {
- exclude(dependency(it))
- }
- sdk_provided_project_dependencies.each {
- exclude(project(path: it))
- }
- excluded_dependencies.each {
- exclude(dependency(it))
- }
- }
}
// Include original source files extracted under
@@ -111,45 +66,6 @@ applyJavaNature(validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE
exclude "META-INF/LICENSE.txt"
exclude "about.html"
- exclude "google/protobuf/*.proto"
- exclude "windmill*.proto"
-
- if (is_legacy_worker()) {
- relocate("com.", getWorkerRelocatedPath("com.")) {
- exclude "com.fasterxml.jackson.**"
- exclude "com.google.api.client.**"
- exclude "com.google.api.services.bigquery.**"
- exclude "com.google.api.services.clouddebugger.**"
- exclude "com.google.api.services.dataflow.**"
- exclude "com.google.api.services.datastore.**"
- exclude "com.google.api.services.pubsub.**"
- exclude "com.google.api.services.storage.**"
- exclude "com.google.auth.**"
- exclude "com.google.cloud.dataflow.**"
- exclude "com.sun.management*"
- exclude "com.sun.management.**"
- }
- relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet"))
- relocate("io.", getWorkerRelocatedPath("io."))
- relocate("okio.", getWorkerRelocatedPath("okio."))
- relocate("org.", getWorkerRelocatedPath("org.")) {
- // Exclude netty-tcnative from shading since gRPC relies on Netty to be
able
- // to load org.apache.tomcat.jni.SSL to provide an SSL context.
- exclude "org.apache.avro.**"
- exclude "org.apache.beam.**"
- exclude "org.apache.tomcat.jni.**"
- exclude "org.conscrypt.**"
- exclude "org.eclipse.jetty.alpn.**"
- exclude "org.eclipse.jetty.npn.**"
- exclude "org.hamcrest.**"
- exclude "org.joda.time.**"
- exclude "org.junit.**"
- exclude "org.slf4j.**"
- exclude "org.w3c.dom.**"
- }
- relocate("org.apache.beam.runners.core.construction.",
- getWorkerRelocatedPath("org."))
- }
})
/******************************************************************************/
@@ -182,23 +98,12 @@ def common_test_compile_project_dependencies = [
]
dependencies {
- // Set dependencies to shadow scope by default, but in a property so they can
- // be downgraded when building a legacy (non-FnAPI) worker.
- if (is_legacy_worker()) {
- sdk_provided_dependencies.each {
- provided(it)
- }
- sdk_provided_project_dependencies.each {
- provided project(path: it, configuration: "shadow")
- }
- } else {
sdk_provided_dependencies.each {
shadow(it)
}
sdk_provided_project_dependencies.each {
shadow project(path: it, configuration: "shadow")
}
- }
common_compile_project_dependencies.each {
compile project(path: it, configuration: "shadow")
@@ -231,7 +136,5 @@ dependencies {
//TODO(BEAM-5657): checktyle task should be enabled in the future.
checkstyleMain.enabled = false
checkstyleTest.enabled = false
-//TODO(BEAM-5658): fingbugs task should be enabled in the future.
-findbugsMain.enabled = false
//TODO(BEAM-5659): javadoc task should be enabled in the future.
javadoc.enabled = false
diff --git
a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
new file mode 100644
index 0000000..3b45dd0
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// There is no actual src dir under legacy-worker/ since legacy-worker is
built against the
+// same sourceSets as fn-api-worker. The reason why legacy-worker has a
separated dir is:
+// 1. Instead of point both legacy-worker and fn-api-worker project to the
same worker/,
+// make them point to different dirs.
+// 2. Make built artifacts and test reports go to worker/build for
fn-api-worker and
+// legacy-worker/build for legacy worker.
+
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+
+// Set a specific version of 'com.google.apis:google-api-services-dataflow'
+// by adding -Pdataflow.version=<version> in Gradle command. Otherwise,
+// 'google_clients_version' defined in BeamModulePlugin will be used as
default.
+def DATAFLOW_VERSION = "dataflow.version"
+
+// Get full dependency of 'com.google.apis:google-api-services-dataflow'
+def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ?
"com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION)
: library.java.google_api_services_dataflow
+
+// Returns a string representing the relocated path to be used with the shadow
+// plugin when given a suffix such as "com.".
+def getWorkerRelocatedPath = { String suffix ->
+ return ("org.apache.beam.runners.dataflow.worker.repackaged."
+ + suffix)
+}
+
+// Following listed dependencies will be shaded only in fnapi worker, not
legacy
+// worker
+def sdk_provided_dependencies = [
+ google_api_services_dataflow,
+ library.java.avro,
+ library.java.google_api_client,
+ library.java.google_http_client,
+ library.java.google_http_client_jackson,
+ library.java.jackson_annotations,
+ library.java.jackson_core,
+ library.java.jackson_databind,
+ library.java.joda_time,
+]
+
+def sdk_provided_project_dependencies = [
+ ":beam-runners-google-cloud-dataflow-java",
+ ":beam-sdks-java-core",
+ ":beam-sdks-java-extensions-google-cloud-platform-core",
+ ":beam-sdks-java-io-google-cloud-platform",
+]
+
+// Exclude unneeded dependencies when building jar
+def excluded_dependencies = [
+ "com.google.auto.service:auto-service", // Provided scope added
from applyJavaNature
+ "com.google.auto.value:auto-value", // Provided scope added
from applyJavaNature
+ "org.codehaus.jackson:jackson-core-asl", // Exclude an old version
of jackson-core-asl introduced by google-http-client-jackson
+ "org.objenesis:objenesis", // Transitive dependency
introduced from Beam
+ "org.tukaani:xz", // Transitive dependency
introduced from Beam
+ library.java.commons_compress, // Transitive dependency
introduced from Beam
+ library.java.error_prone_annotations, // Provided scope added
in worker
+ library.java.hamcrest_core, // Test only
+ library.java.hamcrest_library, // Test only
+ library.java.junit, // Test only
+]
+
+applyJavaNature(publish: false, enableFindbugs: false /* TODO(BEAM-5658):
enable findbugs */, validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE << {
+ dependencies {
+ include(project(path:
":beam-runners-google-cloud-dataflow-java-windmill", configuration: "shadow"))
+ include(dependency(".*:.*"))
+
+ sdk_provided_dependencies.each {
+ exclude(dependency(it))
+ }
+ sdk_provided_project_dependencies.each {
+ exclude(project(path: it))
+ }
+ excluded_dependencies.each {
+ exclude(dependency(it))
+ }
+ }
+
+ // Include original source files extracted under
+ // '$buildDir/original_sources_to_package' to jar
+ from "$buildDir/original_sources_to_package"
+
+ exclude "META-INF/LICENSE.txt"
+ exclude "about.html"
+
+ relocate("com.", getWorkerRelocatedPath("com.")) {
+ exclude "com.fasterxml.jackson.**"
+ exclude "com.google.api.client.**"
+ exclude "com.google.api.services.bigquery.**"
+ exclude "com.google.api.services.clouddebugger.**"
+ exclude "com.google.api.services.dataflow.**"
+ exclude "com.google.api.services.datastore.**"
+ exclude "com.google.api.services.pubsub.**"
+ exclude "com.google.api.services.storage.**"
+ exclude "com.google.auth.**"
+ exclude "com.google.cloud.dataflow.**"
+ exclude "com.sun.management*"
+ exclude "com.sun.management.**"
+ }
+ relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet"))
+ relocate("io.", getWorkerRelocatedPath("io."))
+ relocate("okio.", getWorkerRelocatedPath("okio."))
+ relocate("org.", getWorkerRelocatedPath("org.")) {
+ // Exclude netty-tcnative from shading since gRPC relies on Netty to
be able
+ // to load org.apache.tomcat.jni.SSL to provide an SSL context.
+ exclude "org.apache.avro.**"
+ exclude "org.apache.beam.**"
+ exclude "org.apache.tomcat.jni.**"
+ exclude "org.conscrypt.**"
+ exclude "org.eclipse.jetty.alpn.**"
+ exclude "org.eclipse.jetty.npn.**"
+ exclude "org.hamcrest.**"
+ exclude "org.joda.time.**"
+ exclude "org.junit.**"
+ exclude "org.slf4j.**"
+ exclude "org.w3c.dom.**"
+ }
+ relocate("org.apache.beam.runners.core.construction.",
+ getWorkerRelocatedPath("org."))
+
+})
+
+/******************************************************************************/
+// Configure the worker root project
+
+configurations {
+ sourceFile
+
+ // Ban these dependencies from all configurations
+ all {
+ // Ban the usage of AppleJavaExtensions in findbugs.
+ exclude group: "com.apple", module: "AppleJavaExtensions"
+ }
+}
+
+def common_compile_project_dependencies = [
+ ":beam-model-fn-execution",
+ ":beam-model-pipeline",
+ ":beam-runners-core-construction-java",
+ ":beam-runners-core-java",
+ ":beam-runners-java-fn-execution",
+ ":beam-sdks-java-fn-execution",
+ ":beam-runners-google-cloud-dataflow-java-windmill",
+]
+
+def common_test_compile_project_dependencies = [
+ ":beam-runners-core-java",
+ ":beam-sdks-java-core",
+ ":beam-sdks-java-extensions-google-cloud-platform-core",
+]
+
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker")
+
+compileJava {
+ source
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").sourceSets.main.java.srcDirs
+}
+
+compileTestJava {
+ source
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").sourceSets.test.java.srcDirs
+}
+
+dependencies {
+ sdk_provided_dependencies.each {
+ provided(it)
+ }
+ sdk_provided_project_dependencies.each {
+ provided project(path: it, configuration: "shadow")
+ }
+
+ common_compile_project_dependencies.each {
+ compile project(path: it, configuration: "shadow")
+ }
+
+ compile library.java.guava
+ compile library.java.slf4j_api
+ compile "javax.servlet:javax.servlet-api:3.1.0"
+ compile "org.conscrypt:conscrypt-openjdk:1.1.3:linux-x86_64"
+ compile "org.eclipse.jetty:jetty-server:9.2.10.v20150310"
+ compile "org.eclipse.jetty:jetty-servlet:9.2.10.v20150310"
+
+ provided library.java.error_prone_annotations
+
+ runtime library.java.slf4j_jdk14
+
+ common_test_compile_project_dependencies.each {
+ testCompile project(path: it, configuration: "shadowTest")
+ }
+
+ testCompile project(path: ":beam-runners-direct-java", configuration:
"shadow")
+
+ testCompile library.java.guava_testlib
+ testCompile library.java.hamcrest_core
+ testCompile library.java.hamcrest_library
+ testCompile library.java.junit
+ testCompile library.java.mockito_core
+}
+
+//TODO(BEAM-5657): checktyle task should be enabled in the future.
+checkstyleMain.enabled = false
+checkstyleTest.enabled = false
+//TODO(BEAM-5659): javadoc task should be enabled in the future.
+javadoc.enabled = false
diff --git a/settings.gradle b/settings.gradle
index f0e0705..0d83a38 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -183,7 +183,7 @@ project(":beam-vendor-sdks-java-extensions-protobuf").dir =
file("vendor/sdks-ja
include "beam-website"
project(":beam-website").dir = file("website")
include "beam-runners-google-cloud-dataflow-java-legacy-worker"
-project(":beam-runners-google-cloud-dataflow-java-legacy-worker").dir =
file("runners/google-cloud-dataflow-java/worker")
+project(":beam-runners-google-cloud-dataflow-java-legacy-worker").dir =
file("runners/google-cloud-dataflow-java/worker/legacy-worker")
include "beam-runners-google-cloud-dataflow-java-fn-api-worker"
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").dir =
file("runners/google-cloud-dataflow-java/worker")
include "beam-runners-google-cloud-dataflow-java-windmill"
