[
https://issues.apache.org/jira/browse/BEAM-5780?focusedWorklogId=155624&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-155624
]
ASF GitHub Bot logged work on BEAM-5780:
----------------------------------------
Author: ASF GitHub Bot
Created on: 17/Oct/18 21:57
Start Date: 17/Oct/18 21:57
Worklog Time Spent: 10m
Work Description: lukecwik closed pull request #6715: [BEAM-5780] Make
fn-api-worker and legacy-worker point to the different dir
URL: https://github.com/apache/beam/pull/6715
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle
b/runners/google-cloud-dataflow-java/worker/build.gradle
index 0a9a21f6476..2f6e090c0a7 100644
--- a/runners/google-cloud-dataflow-java/worker/build.gradle
+++ b/runners/google-cloud-dataflow-java/worker/build.gradle
@@ -16,32 +16,13 @@
* limitations under the License.
*/
-/******************************************************************************/
-// Apply BeamModulePlugin
-
-// Reuse project_root/buildSrc in this build.gradle file to reduce the
-// maintenance burden and simpily this file. See BeamModulePlugin for
-// documentation on default build tasks and properties that are enabled in
-// addition to natures that will be applied to worker.
apply plugin: org.apache.beam.gradle.BeamModulePlugin
-group = "org.apache.beam.runners.dataflow"
-
-/******************************************************************************/
-// Apply Java nature with customized configurations
-
// Set a specific version of 'com.google.apis:google-api-services-dataflow'
// by adding -Pdataflow.version=<version> in Gradle command. Otherwise,
// 'google_clients_version' defined in BeamModulePlugin will be used as
default.
def DATAFLOW_VERSION = "dataflow.version"
-// To build FnAPI or legacy worker.
-// Use -PisLegacyWorker in Gradle command if build legacy worker, otherwise,
-// FnAPI worker is considered as default.
-def is_legacy_worker = {
- return project.name.contains("dataflow-java-legacy-worker")
-}
-
// Get full dependency of 'com.google.apis:google-api-services-dataflow'
def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ?
"com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION)
: library.java.google_api_services_dataflow
@@ -73,36 +54,10 @@ def sdk_provided_project_dependencies = [
":beam-sdks-java-io-google-cloud-platform",
]
-// Exclude unneeded dependencies when building jar
-def excluded_dependencies = [
- "com.google.auto.service:auto-service", // Provided scope added from
applyJavaNature
- "com.google.auto.value:auto-value", // Provided scope added from
applyJavaNature
- "org.codehaus.jackson:jackson-core-asl", // Exclude an old version of
jackson-core-asl introduced by google-http-client-jackson
- "org.objenesis:objenesis", // Transitive dependency
introduced from Beam
- "org.tukaani:xz", // Transitive dependency
introduced from Beam
- library.java.commons_compress, // Transitive dependency
introduced from Beam
- library.java.error_prone_annotations, // Provided scope added in
worker
- library.java.hamcrest_core, // Test only
- library.java.hamcrest_library, // Test only
- library.java.junit, // Test only
-]
-
-applyJavaNature(validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE << {
+applyJavaNature(publish: false, enableFindbugs: false /* TODO(BEAM-5658):
enable findbugs */, validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE << {
dependencies {
include(project(path: ":beam-runners-google-cloud-dataflow-java-windmill",
configuration: "shadow"))
include(dependency(".*:.*"))
-
- if (is_legacy_worker()) {
- sdk_provided_dependencies.each {
- exclude(dependency(it))
- }
- sdk_provided_project_dependencies.each {
- exclude(project(path: it))
- }
- excluded_dependencies.each {
- exclude(dependency(it))
- }
- }
}
// Include original source files extracted under
@@ -111,45 +66,6 @@ applyJavaNature(validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE
exclude "META-INF/LICENSE.txt"
exclude "about.html"
- exclude "google/protobuf/*.proto"
- exclude "windmill*.proto"
-
- if (is_legacy_worker()) {
- relocate("com.", getWorkerRelocatedPath("com.")) {
- exclude "com.fasterxml.jackson.**"
- exclude "com.google.api.client.**"
- exclude "com.google.api.services.bigquery.**"
- exclude "com.google.api.services.clouddebugger.**"
- exclude "com.google.api.services.dataflow.**"
- exclude "com.google.api.services.datastore.**"
- exclude "com.google.api.services.pubsub.**"
- exclude "com.google.api.services.storage.**"
- exclude "com.google.auth.**"
- exclude "com.google.cloud.dataflow.**"
- exclude "com.sun.management*"
- exclude "com.sun.management.**"
- }
- relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet"))
- relocate("io.", getWorkerRelocatedPath("io."))
- relocate("okio.", getWorkerRelocatedPath("okio."))
- relocate("org.", getWorkerRelocatedPath("org.")) {
- // Exclude netty-tcnative from shading since gRPC relies on Netty to be
able
- // to load org.apache.tomcat.jni.SSL to provide an SSL context.
- exclude "org.apache.avro.**"
- exclude "org.apache.beam.**"
- exclude "org.apache.tomcat.jni.**"
- exclude "org.conscrypt.**"
- exclude "org.eclipse.jetty.alpn.**"
- exclude "org.eclipse.jetty.npn.**"
- exclude "org.hamcrest.**"
- exclude "org.joda.time.**"
- exclude "org.junit.**"
- exclude "org.slf4j.**"
- exclude "org.w3c.dom.**"
- }
- relocate("org.apache.beam.runners.core.construction.",
- getWorkerRelocatedPath("org."))
- }
})
/******************************************************************************/
@@ -182,23 +98,12 @@ def common_test_compile_project_dependencies = [
]
dependencies {
- // Set dependencies to shadow scope by default, but in a property so they can
- // be downgraded when building a legacy (non-FnAPI) worker.
- if (is_legacy_worker()) {
- sdk_provided_dependencies.each {
- provided(it)
- }
- sdk_provided_project_dependencies.each {
- provided project(path: it, configuration: "shadow")
- }
- } else {
sdk_provided_dependencies.each {
shadow(it)
}
sdk_provided_project_dependencies.each {
shadow project(path: it, configuration: "shadow")
}
- }
common_compile_project_dependencies.each {
compile project(path: it, configuration: "shadow")
@@ -231,7 +136,5 @@ dependencies {
//TODO(BEAM-5657): checktyle task should be enabled in the future.
checkstyleMain.enabled = false
checkstyleTest.enabled = false
-//TODO(BEAM-5658): fingbugs task should be enabled in the future.
-findbugsMain.enabled = false
//TODO(BEAM-5659): javadoc task should be enabled in the future.
javadoc.enabled = false
diff --git
a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
new file mode 100644
index 00000000000..3b45dd01d58
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// There is no actual src dir under legacy-worker/ since legacy-worker is
built against the
+// same sourceSets as fn-api-worker. The reason why legacy-worker has a
separated dir is:
+// 1. Instead of point both legacy-worker and fn-api-worker project to the
same worker/,
+// make them point to different dirs.
+// 2. Make built artifacts and test reports go to worker/build for
fn-api-worker and
+// legacy-worker/build for legacy worker.
+
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+
+// Set a specific version of 'com.google.apis:google-api-services-dataflow'
+// by adding -Pdataflow.version=<version> in Gradle command. Otherwise,
+// 'google_clients_version' defined in BeamModulePlugin will be used as
default.
+def DATAFLOW_VERSION = "dataflow.version"
+
+// Get full dependency of 'com.google.apis:google-api-services-dataflow'
+def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ?
"com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION)
: library.java.google_api_services_dataflow
+
+// Returns a string representing the relocated path to be used with the shadow
+// plugin when given a suffix such as "com.".
+def getWorkerRelocatedPath = { String suffix ->
+ return ("org.apache.beam.runners.dataflow.worker.repackaged."
+ + suffix)
+}
+
+// Following listed dependencies will be shaded only in fnapi worker, not
legacy
+// worker
+def sdk_provided_dependencies = [
+ google_api_services_dataflow,
+ library.java.avro,
+ library.java.google_api_client,
+ library.java.google_http_client,
+ library.java.google_http_client_jackson,
+ library.java.jackson_annotations,
+ library.java.jackson_core,
+ library.java.jackson_databind,
+ library.java.joda_time,
+]
+
+def sdk_provided_project_dependencies = [
+ ":beam-runners-google-cloud-dataflow-java",
+ ":beam-sdks-java-core",
+ ":beam-sdks-java-extensions-google-cloud-platform-core",
+ ":beam-sdks-java-io-google-cloud-platform",
+]
+
+// Exclude unneeded dependencies when building jar
+def excluded_dependencies = [
+ "com.google.auto.service:auto-service", // Provided scope added
from applyJavaNature
+ "com.google.auto.value:auto-value", // Provided scope added
from applyJavaNature
+ "org.codehaus.jackson:jackson-core-asl", // Exclude an old version
of jackson-core-asl introduced by google-http-client-jackson
+ "org.objenesis:objenesis", // Transitive dependency
introduced from Beam
+ "org.tukaani:xz", // Transitive dependency
introduced from Beam
+ library.java.commons_compress, // Transitive dependency
introduced from Beam
+ library.java.error_prone_annotations, // Provided scope added
in worker
+ library.java.hamcrest_core, // Test only
+ library.java.hamcrest_library, // Test only
+ library.java.junit, // Test only
+]
+
+applyJavaNature(publish: false, enableFindbugs: false /* TODO(BEAM-5658):
enable findbugs */, validateShadowJar: false, shadowClosure:
DEFAULT_SHADOW_CLOSURE << {
+ dependencies {
+ include(project(path:
":beam-runners-google-cloud-dataflow-java-windmill", configuration: "shadow"))
+ include(dependency(".*:.*"))
+
+ sdk_provided_dependencies.each {
+ exclude(dependency(it))
+ }
+ sdk_provided_project_dependencies.each {
+ exclude(project(path: it))
+ }
+ excluded_dependencies.each {
+ exclude(dependency(it))
+ }
+ }
+
+ // Include original source files extracted under
+ // '$buildDir/original_sources_to_package' to jar
+ from "$buildDir/original_sources_to_package"
+
+ exclude "META-INF/LICENSE.txt"
+ exclude "about.html"
+
+ relocate("com.", getWorkerRelocatedPath("com.")) {
+ exclude "com.fasterxml.jackson.**"
+ exclude "com.google.api.client.**"
+ exclude "com.google.api.services.bigquery.**"
+ exclude "com.google.api.services.clouddebugger.**"
+ exclude "com.google.api.services.dataflow.**"
+ exclude "com.google.api.services.datastore.**"
+ exclude "com.google.api.services.pubsub.**"
+ exclude "com.google.api.services.storage.**"
+ exclude "com.google.auth.**"
+ exclude "com.google.cloud.dataflow.**"
+ exclude "com.sun.management*"
+ exclude "com.sun.management.**"
+ }
+ relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet"))
+ relocate("io.", getWorkerRelocatedPath("io."))
+ relocate("okio.", getWorkerRelocatedPath("okio."))
+ relocate("org.", getWorkerRelocatedPath("org.")) {
+ // Exclude netty-tcnative from shading since gRPC relies on Netty to
be able
+ // to load org.apache.tomcat.jni.SSL to provide an SSL context.
+ exclude "org.apache.avro.**"
+ exclude "org.apache.beam.**"
+ exclude "org.apache.tomcat.jni.**"
+ exclude "org.conscrypt.**"
+ exclude "org.eclipse.jetty.alpn.**"
+ exclude "org.eclipse.jetty.npn.**"
+ exclude "org.hamcrest.**"
+ exclude "org.joda.time.**"
+ exclude "org.junit.**"
+ exclude "org.slf4j.**"
+ exclude "org.w3c.dom.**"
+ }
+ relocate("org.apache.beam.runners.core.construction.",
+ getWorkerRelocatedPath("org."))
+
+})
+
+/******************************************************************************/
+// Configure the worker root project
+
+configurations {
+ sourceFile
+
+ // Ban these dependencies from all configurations
+ all {
+ // Ban the usage of AppleJavaExtensions in findbugs.
+ exclude group: "com.apple", module: "AppleJavaExtensions"
+ }
+}
+
+def common_compile_project_dependencies = [
+ ":beam-model-fn-execution",
+ ":beam-model-pipeline",
+ ":beam-runners-core-construction-java",
+ ":beam-runners-core-java",
+ ":beam-runners-java-fn-execution",
+ ":beam-sdks-java-fn-execution",
+ ":beam-runners-google-cloud-dataflow-java-windmill",
+]
+
+def common_test_compile_project_dependencies = [
+ ":beam-runners-core-java",
+ ":beam-sdks-java-core",
+ ":beam-sdks-java-extensions-google-cloud-platform-core",
+]
+
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker")
+
+compileJava {
+ source
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").sourceSets.main.java.srcDirs
+}
+
+compileTestJava {
+ source
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").sourceSets.test.java.srcDirs
+}
+
+dependencies {
+ sdk_provided_dependencies.each {
+ provided(it)
+ }
+ sdk_provided_project_dependencies.each {
+ provided project(path: it, configuration: "shadow")
+ }
+
+ common_compile_project_dependencies.each {
+ compile project(path: it, configuration: "shadow")
+ }
+
+ compile library.java.guava
+ compile library.java.slf4j_api
+ compile "javax.servlet:javax.servlet-api:3.1.0"
+ compile "org.conscrypt:conscrypt-openjdk:1.1.3:linux-x86_64"
+ compile "org.eclipse.jetty:jetty-server:9.2.10.v20150310"
+ compile "org.eclipse.jetty:jetty-servlet:9.2.10.v20150310"
+
+ provided library.java.error_prone_annotations
+
+ runtime library.java.slf4j_jdk14
+
+ common_test_compile_project_dependencies.each {
+ testCompile project(path: it, configuration: "shadowTest")
+ }
+
+ testCompile project(path: ":beam-runners-direct-java", configuration:
"shadow")
+
+ testCompile library.java.guava_testlib
+ testCompile library.java.hamcrest_core
+ testCompile library.java.hamcrest_library
+ testCompile library.java.junit
+ testCompile library.java.mockito_core
+}
+
+//TODO(BEAM-5657): checktyle task should be enabled in the future.
+checkstyleMain.enabled = false
+checkstyleTest.enabled = false
+//TODO(BEAM-5659): javadoc task should be enabled in the future.
+javadoc.enabled = false
diff --git a/settings.gradle b/settings.gradle
index f0e070535f2..0d83a3828c4 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -183,7 +183,7 @@ project(":beam-vendor-sdks-java-extensions-protobuf").dir =
file("vendor/sdks-ja
include "beam-website"
project(":beam-website").dir = file("website")
include "beam-runners-google-cloud-dataflow-java-legacy-worker"
-project(":beam-runners-google-cloud-dataflow-java-legacy-worker").dir =
file("runners/google-cloud-dataflow-java/worker")
+project(":beam-runners-google-cloud-dataflow-java-legacy-worker").dir =
file("runners/google-cloud-dataflow-java/worker/legacy-worker")
include "beam-runners-google-cloud-dataflow-java-fn-api-worker"
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").dir =
file("runners/google-cloud-dataflow-java/worker")
include "beam-runners-google-cloud-dataflow-java-windmill"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 155624)
Time Spent: 0.5h (was: 20m)
> fn-api-worker and legacy-worker should point to different dir
> -------------------------------------------------------------
>
> Key: BEAM-5780
> URL: https://issues.apache.org/jira/browse/BEAM-5780
> Project: Beam
> Issue Type: Bug
> Components: runner-dataflow
> Reporter: Boyuan Zhang
> Assignee: Boyuan Zhang
> Priority: Major
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
