This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9f8dda274dc Split standalone transform service application from
launcher jar (#30190)
9f8dda274dc is described below
commit 9f8dda274dc7f7f562fb10bc9153d40ebf593ff0
Author: Yi Hu <[email protected]>
AuthorDate: Fri Feb 2 21:51:47 2024 -0500
Split standalone transform service application from launcher jar (#30190)
---
build.gradle.kts | 1 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 6 +--
sdks/java/extensions/python/build.gradle | 2 +-
sdks/java/harness/build.gradle | 2 +-
.../{launcher => app}/build.gradle | 45 ++++++----------------
sdks/java/transform-service/launcher/build.gradle | 23 +++--------
.../utils/transform_service_launcher.py | 2 +-
settings.gradle.kts | 1 +
.../content/en/documentation/programming-guide.md | 4 +-
9 files changed, 27 insertions(+), 59 deletions(-)
diff --git a/build.gradle.kts b/build.gradle.kts
index 82b3b6356a6..7e4c6cdc71a 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -312,6 +312,7 @@ tasks.register("javaPreCommit") {
dependsOn(":sdks:java:testing:tpcds:build")
dependsOn(":sdks:java:testing:watermarks:build")
dependsOn(":sdks:java:transform-service:build")
+ dependsOn(":sdks:java:transform-service:app:build")
dependsOn(":sdks:java:transform-service:launcher:build")
}
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index f3d79c528f8..10a7ba7f1c8 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2860,7 +2860,7 @@ class BeamModulePlugin implements Plugin<Project> {
project.evaluationDependsOn(":sdks:python")
project.evaluationDependsOn(":runners:core-construction-java")
project.evaluationDependsOn(":sdks:java:extensions:python")
- project.evaluationDependsOn(":sdks:java:transform-service:launcher")
+ project.evaluationDependsOn(":sdks:java:transform-service:app")
def usesDataflowRunner =
config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") ||
config.pythonPipelineOptions.contains("--runner=DataflowRunner")
@@ -2868,7 +2868,7 @@ class BeamModulePlugin implements Plugin<Project> {
def envDir = project.project(":sdks:python").envdir
def pythonDir = project.project(":sdks:python").projectDir
def externalPort = getRandomPort()
- def launcherJar =
project.project(':sdks:java:transform-service:launcher').shadowJar.archivePath
+ def launcherJar =
project.project(':sdks:java:transform-service:app').shadowJar.archivePath
def groupId = project.name + randomUUID().toString()
def transformServiceOpts = [
"transform_service_launcher_jar": launcherJar,
@@ -2895,7 +2895,7 @@ class BeamModulePlugin implements Plugin<Project> {
dependsOn ':sdks:python:expansion-service-container:docker'
dependsOn ':sdks:java:expansion-service:container:docker'
dependsOn ":sdks:python:installGcpTest"
- dependsOn
project.project(':sdks:java:transform-service:launcher').shadowJar.getPath()
+ dependsOn
project.project(':sdks:java:transform-service:app').shadowJar.getPath()
if (usesDataflowRunner) {
dependsOn
":sdks:python:test-suites:dataflow:py${project.ext.pythonVersion.replace('.',
'')}:initializeForDataflowJob"
diff --git a/sdks/java/extensions/python/build.gradle
b/sdks/java/extensions/python/build.gradle
index 40729d5f9ca..a83afac6881 100644
--- a/sdks/java/extensions/python/build.gradle
+++ b/sdks/java/extensions/python/build.gradle
@@ -30,7 +30,7 @@ dependencies {
implementation project(path: ":model:pipeline", configuration: "shadow")
implementation project(path: ":runners:core-construction-java")
implementation project(path: ":sdks:java:core", configuration: "shadow")
- implementation project(path: ":sdks:java:transform-service:launcher",
configuration: "shadow")
+ implementation project(path: ":sdks:java:transform-service:launcher")
testImplementation library.java.junit
testImplementation library.java.hamcrest
testImplementation
project(":runners:core-construction-java").sourceSets.test.output
diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle
index d3b4267424c..3c50f3c8edf 100644
--- a/sdks/java/harness/build.gradle
+++ b/sdks/java/harness/build.gradle
@@ -29,7 +29,7 @@ dependencies {
// :sdks:java:core and transitive dependencies
provided project(path: ":model:pipeline", configuration: "shadow")
provided project(path: ":sdks:java:core", configuration: "shadow")
- provided project(path: ":sdks:java:transform-service:launcher",
configuration: "shadow")
+ provided project(path: ":sdks:java:transform-service:launcher")
provided library.java.avro
provided library.java.joda_time
provided library.java.slf4j_api
diff --git a/sdks/java/transform-service/launcher/build.gradle
b/sdks/java/transform-service/app/build.gradle
similarity index 53%
copy from sdks/java/transform-service/launcher/build.gradle
copy to sdks/java/transform-service/app/build.gradle
index 0952f37109e..5125899bec9 100644
--- a/sdks/java/transform-service/launcher/build.gradle
+++ b/sdks/java/transform-service/app/build.gradle
@@ -15,48 +15,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
apply plugin: 'org.apache.beam.module'
apply plugin: 'application'
-mainClassName =
"org.apache.beam.sdk.transformservice.launcher.TransformServiceLauncher"
applyJavaNature(
- automaticModuleName: 'org.apache.beam.sdk.transform.service',
- exportJavadoc: false,
- validateShadowJar: false,
- shadowClosure: {},
+ automaticModuleName: 'org.apache.beam.sdk.transform.service.app',
+ exportJavadoc: false,
+ validateShadowJar: false,
+ shadowClosure: {},
)
+mainClassName =
"org.apache.beam.sdk.transformservice.launcher.TransformServiceLauncher"
-
-description = "Apache Beam :: SDKs :: Java :: Transform Service :: Launcher"
-ext.summary = """Contains code that can be used to run an transform service."""
-
-
-// Exclude tests that need a runner
-test {
- systemProperty "beamUseDummyRunner", "true"
- useJUnit {
- excludeCategories "org.apache.beam.sdk.testing.NeedsRunner"
- }
-}
+description = "Apache Beam :: SDKs :: Java :: Transform Service :: App"
+ext.summary = """Runs a transform service."""
dependencies {
- shadow library.java.vendored_guava_32_1_2_jre
- shadow library.java.slf4j_api
- shadow library.java.args4j
- shadow library.java.error_prone_annotations
- permitUnusedDeclared(library.java.error_prone_annotations)
- testImplementation library.java.junit
- testImplementation library.java.mockito_core
- testImplementation project(path: ":sdks:java:core")
-}
-
-sourceSets {
- main {
- resources {
- srcDirs "../docker-compose"
- }
- output.resourcesDir = "$buildDir/resources/docker-compose/$name"
- }
+ implementation project(":sdks:java:transform-service:launcher")
+ permitUnusedDeclared project(":sdks:java:transform-service:launcher")
+ runtimeOnly library.java.slf4j_jdk14
}
jar {
diff --git a/sdks/java/transform-service/launcher/build.gradle
b/sdks/java/transform-service/launcher/build.gradle
index 0952f37109e..e225d8da2f1 100644
--- a/sdks/java/transform-service/launcher/build.gradle
+++ b/sdks/java/transform-service/launcher/build.gradle
@@ -16,17 +16,11 @@
* limitations under the License.
*/
apply plugin: 'org.apache.beam.module'
-apply plugin: 'application'
-mainClassName =
"org.apache.beam.sdk.transformservice.launcher.TransformServiceLauncher"
applyJavaNature(
- automaticModuleName: 'org.apache.beam.sdk.transform.service',
- exportJavadoc: false,
- validateShadowJar: false,
- shadowClosure: {},
+ automaticModuleName: 'org.apache.beam.sdk.transform.service',
)
-
description = "Apache Beam :: SDKs :: Java :: Transform Service :: Launcher"
ext.summary = """Contains code that can be used to run an transform service."""
@@ -40,10 +34,11 @@ test {
}
dependencies {
- shadow library.java.vendored_guava_32_1_2_jre
- shadow library.java.slf4j_api
- shadow library.java.args4j
- shadow library.java.error_prone_annotations
+ // This library is designed to be light-weight, having minimum amount of
dependencies and is used by Beam runner artifacts.
+ implementation library.java.vendored_guava_32_1_2_jre
+ implementation library.java.slf4j_api
+ implementation library.java.args4j
+ compileOnly library.java.error_prone_annotations
permitUnusedDeclared(library.java.error_prone_annotations)
testImplementation library.java.junit
testImplementation library.java.mockito_core
@@ -58,9 +53,3 @@ sourceSets {
output.resourcesDir = "$buildDir/resources/docker-compose/$name"
}
}
-
-jar {
- manifest {
- attributes 'Main-Class': application.mainClass
- }
-}
diff --git a/sdks/python/apache_beam/utils/transform_service_launcher.py
b/sdks/python/apache_beam/utils/transform_service_launcher.py
index ac492513aba..ca3e456c523 100644
--- a/sdks/python/apache_beam/utils/transform_service_launcher.py
+++ b/sdks/python/apache_beam/utils/transform_service_launcher.py
@@ -35,7 +35,7 @@ _LOGGER = logging.getLogger(__name__)
_COMMAND_POSSIBLE_VALUES = ['up', 'down', 'ps']
-_EXPANSION_SERVICE_LAUNCHER_JAR = ':sdks:java:transform-service:launcher:build'
+_EXPANSION_SERVICE_LAUNCHER_JAR = ':sdks:java:transform-service:app:build'
class TransformServiceLauncher(object):
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 034df0fbda5..512820fce3f 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -277,6 +277,7 @@ include(":sdks:java:testing:test-utils")
include(":sdks:java:testing:tpcds")
include(":sdks:java:testing:watermarks")
include(":sdks:java:transform-service")
+include(":sdks:java:transform-service:app")
include(":sdks:java:transform-service:launcher")
include(":sdks:java:transform-service:controller-container")
include(":sdks:python")
diff --git a/website/www/site/content/en/documentation/programming-guide.md
b/website/www/site/content/en/documentation/programming-guide.md
index 47808063254..d3333c1f9f6 100644
--- a/website/www/site/content/en/documentation/programming-guide.md
+++ b/website/www/site/content/en/documentation/programming-guide.md
@@ -8165,7 +8165,7 @@ Beam users also have the option to [manually
start](/documentation/programming-g
A Beam Transform service instance can be manually started by using utilities
provided with Apache Beam SDKs.
{{< highlight java >}}
-java -jar beam-sdks-java-transform-service-launcher-<Beam version for the
jar>.jar --port <port> --beam_version <Beam version for the transform service>
--project_name <a unique ID for the transform service> --command up
+java -jar beam-sdks-java-transform-service-app-<Beam version for the jar>.jar
--port <port> --beam_version <Beam version for the transform service>
--project_name <a unique ID for the transform service> --command up
{{< /highlight >}}
{{< highlight py >}}
@@ -8179,7 +8179,7 @@ This feature is currently in development.
To stop the transform service, use the following commands.
{{< highlight java >}}
-java -jar beam-sdks-java-transform-service-launcher-<Beam version for the
jar>.jar --port <port> --beam_version <Beam version for the transform service>
--project_name <a unique ID for the transform service> --command down
+java -jar beam-sdks-java-transform-service-app-<Beam version for the jar>.jar
--port <port> --beam_version <Beam version for the transform service>
--project_name <a unique ID for the transform service> --command down
{{< /highlight >}}
{{< highlight py >}}
