This is an automated email from the ASF dual-hosted git repository.
damondouglas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 09c1c9e9f23 Enable Java SDK Distroless container image variants.
(#33173)
09c1c9e9f23 is described below
commit 09c1c9e9f23ed39b75885b68955960e44bee23de
Author: Damon <damondoug...@users.noreply.github.com>
AuthorDate: Fri Nov 22 16:19:19 2024 -0800
Enable Java SDK Distroless container image variants. (#33173)
* Enable Java SDK Distroless container image variant
* Add LANG environment and /usr/lib/locale
* Use examples tests instead
---
.github/trigger_files/beam_PostCommit_Java.json | 5 +-
.../beam_PostCommit_Java_DataflowV2.json | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 63 ++++++++++++++++++++++
sdks/java/container/Dockerfile-distroless | 42 +++++++++++++++
4 files changed, 110 insertions(+), 2 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Java.json
b/.github/trigger_files/beam_PostCommit_Java.json
index 9e26dfeeb6e..920c8d132e4 100644
--- a/.github/trigger_files/beam_PostCommit_Java.json
+++ b/.github/trigger_files/beam_PostCommit_Java.json
@@ -1 +1,4 @@
-{}
\ No newline at end of file
+{
+ "comment": "Modify this file in a trivial way to cause this test suite to
run",
+ "modification": 1
+}
\ No newline at end of file
diff --git a/.github/trigger_files/beam_PostCommit_Java_DataflowV2.json
b/.github/trigger_files/beam_PostCommit_Java_DataflowV2.json
index 1efc8e9e440..3f63c0c9975 100644
--- a/.github/trigger_files/beam_PostCommit_Java_DataflowV2.json
+++ b/.github/trigger_files/beam_PostCommit_Java_DataflowV2.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 1
+ "modification": 2
}
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 4906d9cf9cb..3a337684bf1 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -273,6 +273,69 @@ def createRunnerV2ValidatesRunnerTest = { Map args ->
}
}
+tasks.register('examplesJavaRunnerV2IntegrationTestDistroless', Test.class) {
+ group = "verification"
+ dependsOn 'buildAndPushDistrolessContainerImage'
+ def javaVer = project.findProperty('testJavaVersion')
+ def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
+ def tag = project.findProperty('dockerTag')
+ def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
+ def pipelineOptions = [
+ "--runner=TestDataflowRunner",
+ "--project=${gcpProject}",
+ "--region=${gcpRegion}",
+ "--tempRoot=${dataflowValidatesTempRoot}",
+ "--sdkContainerImage=${imageURL}",
+ "--experiments=use_unified_worker,use_runner_v2",
+ "--firestoreDb=${firestoreDb}",
+ ]
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions)
+
+ include '**/*IT.class'
+
+ maxParallelForks 4
+ classpath = configurations.examplesJavaIntegrationTest
+ testClassesDirs =
files(project(":examples:java").sourceSets.test.output.classesDirs)
+ useJUnit { }
+}
+
+tasks.register('buildAndPushDistrolessContainerImage', Task.class) {
+ // Only Java 17 and 21 are supported.
+ // See
https://github.com/GoogleContainerTools/distroless/tree/main/java#image-contents.
+ def allowed = ["java17", "java21"]
+ doLast {
+ def javaVer = project.findProperty('testJavaVersion')
+ if (!allowed.contains(javaVer)) {
+ throw new GradleException("testJavaVersion must be one of ${allowed},
got: ${javaVer}")
+ }
+ if (!project.hasProperty('dockerTag')) {
+ throw new GradleException("dockerTag is missing but required")
+ }
+ def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
+ def tag = project.findProperty('dockerTag')
+ def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
+ exec {
+ executable 'docker'
+ workingDir rootDir
+ args = [
+ 'buildx',
+ 'build',
+ '-t',
+ imageURL,
+ '-f',
+ 'sdks/java/container/Dockerfile-distroless',
+
"--build-arg=BEAM_BASE=gcr.io/apache-beam-testing/beam-sdk/beam_${javaVer}_sdk",
+
"--build-arg=DISTROLESS_BASE=gcr.io/distroless/${javaVer}-debian12",
+ '.'
+ ]
+ }
+ exec {
+ executable 'docker'
+ args = ['push', imageURL]
+ }
+ }
+}
+
// Push docker images to a container registry for use within tests.
// NB: Tasks which consume docker images from the registry should depend on
this
// task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures
the correct
diff --git a/sdks/java/container/Dockerfile-distroless
b/sdks/java/container/Dockerfile-distroless
new file mode 100644
index 00000000000..328c4dc6a7b
--- /dev/null
+++ b/sdks/java/container/Dockerfile-distroless
@@ -0,0 +1,42 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+# ARG BEAM_BASE is the Beam SDK container image built using
sdks/python/container/Dockerfile.
+ARG BEAM_BASE
+
+# ARG DISTROLESS_BASE is the distroless container image URL. For available
distroless Java images,
+# see
https://github.com/GoogleContainerTools/distroless/tree/main?tab=readme-ov-file#what-images-are-available.
+# Only Java versions 17 and 21 are supported.
+ARG DISTROLESS_BASE
+FROM ${BEAM_BASE} AS base
+ARG TARGETARCH
+ENV LANG C.UTF-8
+
+LABEL Author="Apache Beam <d...@beam.apache.org>"
+
+RUN if [ -z "${TARGETARCH}" ]; then echo "fatal: TARGETARCH not set; run as
docker buildx build or use --build-arg=TARGETARCH=amd64|arm64" >&2; exit 1; fi
+
+FROM ${DISTROLESS_BASE}:latest-${TARGETARCH} AS distroless
+
+COPY --from=base /opt /opt
+
+# Along with the LANG environment variable above, prevents internally
discovered failing bugs related to Dataflow Flex
+# template character encodings.
+COPY --from=base /usr/lib/locale /usr/lib/locale
+
+ENTRYPOINT ["/opt/apache/beam/boot"]
