This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 390fc454bb2 Java gradle plugin and project split (#68380)
390fc454bb2 is described below
commit 390fc454bb28bcd51dfa583cf79f43bad9440dbe
Author: Tzu-ping Chung <[email protected]>
AuthorDate: Fri Jun 12 17:26:52 2026 +0800
Java gradle plugin and project split (#68380)
---
.github/workflows/ci-amd.yml | 2 +-
.github/workflows/ci-arm.yml | 2 +-
.github/workflows/codeql-analysis.yml | 2 +-
.../language-sdks/java.rst | 205 +++++++++++++++++--
.../tests/airflow_e2e_tests/conftest.py | 58 ++++--
.../tests/airflow_e2e_tests/constants.py | 5 +-
.../airflow_breeze/commands/developer_commands.py | 2 +-
java-sdk/README.md | 48 +++--
java-sdk/{example => bom}/build.gradle.kts | 37 ++--
java-sdk/build.gradle.kts | 61 ------
java-sdk/{example => buildSrc}/build.gradle.kts | 29 +--
.../main/kotlin/airflow-jvm-conventions.gradle.kts | 58 ++++++
.../src/main/kotlin/airflow-publish.gradle.kts | 112 +++++++++++
.../example/{build.gradle.kts => build.gradle} | 28 +--
java-sdk/example/gradle.properties | 1 +
.../example/{build.gradle.kts => settings.gradle} | 31 +--
.../src/resources}/dags/java_examples.py | 0
java-sdk/gradle.properties | 2 +-
java-sdk/plugin/build.gradle.kts | 57 ++++++
.../apache/airflow/sdk/plugin/AirflowSdkPlugin.kt | 221 +++++++++++++++++++++
java-sdk/{example => processor}/build.gradle.kts | 38 ++--
.../org/apache/airflow/sdk/BuilderProcessor.kt} | 70 -------
.../services/javax.annotation.processing.Processor | 0
.../kotlin/org/apache/airflow/sdk/BuilderTest.kt | 0
java-sdk/sdk/build.gradle.kts | 84 +-------
.../main/kotlin/org/apache/airflow/sdk/Builder.kt | 192 +-----------------
java-sdk/settings.gradle.kts | 20 +-
scripts/docker/install_jdk.sh | 2 +-
28 files changed, 824 insertions(+), 543 deletions(-)
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index f411d43d2b3..7329cbda22a 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -55,7 +55,7 @@ env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
# Keep these in sync:
- # - jvmTarget, languageVersion, and sourceCompatibility in
java-sdk/build.gradle.kts
+ # - jvmTarget, languageVersion, and sourceCompatibility in java-sdk/buildSrc
# - TEMURIN_VERSION in scripts/docker/install_jdk.sh
# - JAVA_VERSION in .github/workflows/ci-amd.yml and
.github/workflows/ci-arm.yml
# - java-version in .github/workflows/codeql-analysis.yml
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index 732d1e261ca..73f06289094 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -45,7 +45,7 @@ env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
# Keep these in sync:
- # - jvmTarget, languageVersion, and sourceCompatibility in
java-sdk/build.gradle.kts
+ # - jvmTarget, languageVersion, and sourceCompatibility in java-sdk/buildSrc
# - TEMURIN_VERSION in scripts/docker/install_jdk.sh
# - JAVA_VERSION in .github/workflows/ci-amd.yml and
.github/workflows/ci-arm.yml
# - java-version in .github/workflows/codeql-analysis.yml
diff --git a/.github/workflows/codeql-analysis.yml
b/.github/workflows/codeql-analysis.yml
index 6856afa12b1..9d660f18766 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -115,7 +115,7 @@ jobs:
persist-credentials: false
# Keep these in sync:
- # - jvmTarget, languageVersion, and sourceCompatibility in
java-sdk/build.gradle.kts
+ # - jvmTarget, languageVersion, and sourceCompatibility in
java-sdk/buildSrc
# - TEMURIN_VERSION in scripts/docker/install_jdk.sh
# - JAVA_VERSION in .github/workflows/ci-amd.yml and
.github/workflows/ci-arm.yml
# - java-version in .github/workflows/codeql-analysis.yml
diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst
b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst
index a752bee8017..380ec763733 100644
--- a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst
+++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst
@@ -272,25 +272,40 @@ represented as Java objects when read back via
``getXCom``.
Building and packaging
-----------------------
-The Java SDK is distributed as a JAR. Use any build tool; Gradle is shown here.
+The Java SDK is distributed as a JAR. The sections below show how to build a
bundle with Gradle or Maven.
-**Gradle setup**
+.. _java-sdk/build/gradle:
-Add the SDK dependency to your ``build.gradle.kts``:
+Gradle
+~~~~~~
-.. code-block:: kotlin
+Apply the Airflow SDK Gradle plugin in your ``build.gradle``:
+
+.. code-block:: groovy
+
+ plugins {
+ id("org.apache.airflow.sdk") version "${version}"
+ }
dependencies {
- implementation("org.apache.airflow:airflow-java-sdk:<version>")
- annotationProcessor("org.apache.airflow:airflow-java-sdk:<version>")
+
annotationProcessor("org.apache.airflow:airflow-sdk-processor:${version}")
+ implementation("org.apache.airflow:airflow-sdk:${version}")
}
- tasks.withType<Jar> {
- manifest {
- attributes("Main-Class" to "com.example.Main")
- }
+ airflowBundle {
+ mainClass = "com.example.Main" // Point to your main class instead.
}
+Then run:
+
+.. code-block:: bash
+
+ ./gradlew bundle
+
+The ``build/bundle/`` directory contains all required JAR(s). Copy or mount it
into the directory pointed to
+by ``jars_root`` in the coordinator configuration.
:class:`~airflow.sdk.coordinators.java.JavaCoordinator`
+scans ``jars_root`` recursively and builds the classpath automatically.
+
.. note::
You only need the ``annotationProcessor`` entry if you use the
annotation-based API. It is not needed for
@@ -298,20 +313,172 @@ Add the SDK dependency to your ``build.gradle.kts``:
.. note::
- The ``Main-Class`` manifest value is needed for the coordinator to know how
to run the JAR. You can choose
- to set this *on the coordinator itself* too by adding the ``main_class``
kwarg in coordinator configuration.
+ The plugin generates a fat JAR with the `Shadow
<https://gradleup.com/shadow/>`__ plugin by default. This is
+ generally a good idea since you only deploy one JAR file to avoid dependency
issues between projects. If this
+ does not suit you, set ``fatJar = false`` in ``airflowBundle`` to produce
thin JARs instead. The rest of the
+ process stays the same, but you will need to put all dependency JARs
somewhere Airflow can find with
+ ``jars_root``.
+
+.. _java-sdk/build/maven:
+
+Maven
+~~~~~
+
+Import the ``airflow-sdk-bom`` Bill of Materials so that artifact versions and
the
+``${airflow.supervisor.schema.version}`` property are managed in one place:
+
+.. code-block:: xml
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.airflow</groupId>
+ <artifactId>airflow-sdk-bom</artifactId>
+ <version>${version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+Add the SDK as a dependency (version is managed by the BOM):
+
+.. code-block:: xml
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.airflow</groupId>
+ <artifactId>airflow-sdk</artifactId>
+ </dependency>
+ </dependencies>
+
+Wire the annotation processor through ``maven-compiler-plugin`` so it stays
off the runtime classpath:
+
+.. code-block:: xml
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>org.apache.airflow</groupId>
+ <artifactId>airflow-sdk-processor</artifactId>
+ <version>${version}</version>
+ </path>
+ </annotationProcessorPaths>
+ </configuration>
+ </plugin>
+
+**Option 1 (recommended): fat JAR**
+
+Use ``maven-shade-plugin`` to bundle your code and all dependencies into a
single JAR. This is the
+simplest deployment: one file, no dependency management at runtime.
+
+.. code-block:: xml
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.6.0</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals><goal>shade</goal></goals>
+ <configuration>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <!-- Replace with your BundleBuilder
implementation. -->
+ <mainClass>com.example.Main</mainClass>
+ <manifestEntries>
+ <!-- Resolved from the BOM; do not hard-code
this value. -->
+
<Airflow-Supervisor-Schema-Version>${airflow.supervisor.schema.version}</Airflow-Supervisor-Schema-Version>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+Then run:
-Building a distribution
-~~~~~~~~~~~~~~~~~~~~~~~
+.. code-block:: bash
+
+ mvn package
+
+The fat JAR is written to ``target/<artifactId>-<version>.jar``. Copy it to
the directory configured as
+``jars_root`` in your coordinator.
+
+**Option 2: thin JAR with separate dependencies**
+
+If a fat JAR does not suit your project, use ``maven-jar-plugin`` to set
``Main-Class`` on the regular
+JAR and ``maven-dependency-plugin`` to collect all runtime dependencies
alongside it. Note that
+``Airflow-Supervisor-Schema-Version`` does not need to be set here since
Airflow reads it directly from the
+``airflow-sdk`` JAR on the classpath.
+
+.. code-block:: xml
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifestEntries>
+ <Main-Class>com.example.Main</Main-Class>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals><goal>copy-dependencies</goal></goals>
+ <configuration>
+
<outputDirectory>${project.build.directory}/bundle</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-artifact</id>
+ <phase>package</phase>
+ <goals><goal>copy</goal></goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>${project.artifactId}</artifactId>
+ <version>${project.version}</version>
+
<outputDirectory>${project.build.directory}/bundle</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+Then run:
.. code-block:: bash
- ./gradlew :myproject:installDist
+ mvn package
+
+``target/bundle/`` will contain the thin JAR and all runtime dependency JARs.
Point ``jars_root`` at
+this directory.
+
+.. note::
+
+ You only need the ``annotationProcessorPaths`` entry if you use the
annotation-based API.
+
+.. note::
-The ``lib/`` directory of the resulting distribution contains all required
JARs. Copy or mount it into the
-directory pointed to by ``jars_root`` in the coordinator configuration.
-:class:`~airflow.sdk.coordinators.java.JavaCoordinator` scans ``jars_root``
-recursively and builds the classpath automatically.
+ Unlike the Gradle plugin, Maven has no equivalent of the
``verifyBundleMainClass`` validation step.
+ A wrong ``<mainClass>`` value will not be caught until runtime.
.. _java-sdk/coordinator-config:
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
index 9d6f12d0f73..ce65c6fc9a4 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
@@ -45,8 +45,9 @@ from airflow_e2e_tests.constants import (
GO_SDK_EXAMPLE_BUNDLE_PKG,
JAVA_COMPOSE_PATH,
JAVA_DOCKERFILE_PATH,
- JAVA_SDK_DAGS_PATH,
+ JAVA_SDK_EXAMPLE_DAGS_PATH,
JAVA_SDK_EXAMPLE_LIBS_PATH,
+ JAVA_SDK_MAVEN_CACHE_PATH,
KAFKA_DIR_PATH,
LOCALSTACK_PATH,
LOGS_FOLDER,
@@ -253,15 +254,17 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
Java-capable Airflow worker image, copies the JARs into the temp directory,
and writes the coordinator configuration.
"""
- # Build the example bundle inside an ephemeral JDK container so the host
- # does not need Java installed.
- #
- # --user keeps build outputs owned by the current user (not root).
- # GRADLE_USER_HOME persists the Gradle distribution and dependency cache in
- # java-sdk/.gradle/ (already gitignored) so the first run downloads once
- # and subsequent runs skip straight to compilation.
- # --no-daemon avoids a background JVM that would outlive the container.
- console.print("[yellow]Building Java SDK example bundle
(eclipse-temurin:17-jdk)...")
+ # * --user keeps build outputs owned by the current user (not root).
+ # * --no-daemon avoids a background JVM that would outlive the container.
+ # * GRADLE_USER_HOME persists the Gradle distribution and dependency cache
+ # in java-sdk/.gradle/ so subsequent runs skip straight to compilation.
+ # * HOME is set explicitly because --user runs as the host UID which has no
+ # entry in the container's /etc/passwd; Docker would otherwise inherit
the
+ # image's HOME (/root) which the non-root process cannot write to.
+ # * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes
+ # there without nesting, and its contents are visible on the host.
+ console.print("[yellow]Publishing Java SDK artifacts to local Maven
repository...")
+ JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
subprocess.run(
[
"docker",
@@ -271,14 +274,43 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
f"{os.getuid()}:{os.getgid()}",
"-e",
"GRADLE_USER_HOME=/repo/java-sdk/.gradle",
- # Mount java-sdk/ at /java-sdk (the Gradle root project).
+ "-e",
+ "HOME=/workspace-home",
+ "-v",
+ f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
"-v",
f"{AIRFLOW_ROOT_PATH}:/repo",
"-w",
"/repo/java-sdk",
"eclipse-temurin:17-jdk",
"./gradlew",
- ":example:installDist",
+ "publishToMavenLocal",
+ "-PskipSigning=true",
+ "--no-daemon",
+ ],
+ check=True,
+ )
+ console.print("[yellow]Building Java SDK example bundle
(eclipse-temurin:17-jdk)...")
+ subprocess.run(
+ [
+ "docker",
+ "run",
+ "--rm",
+ "--user",
+ f"{os.getuid()}:{os.getgid()}",
+ "-e",
+ "GRADLE_USER_HOME=/repo/java-sdk/.gradle",
+ "-e",
+ "HOME=/workspace-home",
+ "-v",
+ f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
+ "-v",
+ f"{AIRFLOW_ROOT_PATH}:/repo",
+ "-w",
+ "/repo/java-sdk/example",
+ "eclipse-temurin:17-jdk",
+ "../gradlew",
+ "bundle",
"--no-daemon",
],
check=True,
@@ -293,7 +325,7 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
copytree(JAVA_SDK_EXAMPLE_LIBS_PATH, tmp_dir / "jars")
# Copy the Java SDK example Dag file so Airflow can discover it.
- copyfile(JAVA_SDK_DAGS_PATH / "java_examples.py", tmp_dir / "dags" /
"java_examples.py")
+ copyfile(JAVA_SDK_EXAMPLE_DAGS_PATH / "java_examples.py", tmp_dir / "dags"
/ "java_examples.py")
# Build a local Docker image that extends DOCKER_IMAGE with a JRE.
# We do this explicitly so testcontainers' DockerCompose.start() does not
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
index 259f8256fd8..ae3b64a94fa 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
@@ -52,8 +52,9 @@ KAFKA_DIR_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" /
"docker" / "kafka"
# Java SDK E2E test paths
JAVA_SDK_ROOT_PATH = AIRFLOW_ROOT_PATH / "java-sdk"
-JAVA_SDK_DAGS_PATH = JAVA_SDK_ROOT_PATH / "dags"
-JAVA_SDK_EXAMPLE_LIBS_PATH = JAVA_SDK_ROOT_PATH / "example" / "build" /
"install" / "example" / "lib"
+JAVA_SDK_EXAMPLE_DAGS_PATH = JAVA_SDK_ROOT_PATH / "example" / "src" /
"resources" / "dags"
+JAVA_SDK_EXAMPLE_LIBS_PATH = JAVA_SDK_ROOT_PATH / "example" / "build" /
"bundle"
+JAVA_SDK_MAVEN_CACHE_PATH = AIRFLOW_ROOT_PATH / "files" / "m2"
JAVA_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"java.yml"
JAVA_DOCKERFILE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"Dockerfile.java"
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 3768a3e2461..5e76822a35b 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -763,7 +763,7 @@ def _get_java_sdk_version() -> str:
"""Read the Java SDK version from 'java-sdk/gradle.properties'."""
props_path = AIRFLOW_ROOT_PATH / "java-sdk" / "gradle.properties"
for line in props_path.read_text().splitlines():
- if match := re.match(r"^sdkVersion\s*=\s*(\S+)$", line.strip()):
+ if match := re.match(r"^projectVersion\s*=\s*(\S+)$", line.strip()):
return match.group(1)
raise RuntimeError(f"Java SDK version not found in {props_path}")
diff --git a/java-sdk/README.md b/java-sdk/README.md
index 5123a594022..9d45f59d32e 100644
--- a/java-sdk/README.md
+++ b/java-sdk/README.md
@@ -43,16 +43,26 @@ This generates both an HTML representation and Javadoc.
## Running the example
-* Put the [DAG with stub tasks](./dags) to somewhere Airflow can find.
+The SDK projects must first built and published:
+
+```bash
+./gradlew publishToMavenLocal -PskipSigning=true
+```
+
+After the build is successful, you should be able to see directories in
`~/.m2/repository/org/apache/airflow/`.
+
+Now `cd example` into the example project, and
+
+* Put the [DAG with stub tasks](./example/src/resources/dags) to somewhere
Airflow can find.
* Ensure the `java` command is available in the same environment the Airflow
task worker is in.
-* Package the example and its dependencies into JARs in
- `./example/build/install/example/lib`
+* Package the example to `./example/build/bundle`
```bash
- ./gradlew :example:installDist
+ # We're now in the 'example' directory, so gradlew is in parent.
+ ../gradlew bundle
```
* Configure Airflow to route tasks in the *java* queue to be run with Java:
@@ -61,7 +71,7 @@ This generates both an HTML representation and Javadoc.
export AIRFLOW__SDK__COORDINATORS='{
"java": {
"classpath": "airflow.sdk.coordinators.java.JavaCoordinator",
- "kwargs": {"jars_root":
["/opt/airflow/java-sdk/example/build/install/example/lib"]}
+ "kwargs": {"jars_root": ["/opt/airflow/java-sdk/example/build/bundle"]}
}
}'
export AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{"java": "java"}'
@@ -101,7 +111,7 @@ The full release process follows the
Edit `gradle.properties` and set the version for this release:
```properties
-sdkVersion=1.0.0
+projectVersion=1.0.0
```
Commit the change and push it to the release branch.
@@ -112,8 +122,21 @@ Before touching any remote repository, publish to your
local Maven cache and
inspect the generated POM:
```bash
-./gradlew :sdk:publishToMavenLocal
-cat ~/.m2/repository/org/apache/airflow/airflow-sdk/*/airflow-sdk-*.pom
+rm -rf ~/.m2/repository/org/apache/airflow/ # Start clean.
+
+./gradlew publishToMavenLocal -PskipSigning=true
+
+# The airflow-sdk runtime.
+less ~/.m2/repository/org/apache/airflow/airflow-sdk/*/airflow-sdk-*.pom
+
+# The annotation processor for the builder pattern.
+less
~/.m2/repository/org/apache/airflow/airflow-sdk-processor/*/airflow-sdk-*.pom
+
+# The Gradle plugin for bundling.
+less
~/.m2/repository/org/apache/airflow/airflow-sdk-gradle-plugin/*/airflow-sdk-*.pom
+
+# The Gradle plugin's registration.
+less
~/.m2/repository/org/apache/airflow/sdk/org.apache.airflow.sdk.gradle.plugin/*/*.pom
```
Check that the coordinates, description, license, SCM, and organization fields
@@ -125,13 +148,14 @@ To test the full publish flow without touching ASF
infrastructure, override the
repository URL to a local directory
```bash
-./gradlew :sdk:publish -PmavenUrl=file:///tmp/local-maven-repo -PskipSigning
-ls /tmp/local-maven-repo/org/apache/airflow/airflow-sdk/
+./gradlew publish -PmavenUrl=file:///tmp/local-maven-repo -PskipSigning=true
+ls /tmp/local-maven-repo/org/apache/airflow/
+# This should contain the same components in ~/.m2 as inspected in the
previous step.
```
*NOTE:* Signing is not required since nothing goes to Maven Central. If you
want
to test signing, set the GPG private key and passphrase as described in the
next
-section, and remove `-PskipSigning` from the above command.
+section, and remove `-PskipSigning=true` from the above command.
### Publish to ASF Nexus staging
@@ -147,7 +171,7 @@ signing.password=your-gpg-key-passphrase
Then run the publish task.
```bash
-./gradlew :sdk:publish -P"signing.key=$(gpg --armor --export-secret-keys
your-gpg-key-fingerprint)"
+./gradlew publish -P"signing.key=$(gpg --armor --export-secret-keys
your-gpg-key-fingerprint)"
```
*NOTE:* The signing key is supplied through the command line since it contains
diff --git a/java-sdk/example/build.gradle.kts b/java-sdk/bom/build.gradle.kts
similarity index 53%
copy from java-sdk/example/build.gradle.kts
copy to java-sdk/bom/build.gradle.kts
index 42fa5b4ca91..479cbdde27f 100644
--- a/java-sdk/example/build.gradle.kts
+++ b/java-sdk/bom/build.gradle.kts
@@ -18,29 +18,30 @@
*/
plugins {
- application
+ `java-platform`
+ id("airflow-publish")
}
-dependencies {
- annotationProcessor(project(":sdk"))
- implementation(project(":sdk"))
- implementation("org.slf4j:slf4j-simple:2.0.17")
-}
+val projectVersion: String by project
+val airflowSupervisorSchemaVersion: String by project
-sourceSets {
- main {
- java.srcDir("src/java")
+dependencies {
+ constraints {
+ api("org.apache.airflow:airflow-sdk:$projectVersion")
+ api("org.apache.airflow:airflow-sdk-processor:$projectVersion")
}
}
-application {
- mainClass = "org.apache.airflow.example.ExampleBundleBuilder"
-}
-
-tasks.withType<Jar> {
- manifest {
- attributes(
- "Main-Class" to application.mainClass.get(),
- )
+publishing {
+ publications {
+ create<MavenPublication>("mavenBom") {
+ artifactId = "airflow-sdk-bom"
+ from(components["javaPlatform"])
+ pom {
+ name = "Apache Airflow Java SDK BOM"
+ description = "Bill of Materials for the Apache Airflow Java
SDK."
+ properties.put("airflow.supervisor.schema.version",
airflowSupervisorSchemaVersion)
+ }
+ }
}
}
diff --git a/java-sdk/build.gradle.kts b/java-sdk/build.gradle.kts
deleted file mode 100644
index 7a812bab0c4..00000000000
--- a/java-sdk/build.gradle.kts
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import com.diffplug.gradle.spotless.SpotlessExtension
-import org.jetbrains.kotlin.gradle.dsl.JvmTarget
-
-plugins {
- kotlin("jvm") version "2.3.0"
- id("com.diffplug.spotless") version "7.2.1" // Last version supporting JDK
11.
- id("org.jlleitschuh.gradle.ktlint") version "14.0.1"
-}
-
-allprojects {
- apply(plugin = "com.diffplug.spotless")
- apply(plugin = "org.jetbrains.kotlin.jvm")
- apply(plugin = "org.jlleitschuh.gradle.ktlint")
-
- repositories { mavenCentral() }
-
- // Keep these in sync:
- // - jvmTarget, languageVersion, and sourceCompatibility in
java-sdk/build.gradle.kts
- // - TEMURIN_VERSION in scripts/docker/install_jdk.sh
- // - JAVA_VERSION in .github/workflows/ci-amd.yml and
.github/workflows/ci-arm.yml
- // - java-version in .github/workflows/codeql-analysis.yml
- java {
- toolchain {
- languageVersion.set(JavaLanguageVersion.of(11))
- }
- sourceCompatibility = JavaVersion.VERSION_11
- }
- kotlin {
- compilerOptions {
- jvmTarget = JvmTarget.JVM_11
- }
- }
-
- configure<SpotlessExtension> {
- java {
- target("**/*.java")
- googleJavaFormat().formatJavadoc(false)
- trimTrailingWhitespace()
- endWithNewline()
- }
- }
-}
diff --git a/java-sdk/example/build.gradle.kts
b/java-sdk/buildSrc/build.gradle.kts
similarity index 66%
copy from java-sdk/example/build.gradle.kts
copy to java-sdk/buildSrc/build.gradle.kts
index 42fa5b4ca91..c620dd1e598 100644
--- a/java-sdk/example/build.gradle.kts
+++ b/java-sdk/buildSrc/build.gradle.kts
@@ -18,29 +18,16 @@
*/
plugins {
- application
+ `kotlin-dsl`
}
-dependencies {
- annotationProcessor(project(":sdk"))
- implementation(project(":sdk"))
- implementation("org.slf4j:slf4j-simple:2.0.17")
-}
-
-sourceSets {
- main {
- java.srcDir("src/java")
- }
+repositories {
+ mavenCentral()
+ gradlePluginPortal()
}
-application {
- mainClass = "org.apache.airflow.example.ExampleBundleBuilder"
-}
-
-tasks.withType<Jar> {
- manifest {
- attributes(
- "Main-Class" to application.mainClass.get(),
- )
- }
+dependencies {
+ implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:2.3.0")
+ implementation("com.diffplug.spotless:spotless-plugin-gradle:7.2.1")
+ implementation("org.jlleitschuh.gradle:ktlint-gradle:14.0.1")
}
diff --git
a/java-sdk/buildSrc/src/main/kotlin/airflow-jvm-conventions.gradle.kts
b/java-sdk/buildSrc/src/main/kotlin/airflow-jvm-conventions.gradle.kts
new file mode 100644
index 00000000000..2d541933fc0
--- /dev/null
+++ b/java-sdk/buildSrc/src/main/kotlin/airflow-jvm-conventions.gradle.kts
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.diffplug.gradle.spotless.SpotlessExtension
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
+
+plugins {
+ id("com.diffplug.spotless")
+ id("org.jetbrains.kotlin.jvm")
+ id("org.jlleitschuh.gradle.ktlint")
+}
+
+repositories {
+ mavenCentral()
+}
+
+// Keep these in sync:
+// - jvmTarget, languageVersion, and sourceCompatibility here
+// - TEMURIN_VERSION in scripts/docker/install_jdk.sh
+// - JAVA_VERSION in .github/workflows/ci-amd.yml and
.github/workflows/ci-arm.yml
+// - java-version in .github/workflows/codeql-analysis.yml
+java {
+ toolchain {
+ languageVersion.set(JavaLanguageVersion.of(11))
+ }
+ sourceCompatibility = JavaVersion.VERSION_11
+}
+
+kotlin {
+ compilerOptions {
+ jvmTarget = JvmTarget.JVM_11
+ }
+}
+
+configure<SpotlessExtension> {
+ java {
+ target("**/*.java")
+ googleJavaFormat().formatJavadoc(false)
+ trimTrailingWhitespace()
+ endWithNewline()
+ }
+}
diff --git a/java-sdk/buildSrc/src/main/kotlin/airflow-publish.gradle.kts
b/java-sdk/buildSrc/src/main/kotlin/airflow-publish.gradle.kts
new file mode 100644
index 00000000000..ec727c16856
--- /dev/null
+++ b/java-sdk/buildSrc/src/main/kotlin/airflow-publish.gradle.kts
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Convention plugin shared by all published Apache Airflow Java SDK
subprojects.
+ *
+ * Applying `id("airflow-publish")` in a subproject's `plugins {}` would do the
+ * following:
+ *
+ * - Sets `group = "org.apache.airflow"` and `version = projectVersion`.
+ * - Applies `maven-publish` and `signing`.
+ * - Wires the shared Maven repository (reads `mavenUrl`, `mavenUsername`,
+ * `mavenPassword`, or `ASF_NEXUS_*` env vars; auto-selects snapshot vs.
+ * release URL from `projectVersion`).
+ * - Injects the common POM fields (url, organization, license, SCM) into
+ * every `MavenPublication` in the subproject
+ * - Configures signing from `signing.key` / `SIGNING_KEY` and
+ * `signing.password` / `SIGNING_PASSWORD` (skipped when `skipSigning=true`)
+ *
+ * Each subproject only needs to provide its artifact-specific POM fields
+ * (`name` and `description`) and the `artifactId`.
+ */
+
+plugins {
+ `maven-publish`
+ signing
+}
+
+val projectVersion: String by project
+
+group = "org.apache.airflow"
+version = projectVersion
+
+private fun getProperty(name: String): String? =
providers.gradleProperty(name).orNull
+
+private fun getProperty(
+ name: String,
+ env: String,
+): String? = getProperty(name) ?: System.getenv(env)
+
+publishing {
+ publications.withType<MavenPublication>().configureEach {
+ pom {
+ url = "https://airflow.apache.org";
+ organization {
+ name = "The Apache Software Foundation"
+ url = "https://www.apache.org/";
+ }
+ licenses {
+ license {
+ name = "The Apache License, Version 2.0"
+ url = "https://www.apache.org/licenses/LICENSE-2.0.txt";
+ distribution = "repo"
+ }
+ }
+ scm {
+ connection =
"scm:git:https://gitbox.apache.org/repos/asf/airflow.git";
+ developerConnection =
"scm:git:https://gitbox.apache.org/repos/asf/airflow.git";
+ url = "https://github.com/apache/airflow";
+ }
+ }
+ }
+
+ repositories {
+ maven {
+ name = "maven"
+ val repoPath =
+ getProperty("mavenUrl")
+ ?: if (projectVersion.endsWith("-SNAPSHOT")) {
+
"https://repository.apache.org/content/repositories/snapshots/";
+ } else {
+
"https://repository.apache.org/service/local/staging/deploy/maven2/";
+ }
+ url = uri(repoPath)
+ if (!repoPath.startsWith("file:")) {
+ val user = getProperty("mavenUsername", "ASF_NEXUS_USERNAME")
+ val pass = getProperty("mavenPassword", "ASF_NEXUS_PASSWORD")
+ if (user != null && pass != null) {
+ credentials {
+ username = user
+ password = pass
+ }
+ }
+ }
+ }
+ }
+}
+
+signing {
+ if (!providers.gradleProperty("skipSigning").map { it.toBoolean()
}.getOrElse(false)) {
+ val signingKey = getProperty("signing.key", "SIGNING_KEY")
+ val signingPassword = getProperty("signing.password",
"SIGNING_PASSWORD")
+ useInMemoryPgpKeys(signingKey, signingPassword)
+ publishing.publications.configureEach { sign(this) }
+ }
+}
diff --git a/java-sdk/example/build.gradle.kts b/java-sdk/example/build.gradle
similarity index 69%
copy from java-sdk/example/build.gradle.kts
copy to java-sdk/example/build.gradle
index 42fa5b4ca91..497ea8f5103 100644
--- a/java-sdk/example/build.gradle.kts
+++ b/java-sdk/example/build.gradle
@@ -18,29 +18,33 @@
*/
plugins {
- application
+ id("org.apache.airflow.sdk") version "${projectVersion}"
+}
+
+repositories {
+ mavenLocal() // Not needed for your own project.
+ mavenCentral()
}
dependencies {
- annotationProcessor(project(":sdk"))
- implementation(project(":sdk"))
+
annotationProcessor("org.apache.airflow:airflow-sdk-processor:${projectVersion}")
+ implementation("org.apache.airflow:airflow-sdk:${projectVersion}")
implementation("org.slf4j:slf4j-simple:2.0.17")
}
+java {
+ toolchain {
+ languageVersion.set(JavaLanguageVersion.of(11))
+ }
+ sourceCompatibility = JavaVersion.VERSION_11
+}
+
sourceSets {
main {
java.srcDir("src/java")
}
}
-application {
+airflowBundle {
mainClass = "org.apache.airflow.example.ExampleBundleBuilder"
}
-
-tasks.withType<Jar> {
- manifest {
- attributes(
- "Main-Class" to application.mainClass.get(),
- )
- }
-}
diff --git a/java-sdk/example/gradle.properties
b/java-sdk/example/gradle.properties
new file mode 120000
index 00000000000..7677fb73be8
--- /dev/null
+++ b/java-sdk/example/gradle.properties
@@ -0,0 +1 @@
+../gradle.properties
\ No newline at end of file
diff --git a/java-sdk/example/build.gradle.kts
b/java-sdk/example/settings.gradle
similarity index 65%
copy from java-sdk/example/build.gradle.kts
copy to java-sdk/example/settings.gradle
index 42fa5b4ca91..83bae985ad8 100644
--- a/java-sdk/example/build.gradle.kts
+++ b/java-sdk/example/settings.gradle
@@ -17,30 +17,17 @@
* under the License.
*/
-plugins {
- application
-}
-
-dependencies {
- annotationProcessor(project(":sdk"))
- implementation(project(":sdk"))
- implementation("org.slf4j:slf4j-simple:2.0.17")
-}
-
-sourceSets {
- main {
- java.srcDir("src/java")
+// This is only needed since we want to route the plugin to the local build.
+// You don't need it in your own project.
+pluginManagement {
+ repositories {
+ mavenLocal()
+ gradlePluginPortal()
}
}
-application {
- mainClass = "org.apache.airflow.example.ExampleBundleBuilder"
+plugins {
+ id("org.gradle.toolchains.foojay-resolver-convention") version "0.10.0"
}
-tasks.withType<Jar> {
- manifest {
- attributes(
- "Main-Class" to application.mainClass.get(),
- )
- }
-}
+rootProject.name = "airflow-java-sdk-example"
diff --git a/java-sdk/dags/java_examples.py
b/java-sdk/example/src/resources/dags/java_examples.py
similarity index 100%
rename from java-sdk/dags/java_examples.py
rename to java-sdk/example/src/resources/dags/java_examples.py
diff --git a/java-sdk/gradle.properties b/java-sdk/gradle.properties
index 30b8ff333aa..47099646bfb 100644
--- a/java-sdk/gradle.properties
+++ b/java-sdk/gradle.properties
@@ -19,4 +19,4 @@ org.gradle.configuration-cache=true
airflowSupervisorSchemaVersion=2026-06-16
-sdkVersion=1.0.0-SNAPSHOT
+projectVersion=1.0.0-SNAPSHOT
diff --git a/java-sdk/plugin/build.gradle.kts b/java-sdk/plugin/build.gradle.kts
new file mode 100644
index 00000000000..1c242e71b5c
--- /dev/null
+++ b/java-sdk/plugin/build.gradle.kts
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ `java-gradle-plugin`
+ id("airflow-jvm-conventions")
+ id("airflow-publish")
+}
+
+repositories {
+ gradlePluginPortal()
+}
+
+dependencies {
+ implementation("com.gradleup.shadow:shadow-gradle-plugin:9.1.0") // Last
supporting Java 11.
+}
+
+gradlePlugin {
+ plugins {
+ create("airflowSdk") {
+ id = "org.apache.airflow.sdk"
+ implementationClass =
"org.apache.airflow.sdk.plugin.AirflowSdkPlugin"
+ displayName = "Apache Airflow SDK Plugin"
+ description = "Pack a Dag bundle JAR for Airflow to consume."
+ }
+ }
+}
+
+publishing {
+ publications {
+ withType<MavenPublication>().configureEach {
+ if (name == "pluginMaven") {
+ artifactId = "airflow-sdk-gradle-plugin"
+ pom {
+ name = "Apache Airflow SDK Gradle Plugin"
+ description = "Gradle plugin for building Apache Airflow
Dag bundle JARs."
+ }
+ }
+ }
+ }
+}
diff --git
a/java-sdk/plugin/src/main/kotlin/org/apache/airflow/sdk/plugin/AirflowSdkPlugin.kt
b/java-sdk/plugin/src/main/kotlin/org/apache/airflow/sdk/plugin/AirflowSdkPlugin.kt
new file mode 100644
index 00000000000..95836c12fc5
--- /dev/null
+++
b/java-sdk/plugin/src/main/kotlin/org/apache/airflow/sdk/plugin/AirflowSdkPlugin.kt
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airflow.sdk.plugin
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.SourceSetContainer
+import org.gradle.api.tasks.bundling.Jar
+import java.lang.reflect.Modifier
+import java.net.URLClassLoader
+import java.util.jar.JarFile
+import kotlin.jvm.java
+
+/**
+ * DSL extension registered by [AirflowSdkPlugin] as the `airflowBundle` block.
+ *
+ * ```kotlin
+ * airflowBundle {
+ * mainClass = "com.example.ExampleBundleBuilder"
+ * // fatJar = false // opt out of shadow JAR creation
+ * }
+ * ```
+ */
+abstract class AirflowBundleExtension {
+ /** Fully qualified name of the entry-point class.
+ *
+ * This is written to the JAR manifest under the standard `Main-Class` key.
The
+ * coordinator uses the value to execute the bundle JAR correctly.
+ */
+ @get:Input
+ abstract val mainClass: Property<String>
+
+ /**
+ * Whether to build a fat JAR using the Shadow plugin (default: `true`).
+ *
+ * When `true`, the Shadow plugin is applied automatically and
+ * `Airflow-Supervisor-Schema-Version` is injected into the shadow JAR
manifest.
+ * When `false`, no fat JAR is produced; only `Main-Class` is set on the
+ * regular `jar` task.
+ */
+ @get:Input
+ abstract val fatJar: Property<Boolean>
+}
+
+/**
+ * Gradle plugin for building Apache Airflow Java SDK bundles.
+ *
+ * Minimal usage:
+ *
+ * ```kotlin
+ * plugins {
+ * id("org.apache.airflow.sdk") version <version>
+ * }
+ *
+ * airflowBundle {
+ * mainClass = "com.example.ExampleBundleBuilder"
+ * }
+ * ```
+ *
+ * See [AirflowBundleExtension] to understand how to configure the plugin with
the
+ * `airflowBundle` block.
+ *
+ * The plugin automatically sets the `Main-Class` metadata, and provides a new
+ * task `bundle` to create a Dag bundle in one command. This builds
deploy-ready
+ * artifacts to `build/bundle/` that can be copied directly into an Airflow
Java
+ * coordinator's `jars_root`.
+ *
+ * By default, plugin `com.github.johnrengelman.shadow` is applied
automatically
+ * to enable fat JAR build. In this mode, one single JAR with user code and all
+ * dependencies is produced by the `bundle` task. An additional metadata entry
+ * `Airflow-Supervisor-Schema-Version` is also added to the JAR for Airflow to
+ * identify which version of the Supervisor Schema it should use to communicate
+ * with the built JAR.
+ *
+ * If `fatJar` is explicitly set to `false`, the `bundle` task builds a bare
JAR
+ * containing only the Dag bundle, and collect all dependency JARs into the
target
+ * directory instead. In this mode, `Airflow-Supervisor-Schema-Version` lives
in
+ * the `airflow-sdk` JAR instead. The bundle JAR still contains `Main-Class`.
+ *
+ */
+class AirflowSdkPlugin : Plugin<Project> {
+ override fun apply(project: Project) {
+ project.plugins.apply("java")
+
+ val ext = project.extensions.create("airflowBundle",
AirflowBundleExtension::class.java)
+ ext.fatJar.convention(true)
+
+ project.afterEvaluate {
+ project.tasks.withType(Jar::class.java).configureEach { task ->
+ task.doFirst {
+ ext.mainClass.orNull?.let { className ->
+ task.manifest.attributes(mapOf("Main-Class" to className))
+ }
+ }
+ }
+
+ val classFiles =
+ project.objects.fileCollection().from(
+ project.extensions
+ .getByType(SourceSetContainer::class.java)
+ .getByName("main")
+ .output
+ .classesDirs,
+ project.configurations.getByName("runtimeClasspath"),
+ )
+
+ val verifyTask =
+ project.tasks.register("verifyBundleMainClass") { task ->
+ task.group = "verification"
+ task.description =
+ "Verifies that mainClass exists in the compiled output and
declares a public static main method."
+ task.dependsOn(project.tasks.named("classes"))
+ task.inputs.files(classFiles).withPropertyName("classFiles")
+ task.inputs.property("mainClass", ext.mainClass)
+ task.doLast {
+ val className =
+ ext.mainClass.orNull
+ ?: error("airflowBundle.mainClass is not set. Add it to your
airflowBundle { } block.")
+ val urls = classFiles.map { it.toURI().toURL() }.toTypedArray()
+
+ URLClassLoader(urls, ClassLoader.getPlatformClassLoader()).use {
loader ->
+ val klass =
+ try {
+ loader.loadClass(className)
+ } catch (_: ClassNotFoundException) {
+ error("Configured main class '$className' is not found.")
+ }
+ val main =
+ try {
+ klass.getMethod("main", Array<String>::class.java)
+ } catch (_: NoSuchMethodException) {
+ error("mainClass '$className' does not have a public static
main method.")
+ }
+ check(Modifier.isStatic(main.modifiers)) {
+ "main method on '$className' is not static."
+ }
+ check(main.returnType == Void.TYPE) {
+ "main method on '$className' must return void."
+ }
+ }
+ }
+ }
+
+ if (ext.fatJar.get()) {
+ project.plugins.apply("com.gradleup.shadow")
+
+ val schemaVersionProvider =
+ project.providers.provider {
+ val runtimeClasspath =
+ project.configurations.findByName("runtimeClasspath")
+ ?: error(
+ "No runtimeClasspath configuration found. " +
+ "Make sure the java or application plugin is applied
before " +
+ "org.apache.airflow.sdk.",
+ )
+ runtimeClasspath.resolvedConfiguration.resolvedArtifacts
+ .firstOrNull { a ->
+ a.moduleVersion.id.group == "org.apache.airflow" && a.name ==
"airflow-sdk"
+ }?.let { artifact ->
+ JarFile(artifact.file).use { jar ->
+
jar.manifest?.mainAttributes?.getValue("Airflow-Supervisor-Schema-Version")
+ }
+ } ?: error(
+ "airflow-sdk not found in runtimeClasspath, or its JAR is
missing the " +
+ "Airflow-Supervisor-Schema-Version manifest attribute. " +
+ "Make sure org.apache.airflow:airflow-sdk is declared as an " +
+ "implementation dependency.",
+ )
+ }
+
+ project.tasks.withType(ShadowJar::class.java).configureEach { task ->
+ task.doFirst {
+ task.manifest.attributes(
+ mapOf("Airflow-Supervisor-Schema-Version" to
schemaVersionProvider.get()),
+ )
+ }
+ }
+
+ project.tasks.register("bundle", Copy::class.java) { task ->
+ task.group = "build"
+ task.description = "Assembles a fat JAR bundle for deployment to
Airflow."
+ task.dependsOn(verifyTask)
+ task.from(project.tasks.named("shadowJar"))
+ task.into(project.layout.buildDirectory.dir("bundle"))
+ }
+ } else {
+ // bundle copies the thin JAR and all runtime dependency JARs into
+ // build/bundle/, mirroring what installDist puts in lib/.
+ project.tasks.register("bundle", Copy::class.java) { task ->
+ task.group = "build"
+ task.description = "Assembles a thin JAR bundle directory for
deployment to Airflow."
+ task.dependsOn(verifyTask)
+ task.from(project.configurations.getByName("runtimeClasspath"))
+ task.from(project.tasks.named("jar"))
+ task.into(project.layout.buildDirectory.dir("bundle"))
+ }
+ }
+ }
+ }
+}
diff --git a/java-sdk/example/build.gradle.kts
b/java-sdk/processor/build.gradle.kts
similarity index 50%
rename from java-sdk/example/build.gradle.kts
rename to java-sdk/processor/build.gradle.kts
index 42fa5b4ca91..bddafe1a051 100644
--- a/java-sdk/example/build.gradle.kts
+++ b/java-sdk/processor/build.gradle.kts
@@ -18,29 +18,39 @@
*/
plugins {
- application
+ `java-library`
+ id("airflow-jvm-conventions")
+ id("airflow-publish")
}
dependencies {
- annotationProcessor(project(":sdk"))
+ compileOnly("javax.annotation:javax.annotation-api:1.3.2")
implementation(project(":sdk"))
- implementation("org.slf4j:slf4j-simple:2.0.17")
+ implementation("com.squareup:javapoet:1.13.0")
+
+ testImplementation(kotlin("test"))
+ testImplementation("com.google.testing.compile:compile-testing:0.23.0")
}
-sourceSets {
- main {
- java.srcDir("src/java")
- }
+java {
+ withSourcesJar()
}
-application {
- mainClass = "org.apache.airflow.example.ExampleBundleBuilder"
+tasks.withType<Test> {
+ useJUnitPlatform()
}
-tasks.withType<Jar> {
- manifest {
- attributes(
- "Main-Class" to application.mainClass.get(),
- )
+publishing {
+ publications {
+ create<MavenPublication>("mavenJava") {
+ artifactId = "airflow-sdk-processor"
+ from(components["java"])
+ pom {
+ name = "Apache Airflow Java SDK — Annotation Processor"
+ description =
+ "Annotation processor for the Apache Airflow Java SDK; " +
+ "generates *Builder classes from @Builder.Dag-annotated
sources."
+ }
+ }
}
}
diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Builder.kt
b/java-sdk/processor/src/main/kotlin/org/apache/airflow/sdk/BuilderProcessor.kt
similarity index 76%
copy from java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Builder.kt
copy to
java-sdk/processor/src/main/kotlin/org/apache/airflow/sdk/BuilderProcessor.kt
index 2231f4db10d..979615c12f2 100644
--- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Builder.kt
+++
b/java-sdk/processor/src/main/kotlin/org/apache/airflow/sdk/BuilderProcessor.kt
@@ -39,76 +39,6 @@ import javax.lang.model.type.TypeKind
import javax.lang.model.type.TypeMirror
import javax.tools.Diagnostic
-/**
- * Container for the annotation-based Dag-authoring API.
- *
- * This class is not instantiated directly. Its nested annotations drive the
- * [BuilderProcessor] annotation processor, which generates a `*Builder` class
- * for each class annotated with [Dag].
- *
- * Example:
- *
- * ```java
- * @Builder.Dag(id = "my_pipeline")
- * public class MyPipeline {
- *
- * @Builder.Task(id = "extract")
- * public long extract(Client client) { ... }
- *
- * @Builder.Task(id = "transform")
- * public long transform(Client client, @Builder.XCom(task = "extract")
long extracted) { ... }
- * }
- * ```
- *
- * The processor generates `MyPipelineBuilder.build()`, which returns a
- * fully wired-up [Dag] ready to add to a [Bundle].
- */
-class Builder internal constructor() {
- /**
- * Annotation to automate a Dag-builder pattern.
- *
- * When applied on a class Foo, this generates a FooBuilder class with a
- * static build method to create the Dag structure automatically.
- *
- * @param id Override the Dag ID. If empty or not provided, the annotated
- * class's name is used by default.
- * @param to Name of the Dag-builder class. If empty or not provided, use the
- * annotated class name + "Builder".
- */
- @Target(AnnotationTarget.CLASS)
- @MustBeDocumented
- annotation class Dag(
- val id: String = "",
- val to: String = "",
- )
-
- /**
- * Annotation to automate task definition in a Dag-builder pattern.
- *
- * @param id Override the task ID. If empty or not provided, the annotated
- * function's name is used by default.
- */
- @Target(AnnotationTarget.FUNCTION)
- @MustBeDocumented
- annotation class Task(
- val id: String = "",
- )
-
- /**
- * Annotation to mark a task definition's method parameter as an XCom input.
- *
- * @param task The task ID to pull. If empty or not given, the annotated
- * parameter's name is used by default.
- * @param key The XCom key to pull. Defaults to the task's return value.
- */
- @Target(AnnotationTarget.VALUE_PARAMETER)
- @MustBeDocumented
- annotation class XCom(
- val task: String = "",
- val key: String = Client.XCOM_RETURN_KEY,
- )
-}
-
/**
* @suppress
*
diff --git
a/java-sdk/sdk/src/main/resources/META-INF/services/javax.annotation.processing.Processor
b/java-sdk/processor/src/main/resources/META-INF/services/javax.annotation.processing.Processor
similarity index 100%
rename from
java-sdk/sdk/src/main/resources/META-INF/services/javax.annotation.processing.Processor
rename to
java-sdk/processor/src/main/resources/META-INF/services/javax.annotation.processing.Processor
diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt
b/java-sdk/processor/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt
similarity index 100%
rename from java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt
rename to
java-sdk/processor/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt
diff --git a/java-sdk/sdk/build.gradle.kts b/java-sdk/sdk/build.gradle.kts
index efeacace3a8..216ef765131 100644
--- a/java-sdk/sdk/build.gradle.kts
+++ b/java-sdk/sdk/build.gradle.kts
@@ -17,30 +17,16 @@
* under the License.
*/
-buildscript {
- repositories {
- mavenCentral()
- }
-}
-
val airflowSupervisorSchemaVersion: String by project
-val sdkArtifact = "airflow-sdk"
-val sdkVersion: String by project
-
-// Full Maven coordinate: org.apache.airflow:airflow-sdk:<version>
-// artifactId is set explicitly on the MavenPublication below.
-group = "org.apache.airflow"
-version = sdkVersion
-
plugins {
`java-library`
- `maven-publish`
- signing
- kotlin("plugin.serialization") version "2.3.0"
+ id("airflow-jvm-conventions")
+ id("airflow-publish")
id("org.jetbrains.dokka") version "2.2.0"
id("org.jetbrains.dokka-javadoc") version "2.2.0"
id("org.jsonschema2pojo") version "1.2.2"
+ kotlin("plugin.serialization") version "2.3.0"
}
// TODO: Use a hosted file instead.
@@ -58,7 +44,6 @@ dependencies {
implementation("com.fasterxml.jackson.core:jackson-databind:2.21.0")
implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.21.0")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.0")
- implementation("com.squareup:javapoet:1.13.0")
implementation("com.xenomachina:kotlin-argparser:2.0.7")
implementation("io.ktor:ktor-network:3.3.3")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2")
@@ -68,7 +53,6 @@ dependencies {
implementation("org.msgpack:jackson-dataformat-msgpack:0.9.11")
testImplementation(kotlin("test"))
- testImplementation("com.google.testing.compile:compile-testing:0.23.0")
testImplementation("com.squareup.okhttp3:mockwebserver:4.12.0")
}
@@ -214,7 +198,7 @@ sourceSets {
}
dokka {
- moduleVersion.set(sdkVersion)
+ moduleVersion.set(project.version.toString())
dokkaSourceSets.configureEach {
// Suppress everything in 'execution' since it's implementation detail.
perPackageOption {
@@ -262,74 +246,16 @@ tasks.withType<Test> {
useJUnitPlatform()
}
-private fun getProperty(name: String) = providers.gradleProperty(name).orNull
-
-private fun getProperty(
- name: String,
- env: String,
-): String? = getProperty(name) ?: System.getenv(env)
-
publishing {
publications {
create<MavenPublication>("mavenJava") {
- artifactId = sdkArtifact
+ artifactId = "airflow-sdk"
from(components["java"])
artifact(javadocJar)
pom {
name = "Apache Airflow Java SDK"
description = "Java SDK for implementing Apache Airflow task
logic on the JVM."
- url = "https://airflow.apache.org";
-
- organization {
- name = "The Apache Software Foundation"
- url = "https://www.apache.org/";
- }
- licenses {
- license {
- name = "The Apache License, Version 2.0"
- url = "https://www.apache.org/licenses/LICENSE-2.0.txt";
- distribution = "repo"
- }
- }
- scm {
- connection =
"scm:git:https://gitbox.apache.org/repos/asf/airflow.git";
- developerConnection =
"scm:git:https://gitbox.apache.org/repos/asf/airflow.git";
- url = "https://github.com/apache/airflow";
- }
}
}
}
-
- repositories {
- maven {
- name = "mavenRepo"
- val repoPath =
- getProperty("mavenUrl")
- ?: if (sdkVersion.endsWith("-SNAPSHOT")) {
-
"https://repository.apache.org/content/repositories/snapshots/";
- } else {
-
"https://repository.apache.org/service/local/staging/deploy/maven2/";
- }
- url = uri(repoPath)
- if (!repoPath.startsWith("file:")) {
- val user = getProperty("mavenUsername", "ASF_NEXUS_USERNAME")
- val pass = getProperty("mavenPassword", "ASF_NEXUS_PASSWORD")
- if (user != null && pass != null) {
- credentials {
- username = user
- password = pass
- }
- }
- }
- }
- }
-}
-
-signing {
- if (!providers.gradleProperty("skipSigning").map { it.toBoolean()
}.getOrElse(false)) {
- val signingKey = getProperty("signing.key", "SIGNING_KEY")
- val signingPassword = getProperty("signing.password",
"SIGNING_PASSWORD")
- useInMemoryPgpKeys(signingKey, signingPassword)
- sign(publishing.publications["mavenJava"])
- }
}
diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Builder.kt
b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Builder.kt
index 2231f4db10d..c12a53f83e5 100644
--- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Builder.kt
+++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Builder.kt
@@ -17,34 +17,14 @@
* under the License.
*/
-@file:Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
-
package org.apache.airflow.sdk
-import com.squareup.javapoet.ClassName
-import com.squareup.javapoet.JavaFile
-import com.squareup.javapoet.MethodSpec
-import com.squareup.javapoet.TypeName
-import com.squareup.javapoet.TypeSpec
-import javax.annotation.processing.AbstractProcessor
-import javax.annotation.processing.ProcessingEnvironment
-import javax.annotation.processing.RoundEnvironment
-import javax.annotation.processing.SupportedAnnotationTypes
-import javax.annotation.processing.SupportedSourceVersion
-import javax.lang.model.SourceVersion
-import javax.lang.model.element.ExecutableElement
-import javax.lang.model.element.Modifier
-import javax.lang.model.element.TypeElement
-import javax.lang.model.type.TypeKind
-import javax.lang.model.type.TypeMirror
-import javax.tools.Diagnostic
-
/**
* Container for the annotation-based Dag-authoring API.
*
* This class is not instantiated directly. Its nested annotations drive the
- * [BuilderProcessor] annotation processor, which generates a `*Builder` class
- * for each class annotated with [Dag].
+ * `BuilderProcessor` annotation processor in the :processor project,
+ * which generates a `*Builder` class for each class annotated with [Dag].
*
* Example:
*
@@ -108,171 +88,3 @@ class Builder internal constructor() {
val key: String = Client.XCOM_RETURN_KEY,
)
}
-
-/**
- * @suppress
- *
- * Annotation processor for [Builder.Dag].
- *
- * This is registered as a standard javac processor via
- * `META-INF/services/javax.annotation.processing.Processor`; not intended to
be
- * instantiated or referenced directly.
- *
- * For each class annotated with [Builder.Dag], generates a `*Builder` class
- * containing:
- *
- * - One inner class per [Builder.Task]-annotated method, implementing [Task].
- * - A static `build()` method that constructs the [Dag] and registers those
- * inner classes as tasks.
- *
- * [Builder.XCom]-annotated parameters are resolved via `client.getXCom` in the
- * generated `execute` body, with the result cast to the parameter's declared
- * type. Non-`void` return values are forwarded to `client.setXCom`.
- */
-@SupportedAnnotationTypes("org.apache.airflow.sdk.Builder.Dag")
-@SupportedSourceVersion(SourceVersion.RELEASE_11)
-class BuilderProcessor : AbstractProcessor() {
- override fun process(
- annotations: Set<TypeElement>,
- roundEnv: RoundEnvironment,
- ): Boolean {
- if (annotations.isEmpty()) return false
-
roundEnv.getElementsAnnotatedWith(Builder.Dag::class.java).filterIsInstance<TypeElement>().forEach
{ el ->
- with(processingEnv) {
- runCatching {
- JavaFile
- .builder(
- elementUtils.getPackageOf(el).qualifiedName.toString(),
- buildDag(el),
- ).build()
- .writeTo(filer)
- }.onFailure { e ->
- messager.printMessage(
- Diagnostic.Kind.ERROR,
- e.message ?: "Unknown error",
- el,
- )
- }
- }
- }
- return true
- }
-
- private fun buildDag(el: TypeElement): TypeSpec {
- val ann = el.getAnnotation(Builder.Dag::class.java)!!
-
- val builderClass =
- TypeSpec
- .classBuilder(ann.to.ifBlank { "${el.simpleName}Builder" })
- .addModifiers(Modifier.PUBLIC, Modifier.FINAL)
-
- val buildMethod =
- MethodSpec
- .methodBuilder("build")
- .addModifiers(Modifier.PUBLIC, Modifier.STATIC)
- .returns(ClassName.get(Dag::class.java))
- .addStatement($$"var dag = new $T($S)",
ClassName.get(Dag::class.java), ann.id.ifBlank { el.simpleName })
-
- for (inner in el.enclosedElements) {
- if (inner !is ExecutableElement) continue
- if (inner.isVarArgs) throw IllegalArgumentException("Cannot create task
from vararg function ${inner.simpleName}")
-
- val ann = inner.getAnnotation(Builder.Task::class.java) ?: continue
- val innerName =
inner.simpleName.toString().replaceFirstChar(Char::uppercase)
-
- val task = buildTask(innerName, inner, el)
- builderClass.addType(task.spec)
-
- buildMethod.addStatement(
- $$"dag.addTask($S, $L.class)",
- ann.id.ifBlank { inner.simpleName },
- innerName,
- )
- }
-
- buildMethod.addStatement("return dag")
- builderClass.addMethod(buildMethod.build())
- return builderClass.build()
- }
-
- private fun buildTask(
- name: String,
- inner: ExecutableElement,
- parent: TypeElement,
- ): BuildTaskResult {
- val clientType = ClassName.get(Client::class.java)
- val contextType = ClassName.get(Context::class.java)
-
- val executeSpec =
- MethodSpec
- .methodBuilder("execute")
- .addAnnotation(Override::class.java)
- .addModifiers(Modifier.PUBLIC)
- .returns(TypeName.VOID)
- .addParameter(contextType, "context")
- .addParameter(clientType, "client")
- .addException(Exception::class.java)
-
- val required = mutableListOf<RequiredXCom>()
- val innerArgs =
- with(processingEnv) {
- inner.parameters.joinToString { param ->
- val anno = param.getAnnotation(Builder.XCom::class.java)
- val type = param.asType()
- when {
- anno != null ->
- param.simpleName.toString().also {
- required += RequiredXCom(type, it, anno.task.ifBlank { it })
- }
- isType(type, clientType) -> "client"
- isType(type, contextType) -> "context"
- else -> throw IllegalArgumentException("Unsupported task parameter
'${param.simpleName}' with type: $type")
- }
- }
- }
- required.forEach {
- executeSpec.addStatement(
- $$"var $L = ($T) client.getXCom($S)",
- it.paramName,
- with(TypeName.get(it.paramType)) { if (isPrimitive) box() else this },
- it.taskId,
- )
- }
- if (inner.returnType.kind == TypeKind.VOID) {
- $$"new $T().$L($L)"
- } else {
- $$"client.setXCom(new $T().$L($L))"
- }.also {
- executeSpec.addStatement(
- it,
- ClassName.get(parent),
- inner.simpleName,
- innerArgs,
- )
- }
-
- val spec =
- TypeSpec
- .classBuilder(name)
- .addSuperinterface(Task::class.java)
- .addModifiers(Modifier.PUBLIC, Modifier.FINAL, Modifier.STATIC)
- .addMethod(executeSpec.build())
- .build()
- return BuildTaskResult(spec)
- }
-}
-
-private fun ProcessingEnvironment.isType(
- t: TypeMirror,
- c: ClassName,
-): Boolean = typeUtils.isSameType(t,
elementUtils.getTypeElement(c.canonicalName()).asType())
-
-private data class RequiredXCom(
- val paramType: TypeMirror,
- val paramName: String,
- val taskId: String,
-)
-
-private data class BuildTaskResult(
- val spec: TypeSpec,
-)
diff --git a/java-sdk/settings.gradle.kts b/java-sdk/settings.gradle.kts
index 0892437c13d..fd4f4ac001c 100644
--- a/java-sdk/settings.gradle.kts
+++ b/java-sdk/settings.gradle.kts
@@ -1,8 +1,20 @@
/*
- * This file was generated by the Gradle 'init' task.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * The settings file is used to specify which projects to include in your
build.
- * For more detailed information on multi-project builds, please refer to
https://docs.gradle.org/9.2.1/userguide/multi_project_builds.html in the Gradle
documentation.
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
plugins {
@@ -10,4 +22,4 @@ plugins {
}
rootProject.name = "airflow-java-sdk"
-include("example", "sdk")
+include("bom", "plugin", "processor", "sdk")
diff --git a/scripts/docker/install_jdk.sh b/scripts/docker/install_jdk.sh
index afe6eb3a69e..1c93483c80c 100755
--- a/scripts/docker/install_jdk.sh
+++ b/scripts/docker/install_jdk.sh
@@ -24,7 +24,7 @@ set -euo pipefail
common::get_colors
# Keep these in sync:
-# - jvmTarget, languageVersion, and sourceCompatibility in
java-sdk/build.gradle.kts
+# - jvmTarget, languageVersion, and sourceCompatibility in java-sdk/buildSrc
# - TEMURIN_VERSION in scripts/docker/install_jdk.sh
# - JAVA_VERSION in .github/workflows/ci-amd.yml and
.github/workflows/ci-arm.yml
# - java-version in .github/workflows/codeql-analysis.yml
