This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 58d108c4044 Add documentation about shared libraries in _shared
folders (#63468)
58d108c4044 is described below
commit 58d108c404479f2adeb8cbf77953f45a36e1f5f9
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Mar 12 20:03:38 2026 +0100
Add documentation about shared libraries in _shared folders (#63468)
* Add documentation about shared libraries in _shared folders
Update airflow-core/src/airflow/_shared/AGENTS..md
Co-authored-by: Kaxil Naik <[email protected]>
* Update airflow-core/src/airflow/_shared/AGENTS.md
Co-authored-by: Kaxil Naik <[email protected]>
---------
Co-authored-by: Kaxil Naik <[email protected]>
---
.pre-commit-config.yaml | 12 +++++-----
AGENTS.md | 13 +++++++++++
airflow-core/pyproject.toml | 4 ++++
airflow-core/src/airflow/_shared/AGENTS.md | 14 ++++++++++++
airflow-core/src/airflow/_shared/README.md | 35 ++++++++++++++++++++++++++++++
task-sdk/pyproject.toml | 13 ++++++++++-
task-sdk/src/airflow/sdk/_shared/AGENTS.md | 1 +
task-sdk/src/airflow/sdk/_shared/README.md | 1 +
8 files changed, 86 insertions(+), 7 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c519b6323bf..dd52f134ffa 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -165,9 +165,9 @@ repos:
(?x)
^\.github/.*\.md$|
^\.claude/|
- AGENTS\.md$|
- CLAUDE\.md$|
- SKILL\.md$|
+ ^.*/AGENTS\.md$|
+ ^.*/CLAUDE\.md$|
+ ^.*/SKILL\.md$|
^scripts/ci/license-templates/
- id: insert-license
name: Add short license for agentic Markdown files
@@ -181,9 +181,9 @@ repos:
(?x)
^\.github/.*\.md$|
^\.claude/|
- AGENTS\.md$|
- CLAUDE\.md$|
- SKILL\.md$
+ ^.*/AGENTS\.md$|
+ ^.*/CLAUDE\.md$|
+ ^.*/SKILL\.md$
exclude:
(?x)
^scripts/ci/license-templates/|
diff --git a/AGENTS.md b/AGENTS.md
index 3cf35fc9de5..26608d69cff 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -65,6 +65,18 @@ UV workspace monorepo. Key paths:
4. Workers execute tasks via Task SDK and communicate with the API server
through the Execution API — **never access the metadata DB directly**.
5. API Server serves the React UI and handles all client-database interactions.
6. Triggerer evaluates deferred tasks/sensors in isolated processes.
+7. Shared libraries that are symbolically linked to different Python
distributions are in `shared` folder.
+8. Airflow uses `uv workspace` feature to keep all the distributions sharing
dependencies and venv
+9. Each of the distributions should declare other needed distributions: `uv
--project <FOLDER> sync` command acts on the selected project in the monorepo
with only dependencies that it has
+
+# Shared libraries
+
+- shared libraries provide implementation of some common utilities like
logging, configuration where the code should be reused in different
distributions (potentially in different versions)
+- we have a number of shared libraries that are separate, small Python
distributions located under `shared` folder
+- each of the libraries has it's own src, tests, pyproject.toml and
dependencies
+- sources of those libraries are symbolically linked to the distributions that
are using them (`airflow-core`, `task-sdk` for example)
+- tests for the libraries (internal) are in the shared distribution's test and
can be run from the shared distributions
+- tests of the consumers using the shared libraries are present in the
distributions that use the libraries and can be run from there
## Coding Standards
@@ -86,6 +98,7 @@ UV workspace monorepo. Key paths:
- Test fixtures: `devel-common/src/tests_common/pytest_plugin.py`.
- Test location mirrors source: `airflow/cli/cli_parser.py` →
`tests/cli/test_cli_parser.py`.
+
## Commits and PRs
Write commit messages focused on user impact, not implementation details.
diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml
index 6d928f8a7d3..97ab77007a9 100644
--- a/airflow-core/pyproject.toml
+++ b/airflow-core/pyproject.toml
@@ -237,6 +237,8 @@ exclude = [
"src/airflow/ui/node_modules/",
"src/airflow/api_fastapi/auth/managers/simple/ui/node_modules",
"src/airflow/ui/openapi.merged.json",
+ "src/airflow/_shared/AGENTS.md",
+ "src/airflow/_shared/README.md",
]
[tool.hatch.build.targets.sdist.force-include]
@@ -273,6 +275,8 @@ exclude = [
# Only dist/ (declared as artifacts) is needed
"src/airflow/ui/**",
"src/airflow/api_fastapi/auth/managers/simple/ui/**",
+ "src/airflow/_shared/AGENTS.md",
+ "src/airflow/_shared/README.md",
]
[dependency-groups]
diff --git a/airflow-core/src/airflow/_shared/AGENTS.md
b/airflow-core/src/airflow/_shared/AGENTS.md
new file mode 100644
index 00000000000..1947c43f88f
--- /dev/null
+++ b/airflow-core/src/airflow/_shared/AGENTS.md
@@ -0,0 +1,14 @@
+
+<!-- SPDX-License-Identifier: Apache-2.0
+ https://www.apache.org/licenses/LICENSE-2.0 -->
+
+# The `_shared` package — Agent Instructions
+
+Each shared library is a symbolic link to the library package sources from the
shared library
+located in the [shared folder](../../shared). In the shared folder each
library is a separate
+distribution that has it's own tests and dependencies. Those dependencies and
links to those
+libraries are maintained by `prek` hook automatically.
+
+When you modify any of the files in place here - because those are symbolic
links - they are
+modified in the corresponding shared library, so any modification here should
result in modifying
+and running tests in the shared library the folder points to.
diff --git a/airflow-core/src/airflow/_shared/README.md
b/airflow-core/src/airflow/_shared/README.md
new file mode 100644
index 00000000000..658af6b5d97
--- /dev/null
+++ b/airflow-core/src/airflow/_shared/README.md
@@ -0,0 +1,35 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+## Why symbolic links here ?
+
+The sub-folders you see in this folder are symbolic links to the actual code
in the `shared` folder.
+The reason we are doing it is that we want to be able to use the shared
libraries in different
+distributions potentially in different versions - when several packages are
using the same shared library.
+
+Python - unlike for example npm - does not have a way to install different
versions of the same distribution,
+so what we effectively have to do is to vendor-in those shared libraries in
different packages that are
+using those libraries.
+
+By employing symbolic links we can avoid code duplication (single source of
current version of the shared
+library code is stored in "shared" folder) - and at the same time we can have
different versions of the
+same shared library in different packages when for example `airflow-core` and
`task-sdk` package are
+installed together in different version.
+
+You can read about it in [the shared README.md](../../shared/README.md)
document.
diff --git a/task-sdk/pyproject.toml b/task-sdk/pyproject.toml
index ed67188b87c..e3f128077e2 100644
--- a/task-sdk/pyproject.toml
+++ b/task-sdk/pyproject.toml
@@ -147,7 +147,18 @@ path = "src/airflow/sdk/__init__.py"
[tool.hatch.build.targets.wheel]
packages = ["src/airflow"]
# This file only exists to make pyright/VSCode happy, don't ship it
-exclude = ["src/airflow/__init__.py"]
+exclude = [
+ "src/airflow/__init__.py",
+ "src/airflow/sdk/_shared/AGENTS.md",
+ "src/airflow/sdk/_shared/README.md",
+]
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+ "src/airflow/__init__.py",
+ "src/airflow/sdk/_shared/AGENTS.md",
+ "src/airflow/sdk/_shared/README.md",
+]
[tool.ruff]
extend = "../pyproject.toml"
diff --git a/task-sdk/src/airflow/sdk/_shared/AGENTS.md
b/task-sdk/src/airflow/sdk/_shared/AGENTS.md
new file mode 120000
index 00000000000..8d2c7cf9d33
--- /dev/null
+++ b/task-sdk/src/airflow/sdk/_shared/AGENTS.md
@@ -0,0 +1 @@
+../../../../../airflow-core/src/airflow/_shared/AGENTS..md
\ No newline at end of file
diff --git a/task-sdk/src/airflow/sdk/_shared/README.md
b/task-sdk/src/airflow/sdk/_shared/README.md
new file mode 120000
index 00000000000..7ee9992c531
--- /dev/null
+++ b/task-sdk/src/airflow/sdk/_shared/README.md
@@ -0,0 +1 @@
+../../../../../airflow-core/src/airflow/_shared/README.md
\ No newline at end of file
