This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch ci-upgrade-main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/ci-upgrade-main by this push:
new 7f4f484a1b1 Switch Breeze from mprocs to dekit
7f4f484a1b1 is described below
commit 7f4f484a1b1f50044b3038a89332265db266331c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Oct 6 00:37:02 2026 +0200
Switch Breeze from mprocs to dekit
mprocs is released as dekit since 0.10.0, with no mprocs assets any
more, so the scheduled CI-environment upgrade broke the CI image build
by plugging 0.10.0 into the old mprocs download URL.
Breeze now installs dekit and runs the Airflow components as tasks of a
native dekit.yaml project rather than through dekit's mprocs mode, so it
uses the current configuration format directly. "mprocs" stays accepted
as a hidden alias of "dekit" for --terminal-multiplexer, the
TERMINAL_MULTIPLEXER variable and inside the container, and a
remembered "mprocs" choice is migrated to "dekit" so users keep their
preference.
Generated-by: Claude Opus 5
---
Dockerfile.ci | 29 ++--
contributing-docs/03_contributors_quick_start.rst | 25 ++--
.../03a_contributors_quick_start_beginners.rst | 4 +-
.../20_debugging_airflow_components.rst | 22 +--
contributing-docs/dekit/DEKIT_QUICK_REFERENCE.md | 156 +++++++++++++++++++++
.../{mprocs/mprocs.yaml => dekit/dekit.yaml} | 58 ++++----
contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md | 148 -------------------
dev/README_RELEASE_PROVIDERS.md | 2 +-
dev/breeze/doc/03_developer_tasks.rst | 2 +-
dev/breeze/doc/images/output_setup_config.svg | 2 +-
dev/breeze/doc/images/output_setup_config.txt | 2 +-
dev/breeze/doc/images/output_start-airflow.svg | 4 +-
dev/breeze/doc/images/output_start-airflow.txt | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 6 +-
dev/breeze/src/airflow_breeze/utils/cache.py | 16 ++-
.../src/airflow_breeze/utils/custom_param_types.py | 2 +
dev/breeze/tests/test_cache.py | 61 +++++++-
scripts/ci/prek/upgrade_important_versions.py | 9 +-
scripts/docker/entrypoint_ci.sh | 7 +-
...e_mprocs_config.py => generate_dekit_config.py} | 80 +++++------
scripts/in_container/bin/run_dekit | 39 ++++++
scripts/in_container/bin/run_mprocs | 28 ----
.../in_container/test_generate_dekit_config.py | 93 ++++++++++++
23 files changed, 492 insertions(+), 305 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index d40106f147f..52b99f6bf34 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1383,9 +1383,10 @@ SSH_CONFIG
if [[ ${SKIP_ASSETS_COMPILATION:="false"} == "false" ]]; then
wait_for_asset_compilation
fi
- if [[ ${TERMINAL_MULTIPLEXER:="mprocs"} == "mprocs" ]]; then
- # shellcheck source=scripts/in_container/bin/run_mprocs
- exec run_mprocs
+ # "mprocs" is the old name of dekit, still accepted from older Breeze
versions.
+ if [[ ${TERMINAL_MULTIPLEXER:="dekit"} == "dekit" ||
${TERMINAL_MULTIPLEXER} == "mprocs" ]]; then
+ # shellcheck source=scripts/in_container/bin/run_dekit
+ exec run_dekit
else
# shellcheck source=scripts/in_container/bin/run_tmux
exec run_tmux
@@ -1844,17 +1845,17 @@ RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
&& rm -f /tmp/helm.tar.gz \
&& chmod +x /usr/local/bin/helm
-# Install mprocs - a modern process manager for managing multiple Airflow
components
-ARG MPROCS_VERSION="0.10.0"
-
-RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
- && PLATFORM="$(uname -m)" \
- &&
MPROCS_URL="https://github.com/pvolok/mprocs/releases/download/v${MPROCS_VERSION}/mprocs-${MPROCS_VERSION}-${SYSTEM}-${PLATFORM}-musl.tar.gz";
\
- && echo "Downloading mprocs from ${MPROCS_URL}" \
- && curl --silent --location --fail --retry 3 --retry-delay 5 -o
/tmp/mprocs.tar.gz "${MPROCS_URL}" \
- && tar -xzf /tmp/mprocs.tar.gz -C /usr/local/bin/ mprocs \
- && rm -f /tmp/mprocs.tar.gz \
- && chmod +x /usr/local/bin/mprocs
+# Install dekit - the process manager formerly released as mprocs - that runs
the Airflow
+# components of `breeze start-airflow` (see
scripts/in_container/bin/run_dekit).
+ARG DEKIT_VERSION="0.10.0"
+
+RUN ARCH="$(uname -m)" \
+ &&
DEKIT_URL="https://github.com/pvolok/mprocs/releases/download/v${DEKIT_VERSION}/dekit-${ARCH}-unknown-linux-musl.tar.gz";
\
+ && echo "Downloading dekit from ${DEKIT_URL}" \
+ && curl --silent --location --fail --retry 3 --retry-delay 5 -o
/tmp/dekit.tar.gz "${DEKIT_URL}" \
+ && tar -xzf /tmp/dekit.tar.gz -C /usr/local/bin/ dekit \
+ && rm -f /tmp/dekit.tar.gz \
+ && chmod +x /usr/local/bin/dekit
WORKDIR ${AIRFLOW_SOURCES}
diff --git a/contributing-docs/03_contributors_quick_start.rst
b/contributing-docs/03_contributors_quick_start.rst
index ae53fd1e4f9..fb0df5f3896 100644
--- a/contributing-docs/03_contributors_quick_start.rst
+++ b/contributing-docs/03_contributors_quick_start.rst
@@ -644,7 +644,7 @@ Using Breeze
If Breeze was started with ``breeze start-airflow``, first exit the terminal
multiplexer so that the
Breeze container stops and releases its forwarded ports:
-* With mprocs (the default), press ``q`` in the mprocs interface.
+* With dekit (the default), press ``q`` in the dekit interface.
* With tmux, run ``stop_airflow`` from the main shell pane:
.. code-block:: bash
@@ -665,15 +665,18 @@ If ``breeze`` was started with ``breeze --python 3.10
--backend postgres`` (or s
breeze down
.. note::
- ``stop_airflow`` is available only when ``breeze start-airflow`` uses
tmux. Use ``q`` to quit
- the default mprocs interface.
+ ``stop_airflow`` is available only when ``breeze start-airflow`` uses
tmux. Use ``q`` to leave
+ the default dekit interface.
-Using tmux Instead of mprocs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Using tmux Instead of dekit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
-By default, ``breeze start-airflow`` uses mprocs to manage Airflow components.
You can use ``tmux`` as an
+By default, ``breeze start-airflow`` uses `dekit
<https://github.com/pvolok/mprocs>`__ (the next version of
+mprocs, which Breeze used before) to manage Airflow components. You can use
``tmux`` as an
alternative with the ``--terminal-multiplexer tmux`` argument. The last choice
of yours will be remembered
-for the next run, you can switch back to ``mprocs`` anytime by using
``--terminal-multiplexer mprocs``.
+for the next run, you can switch back to ``dekit`` anytime by using
``--terminal-multiplexer dekit``.
+``mprocs`` is still accepted as another name for ``dekit``, and a remembered
``mprocs`` choice is switched
+to ``dekit`` automatically.
.. code-block:: bash
@@ -683,14 +686,14 @@ and
.. code-block:: bash
- breeze start-airflow --terminal-multiplexer mprocs
+ breeze start-airflow --terminal-multiplexer dekit
You can also switch terminal multiplexer via breeze config:
.. code-block:: bash
breeze setup config --terminal-multiplexer tmux
- breeze setup config --terminal-multiplexer mprocs
+ breeze setup config --terminal-multiplexer dekit
**Benefits of using tmux:**
@@ -700,7 +703,7 @@ You can also switch terminal multiplexer via breeze config:
* Wide range of plugins and extensions
* ability to see more than one log at a time
-**Benefits of using mprocs:**
+**Benefits of using dekit:**
* Modern terminal UI with better visual feedback
* Easier navigation with mouse and keyboard
@@ -708,7 +711,7 @@ You can also switch terminal multiplexer via breeze config:
* Process status indicators
* Better cross-platform support
-For more information on mprocs, look at `mprocs documentation
<mprocs/MPROCS_QUICK_REFERENCE.md>`__.
+For more information on dekit, look at `dekit documentation
<dekit/DEKIT_QUICK_REFERENCE.md>`__.
1. Knowing more about Breeze
diff --git a/contributing-docs/03a_contributors_quick_start_beginners.rst
b/contributing-docs/03a_contributors_quick_start_beginners.rst
index e7540faa64a..f5709b8e19b 100644
--- a/contributing-docs/03a_contributors_quick_start_beginners.rst
+++ b/contributing-docs/03a_contributors_quick_start_beginners.rst
@@ -97,14 +97,14 @@ Option A – Breeze on Your Laptop
breeze start-airflow
-The command starts a shell and launches multiple terminals using ``mprocs`` by
default
+The command starts a shell and launches multiple terminals using ``dekit`` by
default
and launches all Airflow necessary components in those terminals.
You can also choose to use ``tmux`` via the ``--terminal-multiplexer tmux``
option.
If you are using tmux, check out this cheat sheet to learn more about its
commands: https://tmuxcheatsheet.com/.
Now you can also access the Airflow UI on your local machine at
`http://localhost:28080 <http://localhost:28080>`_ with user name ``admin`` and
password ``admin``.
-To exit breeze, press ``q`` in the ``mprocs`` interface (or in any of the tmux
panes) and clean up the resources by running the following command:
+To exit breeze, press ``q`` in the ``dekit`` interface (or in any of the tmux
panes) and clean up the resources by running the following command:
.. code-block:: bash
diff --git a/contributing-docs/20_debugging_airflow_components.rst
b/contributing-docs/20_debugging_airflow_components.rst
index d7798baab1e..12a346926ca 100644
--- a/contributing-docs/20_debugging_airflow_components.rst
+++ b/contributing-docs/20_debugging_airflow_components.rst
@@ -69,21 +69,25 @@ Breeze supports two debugger options:
# Use PyCharm debugger
breeze start-airflow --debug scheduler --debugger pydevd-pycharm
-Using mprocs Instead of tmux
------------------------------
+Choosing dekit or tmux
+----------------------
-By default, ``breeze start-airflow`` uses tmux to manage multiple Airflow
components. You can use
-mprocs as an alternative process manager with the ``--use-mprocs`` flag:
+By default, ``breeze start-airflow`` uses dekit (the next version of mprocs)
to manage multiple Airflow
+components. You can choose the terminal multiplexer with the
``--terminal-multiplexer`` option, and
+Breeze remembers the choice for the next run:
.. code-block:: bash
- # Use mprocs instead of tmux
- breeze start-airflow --use-mprocs
+ # Use dekit (the default)
+ breeze start-airflow --terminal-multiplexer dekit
- # Use mprocs with debugging
- breeze start-airflow --use-mprocs --debug scheduler --debug triggerer
+ # Use dekit with debugging
+ breeze start-airflow --terminal-multiplexer dekit --debug scheduler
--debug triggerer
-**Benefits of mprocs:**
+ # Use tmux instead
+ breeze start-airflow --terminal-multiplexer tmux
+
+**Benefits of dekit:**
* Modern TUI with intuitive navigation
* Better keyboard shortcuts and mouse support
diff --git a/contributing-docs/dekit/DEKIT_QUICK_REFERENCE.md
b/contributing-docs/dekit/DEKIT_QUICK_REFERENCE.md
new file mode 100644
index 00000000000..e8fc400940f
--- /dev/null
+++ b/contributing-docs/dekit/DEKIT_QUICK_REFERENCE.md
@@ -0,0 +1,156 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Quick Reference: dekit Support in Breeze
+
+[dekit](https://github.com/pvolok/mprocs) is the next version of mprocs under
a new name; Breeze
+used mprocs before. The `mprocs` name is still accepted as an alias of `dekit`
wherever Breeze
+takes a terminal multiplexer, and a remembered `mprocs` choice is switched to
`dekit` automatically.
+
+## Basic Command
+
+Start airflow uses dekit by default:
+
+```bash
+breeze start-airflow
+```
+
+You can switch to using `tmux` instead by specifying the option:
+
+```bash
+breeze start-airflow --terminal-multiplexer tmux
+```
+
+Once you select the terminal multiplexer, Breeze will remember your choice for
future runs.
+
+You can also switch terminal multiplexers by running `breeze setup config`:
+
+```bash
+breeze setup config --terminal-multiplexer dekit
+breeze setup config --terminal-multiplexer tmux
+```
+
+Breeze starts the Airflow components as dekit tasks, so they all run in a
single terminal window.
+It generates the `dekit.yaml` configuration from the selected executor and
options; inside the
+container it is in `/tmp/breeze-dekit/dekit.yaml`.
+
+## Common Usage Patterns
+
+| Command | Description
|
+|----------------------------------------------------------------|----------------------------------------------------|
+| `breeze start-airflow --terminal-multiplexer dekit` | Start
Airflow with dekit (and remember the choice) |
+| `breeze start-airflow --terminal-multiplexer tmux` | Start
Airflow with tmux (and remember the choice) |
+| `breeze start-airflow --debug scheduler` | Debug
scheduler with last selected multiplexer |
+| `breeze start-airflow --dev-mode --terminal-multiplexer dekit` | Use dekit
in dev mode (and remember the choice) |
+
+
+## dekit Keyboard Shortcuts
+
+| Key | Action |
+|--------------|-------------------------------------------------|
+| `↑↓` / `jk` | Navigate between tasks |
+| `r` | Restart selected task |
+| `x` | Stop selected task |
+| `s` | Start selected task |
+| `z` | Zoom into the selected task |
+| `Ctrl+a` | Switch focus between the task list and terminal |
+| `q` | Leave dekit (ends the Breeze session) |
+| `Q` | Stop all tasks and leave dekit |
+| `?` | Show all key bindings |
+
+When stopping the Breeze environment, press `q` or `Q` rather than stopping
only the
+selected task. Leaving dekit lets the `breeze start-airflow` container exit
+and release its forwarded ports. After returning to the host shell, run:
+
+```bash
+breeze down
+```
+
+## Components Managed
+
+- **scheduler** - Airflow scheduler
+- **api_server** (3.x+) / **webserver** (2.x) - Web interface
+- **triggerer** - Handles deferred tasks
+- **dag_processor** - Standalone Dag processor (when enabled)
+- **celery_worker** - Celery worker (with CeleryExecutor)
+- **flower** - Celery monitoring (when enabled)
+- **edge_worker** - Edge worker (with EdgeExecutor)
+- **shell** - A bash shell in the container
+
+## Environment Variables
+
+| Variable | Purpose
|
+|----------------------------|--------------------------------------------------|
+| `TERMINAL_MULTIPLEXER` | `dekit` (default, `mprocs` also works) or
`tmux` |
+| `INTEGRATION_CELERY` | Enable Celery components
|
+| `CELERY_FLOWER` | Enable Flower UI
|
+| `STANDALONE_DAG_PROCESSOR` | Enable standalone Dag processor
|
+| `BREEZE_DEBUG_*` | Enable component debugging
|
+| `DEV_MODE` | Enable development mode
|
+
+## Debug Ports (when debugging enabled)
+
+| Component | Port |
+|---------------|--------|
+| Scheduler | 50231 |
+| Dag Processor | 50232 |
+| Triggerer | 50233 |
+| API Server | 50234 |
+| Celery Worker | 50235 |
+| Edge Worker | 50236 |
+| Web Server | 50237 |
+
+## Installation
+
+dekit is **pre-installed** in the Breeze CI image. No additional setup is
required.
+
+For custom setups, you can install it manually:
+
+```bash
+# Download and install dekit (Linux)
+DEKIT_VERSION="0.10.0"
+curl -L
"https://github.com/pvolok/mprocs/releases/download/v${DEKIT_VERSION}/dekit-$(uname
-m)-unknown-linux-musl.tar.gz" \
+ | tar -xz -C /usr/local/bin/ dekit
+chmod +x /usr/local/bin/dekit
+```
+
+## Mac OS X and iTerm2
+
+Mouse clicks are not captured correctly by default in iTerm2 (which is often
used by developers on MacOS).
+You need to configure "Enable Mouse reporting" to take advantage of the
copying feature and mouse handling:
+
+
+
+## Standalone execution
+
+You can run dekit outside of Breeze for custom setups. Put a `dekit.yaml`
defining your tasks in a
+directory, then start the tasks and attach to them from that directory:
+
+```bash
+dekit up
+dekit
+```
+
+`dekit up` starts every task marked `autostart: true`, and `dekit` attaches to
them. The tasks keep
+running when you leave with `q`; `dekit down` stops them.
+
+An example [dekit.yaml](dekit.yaml) file for Airflow components can be found
in this directory - it has
+default configurations for all major Airflow components, but you can customize
it as needed by copying it
+elsewhere and modifying the task definitions, uncommenting the commented or
adding new tasks. See
+[Coming from mprocs](https://dekit.run/docs/start/from-mprocs) for how dekit
differs from mprocs.
diff --git a/contributing-docs/mprocs/mprocs.yaml
b/contributing-docs/dekit/dekit.yaml
similarity index 52%
rename from contributing-docs/mprocs/mprocs.yaml
rename to contributing-docs/dekit/dekit.yaml
index 9565fff493c..9aca7265975 100644
--- a/contributing-docs/mprocs/mprocs.yaml
+++ b/contributing-docs/dekit/dekit.yaml
@@ -15,44 +15,42 @@
# specific language governing permissions and limitations
# under the License.
-# Basic mprocs configuration for Airflow
+# Basic dekit configuration for Airflow
# This is a static example configuration. For dynamic configuration based on
-# your environment, use the generate_mprocs_config.py script from
../in_container/bin.
+# your environment, use the generate_dekit_config.py script from
../../scripts/in_container/bin.
+# Run `dekit up` in the directory with this file to start the tasks, then
`dekit` to attach to them.
---
-procs:
+defaults:
+ autorestart: always
+ scrollback_len: 100000
+
+tasks:
scheduler:
- shell: airflow scheduler
- restart: always
- scrollback: 100000
+ cmd: airflow scheduler
+ autostart: true
api_server:
- shell: airflow api-server
- restart: always
- scrollback: 100000
+ cmd: airflow api-server
+ autostart: true
triggerer:
- shell: airflow triggerer
- restart: always
- scrollback: 100000
+ cmd: airflow triggerer
+ autostart: true
dag_processor:
- shell: airflow dag-processor
- restart: always
- scrollback: 100000
+ cmd: airflow dag-processor
+ autostart: true
+
+ # # Optional: Uncomment to enable Celery worker (requires CeleryExecutor)
+ # celery_worker:
+ # cmd: airflow celery worker
+ # autostart: true
+
+ # # Optional: Uncomment to enable Flower (requires Celery)
+ # flower:
+ # cmd: airflow celery flower
+ # autostart: true
shell:
- shell: bash
- restart: always
- scrollback: 100000
-
- # # Optional: Uncomment to enable Celery worker (requires CeleryExecutor)
- # celery_worker:
- # shell: airflow celery worker
- # restart: always
- # scrollback: 100000
-
- # # Optional: Uncomment to enable Flower (requires Celery)
- # flower:
- # shell: airflow celery flower
- # restart: always
- # scrollback: 100000
+ cmd: bash
+ autostart: true
diff --git a/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md
b/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md
deleted file mode 100644
index 416fed3b482..00000000000
--- a/contributing-docs/mprocs/MPROCS_QUICK_REFERENCE.md
+++ /dev/null
@@ -1,148 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
- -->
-
-# Quick Reference: mprocs Support in Breeze
-
-## Basic Command
-
-Start airflow uses mprocs by default:
-
-```bash
-breeze start-airflow
-```
-
-You can switch to using `tmux` instead by specifying the option:
-
-```bash
-breeze start-airflow --terminal-multiplexer tmux
-```
-
-Once you select the terminal multiplexer, Breeze will remember your choice for
future runs.
-
-You can also switch terminal multiplexers by running `breeze setup config`:
-
-```bash
-breeze setup config --terminal-multiplexer mprocs
-breeze setup config --terminal-multiplexer tmux
-```
-
-
-When Breeze will start Airflow components using mprocs to manage multiple
processes in a single terminal window.
-It will generate dynamically the required configuration based on the selected
executor and options and use it,
-the generated configuration file is stored in a `files` folder inside the
container, so that you can also
-use it outside of Breeze if needed.
-
-## Common Usage Patterns
-
-| Command |
Description |
-|-----------------------------------------------------------------|-----------------------------------------------------|
-| `breeze start-airflow --terminal-multiplexer mprocs` | Start
Airflow with mprocs (and remember the choice) |
-| `breeze start-airflow --terminal-multiplexer tmux` | Start
Airflow with tmux (and remember the choice) |
-| `breeze start-airflow --debug scheduler` | Debug
scheduler with last selected multiplexer |
-| `breeze start-airflow --dev-mode --terminal-multiplexer mprocs` | Use mprocs
in dev mode (and remember the choice) |
-
-
-## mprocs Keyboard Shortcuts
-
-| Key | Action |
-|------|----------------------------|
-| `↑↓` | Navigate between processes |
-| `r` | Restart selected process |
-| `x` | Stop selected process |
-| `s` | Start selected process |
-| `a` | Add new process |
-| `q` | Quit mprocs |
-| `?` | Show help |
-
-When stopping the Breeze environment, press `q` rather than stopping only the
-selected process. Quitting mprocs lets the `breeze start-airflow` container
exit
-and release its forwarded ports. After returning to the host shell, run:
-
-```bash
-breeze down
-```
-
-## Components Managed
-
-- **scheduler** - Airflow scheduler
-- **api_server** (3.x+) / **webserver** (2.x) - Web interface
-- **triggerer** - Handles deferred tasks
-- **dag_processor** - Standalone Dag processor (when enabled)
-- **celery_worker** - Celery worker (with CeleryExecutor)
-- **flower** - Celery monitoring (when enabled)
-- **edge_worker** - Edge worker (with EdgeExecutor)
-
-## Environment Variables
-
-| Variable | Purpose |
-|----------------------------|---------------------------------|
-| `TERMNAL_MULTIPLEXER` | Use mprocs when set to "mprocs" |
-| `INTEGRATION_CELERY` | Enable Celery components |
-| `CELERY_FLOWER` | Enable Flower UI |
-| `STANDALONE_DAG_PROCESSOR` | Enable standalone Dag processor |
-| `BREEZE_DEBUG_*` | Enable component debugging |
-| `DEV_MODE` | Enable development mode |
-
-## Debug Ports (when debugging enabled)
-
-| Component | Port |
-|---------------|--------|
-| Scheduler | 50231 |
-| Dag Processor | 50232 |
-| Triggerer | 50233 |
-| API Server | 50234 |
-| Celery Worker | 50235 |
-| Edge Worker | 50236 |
-| Web Server | 50237 |
-
-## Installation
-
-mprocs is **pre-installed** in the Breeze CI image as of Airflow 3.x
development. No additional setup is required.
-
-For older images or custom setups, you can install mprocs manually:
-
-```bash
-# Download and install mprocs
-MPROCS_VERSION="0.7.3"
-SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]')
-PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64")
-curl -L
"https://github.com/pvolok/mprocs/releases/download/v${MPROCS_VERSION}/mprocs-${MPROCS_VERSION}-${SYSTEM}-${PLATFORM}.tar.gz";
\
- | tar -xz -C /usr/local/bin/ mprocs
-chmod +x /usr/local/bin/mprocs
-```
-
-## Mac OS X and iTerm2
-
-Mouse clicks are not captured correctly by default in iTerm2 (which is often
used by developers on MacOS).
-You need to configure "Enable Mouse reporting" to take advantage of the
copying feature and mouse handling:
-
-
-
-## Standalone execution
-
-You can run mprocs outside of Breeze for custom setups. Create a `mprocs.yaml`
configuration
-file defining your processes, then start mprocs:
-
-```bash
-mprocs -f mprocs.yaml
-```
-
-An example [mprocs.yaml](mprocs.yaml) file for Airflow components can be found
in this directory - it has
-default configurations for all major Airflow components, but you can customize
it as needed by copying it
-elsewhere and modifying the process definitions, uncommenting the commented or
adding new processes.
diff --git a/dev/README_RELEASE_PROVIDERS.md b/dev/README_RELEASE_PROVIDERS.md
index eb8097f93e0..abb62392c2b 100644
--- a/dev/README_RELEASE_PROVIDERS.md
+++ b/dev/README_RELEASE_PROVIDERS.md
@@ -1660,7 +1660,7 @@ pip install
apache-airflow-providers-<provider>==<VERSION>rc<X>
```
NOTE! After installing the provider package, restart the Airflow components so
the new provider is loaded.
-If you started Breeze with `breeze start-airflow`, in the terminal multiplexer
(mprocs or tmux)
+If you started Breeze with `breeze start-airflow`, in the terminal multiplexer
(dekit or tmux)
use the keyboard shortcuts to **stop** and then **start** each managed
component:
**scheduler**, **api_server**, **triggerer**, and **dag_processor**.
diff --git a/dev/breeze/doc/03_developer_tasks.rst
b/dev/breeze/doc/03_developer_tasks.rst
index b94b4b01727..2ba9135d614 100644
--- a/dev/breeze/doc/03_developer_tasks.rst
+++ b/dev/breeze/doc/03_developer_tasks.rst
@@ -764,7 +764,7 @@ you are running on your host.
If you used ``breeze start-airflow``, first exit its terminal multiplexer so
that the foreground
container releases its forwarded ports:
-* With mprocs (the default), press ``q`` in the mprocs interface.
+* With dekit (the default), press ``q`` in the dekit interface.
* With tmux, run ``stop_airflow`` from its main shell pane.
After returning to the host shell, stop the remaining Docker Compose services:
diff --git a/dev/breeze/doc/images/output_setup_config.svg
b/dev/breeze/doc/images/output_setup_config.svg
index 9f06770cc98..d19cdcc229f 100644
--- a/dev/breeze/doc/images/output_setup_config.svg
+++ b/dev/breeze/doc/images/output_setup_config.svg
@@ -174,7 +174,7 @@
</text><text class="breeze-setup-config-r5" x="0" y="386" textLength="12.2"
clip-path="url(#breeze-setup-config-line-15)">│</text><text
class="breeze-setup-config-r7" x="475.8" y="386" textLength="549"
clip-path="url(#breeze-setup-config-line-15)">(>sqlite< | mysql | postgres | none | custom)</text><text
class="breeze-setup-config-r5" x="1451.8" y="386" textLength="12.2"
clip-path="url(#breeze-setup-config-line-15)">│</text><text class="breez [...]
</text><text class="breeze-setup-config-r5" x="0" y="410.4" textLength="12.2"
clip-path="url(#breeze-setup-config-line-16)">│</text><text
class="breeze-setup-config-r4" x="24.4" y="410.4" textLength="341.6"
clip-path="url(#breeze-setup-config-line-16)">--postgres-version          </text><text
class="breeze-setup-config-r6" x="390.4" y="410.4" textLength="61"
clip-path="url(#breeze-setup-config-line-16)">-P   </text><text
cl [...]
</text><text class="breeze-setup-config-r5" x="0" y="434.8" textLength="12.2"
clip-path="url(#breeze-setup-config-line-17)">│</text><text
class="breeze-setup-config-r4" x="24.4" y="434.8" textLength="341.6"
clip-path="url(#breeze-setup-config-line-17)">--mysql-version             </text><text
class="breeze-setup-config-r6" x="390.4" y="434.8" textLength="61"
clip-path="url(#breeze-setup-config-line-17)">-M    [...]
-</text><text class="breeze-setup-config-r5" x="0" y="459.2" textLength="12.2"
clip-path="url(#breeze-setup-config-line-18)">│</text><text
class="breeze-setup-config-r4" x="24.4" y="459.2" textLength="341.6"
clip-path="url(#breeze-setup-config-line-18)">--terminal-multiplexer      </text><text
class="breeze-setup-config-r6" x="390.4" y="459.2" textLength="61"
clip-path="url(#breeze-setup-config-line-18)">-t   </text><text
class="breeze-setup-co [...]
+</text><text class="breeze-setup-config-r5" x="0" y="459.2" textLength="12.2"
clip-path="url(#breeze-setup-config-line-18)">│</text><text
class="breeze-setup-config-r4" x="24.4" y="459.2" textLength="341.6"
clip-path="url(#breeze-setup-config-line-18)">--terminal-multiplexer      </text><text
class="breeze-setup-config-r6" x="390.4" y="459.2" textLength="61"
clip-path="url(#breeze-setup-config-line-18)">-t   </text><text
class="breeze-setup-co [...]
</text><text class="breeze-setup-config-r5" x="0" y="483.6" textLength="12.2"
clip-path="url(#breeze-setup-config-line-19)">│</text><text
class="breeze-setup-config-r4" x="24.4" y="483.6" textLength="341.6"
clip-path="url(#breeze-setup-config-line-19)">--auth-manager              </text><text
class="breeze-setup-config-r1" x="475.8" y="483.6" textLength="390.4"
clip-path="url(#breeze-setup-config-line-19)">Specify  [...]
</text><text class="breeze-setup-config-r5" x="0" y="508" textLength="12.2"
clip-path="url(#breeze-setup-config-line-20)">│</text><text
class="breeze-setup-config-r7" x="475.8" y="508" textLength="463.6"
clip-path="url(#breeze-setup-config-line-20)">(>SimpleAuthManager< | FabAuthManager)</text><text
class="breeze-setup-config-r5" x="1451.8" y="508" textLength="12.2"
clip-path="url(#breeze-setup-config-line-20)">│</text><text
class="breeze-setup-config-r1" x="1464" y="508" [...]
</text><text class="breeze-setup-config-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-setup-config-line-21)">│</text><text
class="breeze-setup-config-r4" x="24.4" y="532.4" textLength="341.6"
clip-path="url(#breeze-setup-config-line-21)">--llm-model                 </text><text
class="breeze-setup-config-r1" x="475.8" y="532.4" textLength="963.8"
clip-path="url(#breeze-setup-config-line-21 [...]
diff --git a/dev/breeze/doc/images/output_setup_config.txt
b/dev/breeze/doc/images/output_setup_config.txt
index 4bed9f321be..c78a138882b 100644
--- a/dev/breeze/doc/images/output_setup_config.txt
+++ b/dev/breeze/doc/images/output_setup_config.txt
@@ -1 +1 @@
-4fc5026be69f9e924d243365bdaa993d
+7de551475d94895c673732562a5ae6ab
diff --git a/dev/breeze/doc/images/output_start-airflow.svg
b/dev/breeze/doc/images/output_start-airflow.svg
index 92ae687182f..5e2db6e15a6 100644
--- a/dev/breeze/doc/images/output_start-airflow.svg
+++ b/dev/breeze/doc/images/output_start-airflow.svg
@@ -505,8 +505,8 @@
</text><text class="breeze-start-airflow-r5" x="0" y="459.2" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-18)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="459.2" textLength="317.2"
clip-path="url(#breeze-start-airflow-line-18)">--standalone-dag-processor</text><text
class="breeze-start-airflow-r1" x="341.6" y="459.2" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-18)">/</text><text
class="breeze-start-airflow-r4" x="353.8" y="459.2" textLength="3 [...]
</text><text class="breeze-start-airflow-r5" x="0" y="483.6" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-19)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="483.6" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-19)">r</text><text
class="breeze-start-airflow-r1" x="768.6" y="483.6" textLength="317.2"
clip-path="url(#breeze-start-airflow-line-19)">(required for Airflow 3). </text><text
class="breeze-start-airflow-r5" x="1085.8" y= [...]
</text><text class="breeze-start-airflow-r5" x="0" y="508" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-20)">│</text><text
class="breeze-start-airflow-r5" x="768.6" y="508" textLength="305"
clip-path="url(#breeze-start-airflow-line-20)">standalone-dag-processor]</text><text
class="breeze-start-airflow-r5" x="1451.8" y="508" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-20)">│</text><text
class="breeze-start-airflow-r1" x="1464" y="508" textLength="12.2" clip- [...]
-</text><text class="breeze-start-airflow-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-21)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="532.4" textLength="671"
clip-path="url(#breeze-start-airflow-line-21)">--terminal-multiplexer                                 </text><text
clas [...]
-</text><text class="breeze-start-airflow-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-22)">│</text><text
class="breeze-start-airflow-r7" x="768.6" y="556.8" textLength="207.4"
clip-path="url(#breeze-start-airflow-line-22)">(>mprocs< | tmux)</text><text
class="breeze-start-airflow-r5" x="1451.8" y="556.8" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-22)">│</text><text
class="breeze-start-airflow-r1" x="1464" y="556.8" textL [...]
+</text><text class="breeze-start-airflow-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-21)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="532.4" textLength="671"
clip-path="url(#breeze-start-airflow-line-21)">--terminal-multiplexer                                 </text><text
clas [...]
+</text><text class="breeze-start-airflow-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-22)">│</text><text
class="breeze-start-airflow-r7" x="768.6" y="556.8" textLength="195.2"
clip-path="url(#breeze-start-airflow-line-22)">(>dekit< | tmux)</text><text
class="breeze-start-airflow-r5" x="1451.8" y="556.8" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-22)">│</text><text
class="breeze-start-airflow-r1" x="1464" y="556.8" textLe [...]
</text><text class="breeze-start-airflow-r5" x="0" y="581.2" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-23)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="581.2" textLength="671"
clip-path="url(#breeze-start-airflow-line-23)">--auth-manager                                     &
[...]
</text><text class="breeze-start-airflow-r5" x="0" y="605.6" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-24)">│</text><text
class="breeze-start-airflow-r5" x="768.6" y="605.6" textLength="219.6"
clip-path="url(#breeze-start-airflow-line-24)">SimpleAuthManager]</text><text
class="breeze-start-airflow-r7" x="1000.4" y="605.6" textLength="280.6"
clip-path="url(#breeze-start-airflow-line-24)">(>SimpleAuthManager< | </text><text
class="breeze-start-airflow-r5" [...]
</text><text class="breeze-start-airflow-r5" x="0" y="630" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-25)">│</text><text
class="breeze-start-airflow-r7" x="768.6" y="630" textLength="183"
clip-path="url(#breeze-start-airflow-line-25)">FabAuthManager)</text><text
class="breeze-start-airflow-r5" x="1451.8" y="630" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-25)">│</text><text
class="breeze-start-airflow-r1" x="1464" y="630" textLength="12.2"
clip-path="url( [...]
diff --git a/dev/breeze/doc/images/output_start-airflow.txt
b/dev/breeze/doc/images/output_start-airflow.txt
index b6d888c9fcb..69f9857b548 100644
--- a/dev/breeze/doc/images/output_start-airflow.txt
+++ b/dev/breeze/doc/images/output_start-airflow.txt
@@ -1 +1 @@
-14195584ab4f2240677b66688c848ba8
+09f22d94d14c4bc012afd6f3d78cae5c
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index ea254d1fb0c..224c23279fe 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -176,7 +176,11 @@ AUTOCOMPLETE_ALL_INTEGRATIONS = sorted(
]
)
ALLOWED_TTY = ["auto", "enabled", "disabled"]
-ALLOWED_TERMINAL_MULTIPLEXERS = ["mprocs", "tmux"]
+ALLOWED_TERMINAL_MULTIPLEXERS = ["dekit", "tmux"]
+# Old names of a choice, still accepted as hidden aliases of the new value: on
the command line and in
+# environment variables they mean the new value, and a cached old value is
migrated to the new one.
+# mprocs is released as dekit since 0.10.0; Breeze runs dekit in its mprocs
mode.
+TERMINAL_MULTIPLEXER_ALIASES = {"mprocs": "dekit"}
ALLOWED_DOCKER_COMPOSE_PROJECTS = [
"breeze",
"breeze-prek",
diff --git a/dev/breeze/src/airflow_breeze/utils/cache.py
b/dev/breeze/src/airflow_breeze/utils/cache.py
index 533d4f92eae..00b94fdf4ea 100644
--- a/dev/breeze/src/airflow_breeze/utils/cache.py
+++ b/dev/breeze/src/airflow_breeze/utils/cache.py
@@ -37,10 +37,24 @@ def check_if_cache_exists(param_name: str) -> bool:
return (Path(BUILD_CACHE_PATH) / f".{param_name}").exists()
+def resolve_value_alias(param_name: str, param_value: str) -> str:
+ """Return the current value for a hidden alias (an old name) of a choice,
else the value itself."""
+ aliases: dict[str, str] = getattr(global_constants,
f"{param_name.upper()}_ALIASES", {})
+ return aliases.get(param_value, param_value)
+
+
def read_from_cache_file(param_name: str) -> str | None:
cache_exists = check_if_cache_exists(param_name)
if cache_exists:
- return (Path(BUILD_CACHE_PATH) / f".{param_name}").read_text().strip()
+ cached_value = (Path(BUILD_CACHE_PATH) /
f".{param_name}").read_text().strip()
+ resolved_value = resolve_value_alias(param_name, cached_value)
+ if resolved_value != cached_value:
+ # The choice was renamed: keep the user's preference under its new
name.
+ console_print(
+ f"[info]Migrating the cached {param_name} value {cached_value}
to {resolved_value}.[/]"
+ )
+ write_to_cache_file(param_name, resolved_value,
check_allowed_values=False)
+ return resolved_value
return None
diff --git a/dev/breeze/src/airflow_breeze/utils/custom_param_types.py
b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py
index 93f7ccd660c..82204399265 100644
--- a/dev/breeze/src/airflow_breeze/utils/custom_param_types.py
+++ b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py
@@ -30,6 +30,7 @@ from airflow_breeze.utils.cache import (
check_if_values_allowed,
read_and_validate_value_from_cache,
read_from_cache_file,
+ resolve_value_alias,
write_to_cache_file,
)
from airflow_breeze.utils.coertions import coerce_bool_value
@@ -170,6 +171,7 @@ class CacheableChoice(click.Choice):
if not is_cached:
console_print(f"\n[info]Default value of {param.name}
parameter {new_value} used.[/]\n")
else:
+ value = resolve_value_alias(param_name, value)
allowed, allowed_values = check_if_values_allowed(param_name,
value)
if allowed:
new_value = value
diff --git a/dev/breeze/tests/test_cache.py b/dev/breeze/tests/test_cache.py
index e8b9a3f4f91..dcafe2293f0 100644
--- a/dev/breeze/tests/test_cache.py
+++ b/dev/breeze/tests/test_cache.py
@@ -19,14 +19,22 @@ from __future__ import annotations
from pathlib import Path
from unittest import mock
+import click
import pytest
+from click.testing import CliRunner
-from airflow_breeze.global_constants import ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS
+from airflow_breeze.commands.common_options import option_terminal_multiplexer
+from airflow_breeze.global_constants import (
+ ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
+ ALLOWED_TERMINAL_MULTIPLEXERS,
+ TERMINAL_MULTIPLEXER_ALIASES,
+)
from airflow_breeze.utils.cache import (
check_if_cache_exists,
check_if_values_allowed,
delete_cache,
read_from_cache_file,
+ resolve_value_alias,
)
AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
@@ -90,3 +98,54 @@ def test_delete_cache_not_exists(mock_check_if_cache_exists,
mock_path):
mock_check_if_cache_exists.return_value = False
cache_deleted = delete_cache(param)
assert not cache_deleted
+
+
[email protected](
+ ("param_name", "value", "expected"),
+ [
+ ("TERMINAL_MULTIPLEXER", "mprocs", "dekit"),
+ ("TERMINAL_MULTIPLEXER", "dekit", "dekit"),
+ ("TERMINAL_MULTIPLEXER", "tmux", "tmux"),
+ ("BACKEND", "mprocs", "mprocs"),
+ ],
+)
+def test_resolve_value_alias(param_name, value, expected):
+ assert resolve_value_alias(param_name, value) == expected
+
+
+def test_mprocs_is_not_offered_as_a_choice():
+ assert "mprocs" not in ALLOWED_TERMINAL_MULTIPLEXERS
+ assert TERMINAL_MULTIPLEXER_ALIASES["mprocs"] in
ALLOWED_TERMINAL_MULTIPLEXERS
+
+
+def test_cached_old_name_is_migrated(monkeypatch, tmp_path):
+ monkeypatch.setattr("airflow_breeze.utils.cache.BUILD_CACHE_PATH",
tmp_path)
+ (tmp_path / ".TERMINAL_MULTIPLEXER").write_text("mprocs")
+
+ assert read_from_cache_file("TERMINAL_MULTIPLEXER") == "dekit"
+ assert (tmp_path / ".TERMINAL_MULTIPLEXER").read_text() == "dekit"
+
+
+def test_cached_current_value_is_left_alone(monkeypatch, tmp_path):
+ monkeypatch.setattr("airflow_breeze.utils.cache.BUILD_CACHE_PATH",
tmp_path)
+ (tmp_path / ".TERMINAL_MULTIPLEXER").write_text("tmux")
+
+ assert read_from_cache_file("TERMINAL_MULTIPLEXER") == "tmux"
+ assert (tmp_path / ".TERMINAL_MULTIPLEXER").read_text() == "tmux"
+
+
[email protected]("given", ["mprocs", "dekit"])
+def test_option_accepts_the_old_name_and_remembers_the_new_one(monkeypatch,
tmp_path, given):
+ monkeypatch.setattr("airflow_breeze.utils.cache.BUILD_CACHE_PATH",
tmp_path)
+ monkeypatch.delenv("SKIP_SAVING_CHOICES", raising=False)
+
+ @click.command()
+ @option_terminal_multiplexer
+ def command(terminal_multiplexer):
+ click.echo(terminal_multiplexer)
+
+ result = CliRunner().invoke(command, ["--terminal-multiplexer", given])
+
+ assert result.exit_code == 0, result.output
+ assert result.output.strip().splitlines()[-1] == "dekit"
+ assert (tmp_path / ".TERMINAL_MULTIPLEXER").read_text() == "dekit"
diff --git a/scripts/ci/prek/upgrade_important_versions.py
b/scripts/ci/prek/upgrade_important_versions.py
index 1124935d80c..44e77ba8129 100755
--- a/scripts/ci/prek/upgrade_important_versions.py
+++ b/scripts/ci/prek/upgrade_important_versions.py
@@ -766,7 +766,7 @@ UPGRADE_GITPYTHON: bool = get_env_bool("UPGRADE_GITPYTHON")
UPGRADE_GOLANG: bool = get_env_bool("UPGRADE_GOLANG")
UPGRADE_HATCH: bool = get_env_bool("UPGRADE_HATCH")
UPGRADE_HATCHLING: bool = get_env_bool("UPGRADE_HATCHLING")
-UPGRADE_MPROCS: bool = get_env_bool("UPGRADE_MPROCS")
+UPGRADE_DEKIT: bool = get_env_bool("UPGRADE_DEKIT")
UPGRADE_NODE_LTS: bool = get_env_bool("UPGRADE_NODE_LTS")
UPGRADE_PIP: bool = get_env_bool("UPGRADE_PIP")
UPGRADE_PREK: bool = get_env_bool("UPGRADE_PREK")
@@ -885,8 +885,8 @@ SIMPLE_VERSION_PATTERNS: dict[str, list[tuple[str, str]]] =
{
"protoc": [
(r"(rvolosatovs/protoc:)(v[0-9.]+)", "rvolosatovs/protoc:{version}"),
],
- "mprocs": [
- (r"(ARG MPROCS_VERSION=)(\"[0-9.]+\")", 'ARG
MPROCS_VERSION="{version}"'),
+ "dekit": [
+ (r"(ARG DEKIT_VERSION=)(\"[0-9.]+\")", 'ARG
DEKIT_VERSION="{version}"'),
],
"openapi_generator": [
(r"(OPENAPI_GENERATOR_CLI_VER = )(\"[0-9.]+\")",
'OPENAPI_GENERATOR_CLI_VER = "{version}"'),
@@ -942,7 +942,8 @@ def fetch_all_package_versions() -> dict[str, str]:
"protoc": get_latest_image_version("rvolosatovs/protoc") if
UPGRADE_PROTOC else "",
"alpine": get_latest_image_version("alpine") if UPGRADE_ALPINE else "",
"busybox": get_latest_image_version("busybox") if UPGRADE_BUSYBOX else
"",
- "mprocs": get_latest_github_release_version("pvolok/mprocs") if
UPGRADE_MPROCS else "",
+ # dekit is still released from the mprocs repository.
+ "dekit": get_latest_github_release_version("pvolok/mprocs") if
UPGRADE_DEKIT else "",
"openapi_generator": get_latest_openapi_generator_version() if
UPGRADE_OPENAPI_GENERATOR else "",
"sphinx_airflow_theme": get_latest_sphinx_airflow_theme_version()
if UPGRADE_SPHINX_AIRFLOW_THEME
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index b27b986df65..ca076ee7024 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -246,9 +246,10 @@ SSH_CONFIG
if [[ ${SKIP_ASSETS_COMPILATION:="false"} == "false" ]]; then
wait_for_asset_compilation
fi
- if [[ ${TERMINAL_MULTIPLEXER:="mprocs"} == "mprocs" ]]; then
- # shellcheck source=scripts/in_container/bin/run_mprocs
- exec run_mprocs
+ # "mprocs" is the old name of dekit, still accepted from older Breeze
versions.
+ if [[ ${TERMINAL_MULTIPLEXER:="dekit"} == "dekit" ||
${TERMINAL_MULTIPLEXER} == "mprocs" ]]; then
+ # shellcheck source=scripts/in_container/bin/run_dekit
+ exec run_dekit
else
# shellcheck source=scripts/in_container/bin/run_tmux
exec run_tmux
diff --git a/scripts/in_container/bin/generate_mprocs_config.py
b/scripts/in_container/bin/generate_dekit_config.py
similarity index 81%
rename from scripts/in_container/bin/generate_mprocs_config.py
rename to scripts/in_container/bin/generate_dekit_config.py
index 823865da19a..7e8c1d7094d 100755
--- a/scripts/in_container/bin/generate_mprocs_config.py
+++ b/scripts/in_container/bin/generate_dekit_config.py
@@ -22,7 +22,7 @@
# "PyYAML>=6.0",
# ]
# ///
-"""Generate mprocs configuration dynamically based on environment variables."""
+"""Generate the dekit configuration (``dekit.yaml``) for ``breeze
start-airflow`` from environment variables."""
from __future__ import annotations
@@ -46,9 +46,9 @@ def get_env(var_name: str, default: str = "") -> str:
return os.environ.get(var_name, default)
-def generate_mprocs_config() -> str:
- """Generate mprocs YAML configuration based on environment variables."""
- procs = {}
+def get_component_commands() -> dict[str, str]:
+ """Return the shell command of each Airflow component to run, by task
name."""
+ procs: dict[str, str] = {}
# Scheduler
scheduler_cmd = "airflow scheduler"
@@ -56,11 +56,7 @@ def generate_mprocs_config() -> str:
port = get_env("BREEZE_DEBUG_SCHEDULER_PORT", "5678")
scheduler_cmd = f"debugpy --listen 0.0.0.0:{port} --wait-for-client -m
airflow scheduler"
- procs["scheduler"] = {
- "shell": scheduler_cmd,
- "restart": "always",
- "scrollback": 100000,
- }
+ procs["scheduler"] = scheduler_cmd
# API Server or Webserver (depending on Airflow version)
use_airflow_version = get_env("USE_AIRFLOW_VERSION", "")
@@ -84,7 +80,7 @@ def generate_mprocs_config() -> str:
api_cmd = (
f"airflow api-server {api_host_arg} -d" if dev_mode else
f"airflow api-server {api_host_arg}"
)
- procs["api_server"] = {"shell": api_cmd, "restart": "always",
"scrollback": 100000}
+ procs["api_server"] = api_cmd
else:
# Webserver (Airflow 2.x)
if get_env_bool("BREEZE_DEBUG_WEBSERVER"):
@@ -94,11 +90,7 @@ def generate_mprocs_config() -> str:
dev_mode = get_env_bool("DEV_MODE")
web_cmd = "airflow webserver -d" if dev_mode else "airflow
webserver"
- procs["webserver"] = {
- "shell": web_cmd,
- "restart": "always",
- "scrollback": 100000,
- }
+ procs["webserver"] = web_cmd
# Triggerer
triggerer_cmd = "airflow triggerer"
@@ -106,11 +98,7 @@ def generate_mprocs_config() -> str:
port = get_env("BREEZE_DEBUG_TRIGGERER_PORT", "5681")
triggerer_cmd = f"debugpy --listen 0.0.0.0:{port} --wait-for-client -m
airflow triggerer"
- procs["triggerer"] = {
- "shell": triggerer_cmd,
- "restart": "always",
- "scrollback": 100000,
- }
+ procs["triggerer"] = triggerer_cmd
# Celery Worker (conditional)
if get_env_bool("INTEGRATION_CELERY"):
@@ -120,11 +108,7 @@ def generate_mprocs_config() -> str:
else:
celery_cmd = "airflow celery worker"
- procs["celery_worker"] = {
- "shell": celery_cmd,
- "restart": "always",
- "scrollback": 100000,
- }
+ procs["celery_worker"] = celery_cmd
# Flower (conditional)
if get_env_bool("INTEGRATION_CELERY") and get_env_bool("CELERY_FLOWER"):
@@ -134,11 +118,7 @@ def generate_mprocs_config() -> str:
else:
flower_cmd = "airflow celery flower"
- procs["flower"] = {
- "shell": flower_cmd,
- "restart": "always",
- "scrollback": 100000,
- }
+ procs["flower"] = flower_cmd
# Edge Worker (conditional)
executor = get_env("AIRFLOW__CORE__EXECUTOR", "")
@@ -159,7 +139,7 @@ def generate_mprocs_config() -> str:
]
edge_cmd = " && ".join(edge_cmd_parts)
- procs["edge_worker"] = {"shell": edge_cmd, "restart": "always",
"scrollback": 100000}
+ procs["edge_worker"] = edge_cmd
# Dag Processor (conditional)
if get_env_bool("STANDALONE_DAG_PROCESSOR"):
@@ -169,20 +149,28 @@ def generate_mprocs_config() -> str:
else:
dag_proc_cmd = "airflow dag-processor"
- procs["dag_processor"] = {
- "shell": dag_proc_cmd,
- "restart": "always",
- "scrollback": 100000,
- }
+ procs["dag_processor"] = dag_proc_cmd
- procs["shell"] = {
- "shell": "bash",
- "restart": "always",
- "scrollback": 100000,
- }
+ procs["shell"] = "bash"
+ return procs
- # Generate YAML output
- config_dict = {"procs": procs}
+
+def generate_dekit_config(cwd: str) -> str:
+ """Generate the ``dekit.yaml`` that runs every component as an
autostarted, always restarted task."""
+ tasks = {
+ # dekit runs a command without a shell, so each runs through bash as
it did under mprocs.
+ name: {"cmd": ["bash", "-c", command], "autostart": True}
+ for name, command in get_component_commands().items()
+ }
+ config_dict = {
+ "defaults": {
+ # The tasks run where Breeze starts them, not in the dekit project
directory.
+ "cwd": cwd,
+ "autorestart": "always",
+ "scrollback_len": 100000,
+ },
+ "tasks": tasks,
+ }
return yaml.dump(config_dict, default_flow_style=False, sort_keys=False)
@@ -193,14 +181,14 @@ def main():
os.environ["AIRFLOW__CORE__EXECUTOR"] = "LocalExecutor"
# Generate and print configuration
- config = generate_mprocs_config()
+ config = generate_dekit_config(cwd=os.getcwd())
# Determine output path
if len(sys.argv) > 1:
output_path = sys.argv[1]
else:
temp_dir = tempfile.gettempdir()
- output_path = os.path.join(temp_dir, "mprocs.yaml")
+ output_path = os.path.join(temp_dir, "dekit.yaml")
with open(output_path, "w") as f:
f.write(config)
@@ -210,7 +198,7 @@ def main():
console = Console()
console.print(
- f"\n[bold green]✓[/bold green] Generated mprocs configuration at:
[cyan]{output_path}[/cyan]"
+ f"\n[bold green]✓[/bold green] Generated dekit configuration at:
[cyan]{output_path}[/cyan]"
)
# Display configuration with syntax highlighting
diff --git a/scripts/in_container/bin/run_dekit
b/scripts/in_container/bin/run_dekit
new file mode 100755
index 00000000000..1b4bcab3870
--- /dev/null
+++ b/scripts/in_container/bin/run_dekit
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Run the Airflow components as tasks of a dekit project. The project lives in
the container, not in
+# the mounted /files, so no runner state from an earlier Breeze session is
picked up.
+DEKIT_PROJECT_DIR="/tmp/breeze-dekit"
+mkdir -p "${DEKIT_PROJECT_DIR}"
+
+if ! uv run "/opt/airflow/scripts/in_container/bin/generate_dekit_config.py"
"${DEKIT_PROJECT_DIR}/dekit.yaml"; then
+ echo "Failed to generate dekit configuration"
+ exit 1
+fi
+
+# The task list title is a user setting in dekit; set it unless the user has
their own config.
+DEKIT_USER_CONFIG="${XDG_CONFIG_HOME:-${HOME}/.config}/dekit/config.yaml"
+if [[ ! -f ${DEKIT_USER_CONFIG} ]]; then
+ mkdir -p "$(dirname "${DEKIT_USER_CONFIG}")"
+ printf 'tui:\n sidebar:\n title: Apache Airflow\n' >
"${DEKIT_USER_CONFIG}"
+fi
+
+# Start the runner with every task, then attach to its console. In the
console, q detaches, which
+# ends the Breeze session, and Q stops the runner first.
+dekit up -C "${DEKIT_PROJECT_DIR}"
+exec dekit -C "${DEKIT_PROJECT_DIR}"
diff --git a/scripts/in_container/bin/run_mprocs
b/scripts/in_container/bin/run_mprocs
deleted file mode 100755
index 778b393eb7a..00000000000
--- a/scripts/in_container/bin/run_mprocs
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# Generate mprocs configuration dynamically
-
-MPROCS_CONFIG_PATH="/files/mprocs.yaml"
-
-if ! uv run "/opt/airflow/scripts/in_container/bin/generate_mprocs_config.py"
"${MPROCS_CONFIG_PATH}"; then
- echo "Failed to generate mprocs configuration"
- exit 1
-fi
-
-# Start mprocs with the generated configuration
-exec mprocs --config "${MPROCS_CONFIG_PATH}" --proc-list-title "Apache Airflow"
diff --git a/scripts/tests/in_container/test_generate_dekit_config.py
b/scripts/tests/in_container/test_generate_dekit_config.py
new file mode 100644
index 00000000000..1ba09882a66
--- /dev/null
+++ b/scripts/tests/in_container/test_generate_dekit_config.py
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+import yaml
+
+_GENERATOR = Path(__file__).parents[2] / "in_container" / "bin" /
"generate_dekit_config.py"
+_spec = importlib.util.spec_from_file_location("generate_dekit_config",
_GENERATOR)
+assert _spec is not None
+assert _spec.loader is not None
+generator = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(generator)
+
+_COMPONENT_ENV_VARS = (
+ "USE_AIRFLOW_VERSION",
+ "INTEGRATION_CELERY",
+ "CELERY_FLOWER",
+ "STANDALONE_DAG_PROCESSOR",
+ "AIRFLOW__CORE__EXECUTOR",
+ "DEV_MODE",
+)
+
+
[email protected](autouse=True)
+def _clean_env(monkeypatch):
+ for name in _COMPONENT_ENV_VARS:
+ monkeypatch.delenv(name, raising=False)
+
+
+def _config(cwd: str = "/opt/airflow") -> dict:
+ return yaml.safe_load(generator.generate_dekit_config(cwd=cwd))
+
+
+def test_every_component_is_an_autostarted_task_run_through_bash():
+ config = _config()
+
+ assert config["defaults"] == {"cwd": "/opt/airflow", "autorestart":
"always", "scrollback_len": 100000}
+ assert list(config["tasks"]) == ["scheduler", "api_server", "triggerer",
"shell"]
+ for task in config["tasks"].values():
+ assert task["autostart"] is True
+ assert task["cmd"][:2] == ["bash", "-c"]
+ assert config["tasks"]["scheduler"]["cmd"][2] == "airflow scheduler"
+ assert config["tasks"]["shell"]["cmd"][2] == "bash"
+
+
+def test_optional_components_follow_the_environment(monkeypatch):
+ monkeypatch.setenv("INTEGRATION_CELERY", "true")
+ monkeypatch.setenv("CELERY_FLOWER", "true")
+ monkeypatch.setenv("STANDALONE_DAG_PROCESSOR", "true")
+ monkeypatch.setenv(
+ "AIRFLOW__CORE__EXECUTOR",
"airflow.providers.edge3.executors.edge_executor.EdgeExecutor"
+ )
+
+ tasks = _config()["tasks"]
+
+ assert {"celery_worker", "flower", "edge_worker", "dag_processor"} <=
set(tasks)
+ # The edge worker unsets variables first, which needs the shell.
+ assert "unset AIRFLOW__DATABASE__SQL_ALCHEMY_CONN || true &&" in
tasks["edge_worker"]["cmd"][2]
+
+
+def test_airflow_2_runs_the_webserver(monkeypatch):
+ monkeypatch.setenv("USE_AIRFLOW_VERSION", "2.11.0")
+
+ tasks = _config()["tasks"]
+
+ assert "webserver" in tasks
+ assert "api_server" not in tasks
+
+
+def test_no_mprocs_keys_are_left():
+ config = _config()
+
+ assert "procs" not in config
+ for task in config["tasks"].values():
+ assert not {"shell", "restart", "scrollback"} & set(task)
