This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c25b89b0c93 Add push_logs configuration option to Edge executor
(#58125)
c25b89b0c93 is described below
commit c25b89b0c937b2a34ebdd04caf2a6e7443927571
Author: Dheeraj Turaga <[email protected]>
AuthorDate: Tue Nov 11 12:23:46 2025 -0600
Add push_logs configuration option to Edge executor (#58125)
* Add push_logs configuration option to Edge executor
Adds a new boolean configuration option AIRFLOW__EDGE__PUSH_LOGS
to control whether edge workers push log files to the central site.
The option is set to True by default to maintain existing behavior.
When disabled, edge workers will keep logs locally without uploading
them to the central Airflow site, which can be useful for:
- Reducing network bandwidth usage
- Keeping logs on edge sites for compliance/security reasons
- Troubleshooting network connectivity issues
* Add version_added: 1.5.0 to push_logs configuration option
---
providers/edge3/provider.yaml | 9 +++++++++
providers/edge3/src/airflow/providers/edge3/cli/worker.py | 6 +++++-
providers/edge3/src/airflow/providers/edge3/get_provider_info.py | 7 +++++++
3 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/providers/edge3/provider.yaml b/providers/edge3/provider.yaml
index a4698c04ef7..cb2cda5e53b 100644
--- a/providers/edge3/provider.yaml
+++ b/providers/edge3/provider.yaml
@@ -132,6 +132,15 @@ config:
type: integer
example: ~
default: "524288"
+ push_logs:
+ description: |
+ Flag to enable or disable pushing of log files from edge worker to
the central site.
+ When enabled, edge workers will upload task log files in chunks to
the central Airflow site.
+ When disabled, logs will only be available locally on the edge
worker.
+ version_added: 1.5.0
+ type: boolean
+ example: "True"
+ default: "True"
worker_umask:
description: |
The default umask to use for edge worker when run in daemon mode
diff --git a/providers/edge3/src/airflow/providers/edge3/cli/worker.py
b/providers/edge3/src/airflow/providers/edge3/cli/worker.py
index d1e0bcdcf28..7090f627ef0 100644
--- a/providers/edge3/src/airflow/providers/edge3/cli/worker.py
+++ b/providers/edge3/src/airflow/providers/edge3/cli/worker.py
@@ -348,7 +348,11 @@ class EdgeWorker:
else:
used_concurrency += job.edge_job.concurrency_slots
- if job.logfile.exists() and job.logfile.stat().st_size >
job.logsize:
+ if (
+ conf.getboolean("edge", "push_logs")
+ and job.logfile.exists()
+ and job.logfile.stat().st_size > job.logsize
+ ):
with job.logfile.open("rb") as logfile:
push_log_chunk_size = conf.getint("edge",
"push_log_chunk_size")
logfile.seek(job.logsize, os.SEEK_SET)
diff --git a/providers/edge3/src/airflow/providers/edge3/get_provider_info.py
b/providers/edge3/src/airflow/providers/edge3/get_provider_info.py
index 5892ebc76ac..b8a3fb890d0 100644
--- a/providers/edge3/src/airflow/providers/edge3/get_provider_info.py
+++ b/providers/edge3/src/airflow/providers/edge3/get_provider_info.py
@@ -93,6 +93,13 @@ def get_provider_info():
"example": None,
"default": "524288",
},
+ "push_logs": {
+ "description": "Flag to enable or disable pushing of
log files from edge worker to the central site.\nWhen enabled, edge workers
will upload task log files in chunks to the central Airflow site.\nWhen
disabled, logs will only be available locally on the edge worker.\n",
+ "version_added": "1.5.0",
+ "type": "boolean",
+ "example": "True",
+ "default": "True",
+ },
"worker_umask": {
"description": "The default umask to use for edge
worker when run in daemon mode\n\nThis controls the file-creation mode mask
which determines the initial value of file permission bits\nfor newly created
files.\n\nThis value is treated as an octal-integer.\n",
"version_added": None,
