jason810496 commented on code in PR #50371:
URL: https://github.com/apache/airflow/pull/50371#discussion_r2148335724
##########
airflow-core/src/airflow/dag_processing/processor.py:
##########
@@ -253,6 +281,15 @@ def start( # type: ignore[override]
client: Client,
**kwargs,
) -> Self:
+ # Pre-import Airflow modules in parent process to save CPU time and
memory
+ # This prevents them from being re-imported from zero in each
processing subprocess
+ import structlog
+
+ log = structlog.get_logger(logger_name="dag_processor")
Review Comment:
I think we can directly use `log` pass by argument directly instead of
getting logger again.
##########
airflow-core/src/airflow/dag_processing/processor.py:
##########
@@ -99,6 +101,31 @@ class DagFileParsingResult(BaseModel):
]
+def _pre_import_airflow_modules(file_path: str, log: FilteringBoundLogger) ->
None:
+ """
+ Pre-import Airflow modules found in the given file.
+
+ This prevents modules from being re-imported in each processing process,
+ saving CPU time and memory.
+
+ Args:
+ file_path: Path to the file to scan for imports
+ log: Logger instance to use for warnings
+
+ parsing_pre_import_modules:
+ default value is True
+ """
+ if not conf.getboolean("dag_processor", "parsing_pre_import_modules",
fallback=True):
+ return
+
+ log.info(f"[Debug by lz] dag_processor.parsing_pre_import_modules is
{conf.getboolean('dag_processor', 'parsing_pre_import_modules',
fallback=True)}")
Review Comment:
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
