dstandish commented on code in PR #45318:
URL: https://github.com/apache/airflow/pull/45318#discussion_r1904484394
##########
airflow/dag_processing/bundles/manager.py:
##########
@@ -30,56 +31,59 @@
from airflow.dag_processing.bundles.base import BaseDagBundle
+_bundle_config = {}
+
class DagBundlesManager(LoggingMixin):
"""Manager for DAG bundles."""
- @property
- def bundle_configs(self) -> dict[str, dict]:
- """Get all DAG bundle configurations."""
- configured_bundles = conf.getsection("dag_bundles")
-
- if not configured_bundles:
- return {}
-
- # If dags_folder is empty string, we remove it. This allows the
default dags_folder bundle to be disabled.
- if not configured_bundles["dags_folder"]:
- del configured_bundles["dags_folder"]
+ def parse_config(self) -> None:
+ """
+ Get all DAG bundle configurations and store in module variable.
- dict_bundles: dict[str, dict] = {}
- for key in configured_bundles.keys():
- config = conf.getjson("dag_bundles", key)
- if not isinstance(config, dict):
- raise AirflowConfigException(f"Bundle config for {key} is not
a dict: {config}")
- dict_bundles[key] = config
+ If a bundle class for a given name has already been imported, it will
not be imported again.
+ """
+ configured_bundles = conf.getjson("dag_bundles", "backends")
- return dict_bundles
+ if not configured_bundles:
+ return
+
+ if not isinstance(configured_bundles, list):
+ raise AirflowConfigException(
+ "Bundle config is not a list. Check config value"
+ " for section `dag_bundles` and key `backends`."
+ )
+ seen = set()
+ for cfg in configured_bundles:
+ name = cfg["name"]
+ if name in seen:
+ raise ValueError(f"Dag bundle {name} is configured twice.")
+ seen.add(name)
+ if name in _bundle_config:
+ continue
+ class_ = import_string(cfg["classpath"])
+ kwargs = cfg["kwargs"]
+ _bundle_config[name] = (class_, kwargs)
+
+ # remove obsolete bundle configs
Review Comment:
if it's driven from a config file, it could change, couldn't it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
