gopidesupavan commented on code in PR #45259:
URL: https://github.com/apache/airflow/pull/45259#discussion_r1910641006
##########
scripts/ci/pre_commit/update_providers_dependencies.py:
##########
@@ -95,18 +99,46 @@ def visit_ImportFrom(self, node: ImportFrom):
self.process_import(fullname)
+def load_pyproject_toml(pyproject_toml_file_path: Path) -> dict[str, Any]:
+ try:
+ import tomllib
+ except ImportError:
+ import tomli as tomllib
+ return tomllib.loads(pyproject_toml_file_path.read_text())
+
+
def find_all_providers_and_provider_files():
- for root, _, filenames in os.walk(AIRFLOW_PROVIDERS_SRC_DIR):
+ for root, dirs, filenames in os.walk(AIRFLOW_PROVIDERS_DIR):
for filename in filenames:
if filename == "provider.yaml":
- provider_file = Path(root, filename)
- provider_name =
str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_SRC_DIR)).replace(
- os.sep, "."
- )
- provider_info = yaml.safe_load(provider_file.read_text())
+ provider_yaml_file = Path(root, filename)
+ if
provider_yaml_file.is_relative_to(AIRFLOW_PROVIDERS_SRC_DIR):
+ # TODO(potiuk) - remove this when we move all providers to
the new structure
+ provider_name = str(
+
provider_yaml_file.parent.relative_to(AIRFLOW_PROVIDERS_SRC_DIR)
+ ).replace(os.sep, ".")
+ else:
+ provider_name =
str(provider_yaml_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR)).replace(
+ os.sep, "."
+ )
+ NEW_STRUCTURE_PROVIDERS.add(provider_name)
+ PYPROJECT_TOML_CONTENT[provider_name] =
load_pyproject_toml(
+ provider_yaml_file.parent / "pyproject.toml"
+ )
+ # only descend to "src" directory in the new structure
+ # this avoids descending into .venv or "build" directories
in case
+ # someone works on providers in a separate virtualenv
+ if "src" in dirs:
Review Comment:
Nice check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
