This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 01e0ace414e Speed up provider asset change detection by pruning
dependency directories (#72784)
01e0ace414e is described below
commit 01e0ace414e074cc3fa47e741680d15c7ce5d0b2
Author: Andrew Chang <[email protected]>
AuthorDate: Wed Sep 9 19:12:05 2026 +0800
Speed up provider asset change detection by pruning dependency directories
(#72784)
The provider asset hash already excluded node_modules and .pnpm-store,
but only after listing and resolving every path under them. That is
about two seconds per provider on each breeze start-airflow for
nothing. Pruning the directories during the walk removes the cost.
---
scripts/ci/prek/compile_provider_assets.py | 24 ++--
.../tests/ci/prek/test_compile_provider_assets.py | 133 +++++++++++++++++++++
2 files changed, 144 insertions(+), 13 deletions(-)
diff --git a/scripts/ci/prek/compile_provider_assets.py
b/scripts/ci/prek/compile_provider_assets.py
index a0f16fb594f..1edd674fe5a 100755
--- a/scripts/ci/prek/compile_provider_assets.py
+++ b/scripts/ci/prek/compile_provider_assets.py
@@ -20,7 +20,6 @@ from __future__ import annotations
import argparse
import hashlib
import os
-import re
import shutil
import subprocess
import sys
@@ -83,15 +82,16 @@ PROVIDERS_PATHS = {
}
-def get_directory_hash(directory: Path, skip_path_regexps: list[str]) -> str:
- files = sorted(directory.rglob("*"))
- for skip_path_regexp in skip_path_regexps:
- matcher = re.compile(skip_path_regexp)
- files = [file for file in files if not
matcher.match(os.fspath(file.resolve()))]
+SKIPPED_DIRECTORY_NAMES = frozenset({"node_modules", ".pnpm-store"})
+
+
+def get_directory_hash(directory: Path) -> str:
sha = hashlib.sha256()
- for file in files:
- if file.is_file() and not file.name.startswith("."):
- sha.update(file.read_bytes())
+ for root, directory_names, file_names in os.walk(directory):
+ directory_names[:] = sorted(name for name in directory_names if name
not in SKIPPED_DIRECTORY_NAMES)
+ for file_name in sorted(file_names):
+ if not file_name.startswith("."):
+ sha.update((Path(root) / file_name).read_bytes())
return sha.hexdigest()
@@ -103,8 +103,6 @@ if __name__ not in ("__main__", "__mp_main__"):
INTERNAL_SERVER_ERROR = "500 Internal Server Error"
-SKIP_PATH_REGEXPS = [".*/node_modules.*", ".*/.pnpm-store.*"]
-
def compile_assets(provider_name: str):
if provider_name not in PROVIDERS_PATHS:
@@ -117,7 +115,7 @@ def compile_assets(provider_name: str):
provider_paths["hash"].parent.mkdir(exist_ok=True, parents=True)
if dist_directory.exists():
old_hash = provider_paths["hash"].read_text().strip() if
provider_paths["hash"].exists() else ""
- new_hash = get_directory_hash(www_directory,
skip_path_regexps=SKIP_PATH_REGEXPS)
+ new_hash = get_directory_hash(www_directory)
if new_hash == old_hash:
print(f"The '{www_directory}' directory has not changed! Skip
regeneration.")
return
@@ -143,7 +141,7 @@ def compile_assets(provider_name: str):
print(result.stdout + "\n" + result.stderr)
sys.exit(result.returncode)
subprocess.check_call(["pnpm", "build"], cwd=os.fspath(www_directory),
env=env)
- new_hash = get_directory_hash(www_directory,
skip_path_regexps=SKIP_PATH_REGEXPS)
+ new_hash = get_directory_hash(www_directory)
provider_paths["hash"].write_text(new_hash + "\n")
print(f"Assets compiled successfully. New hash: {new_hash}")
diff --git a/scripts/tests/ci/prek/test_compile_provider_assets.py
b/scripts/tests/ci/prek/test_compile_provider_assets.py
new file mode 100644
index 00000000000..fdada1f7a09
--- /dev/null
+++ b/scripts/tests/ci/prek/test_compile_provider_assets.py
@@ -0,0 +1,133 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Tests for the change detection in compile_provider_assets.py.
+
+The script refuses to be imported as a module, so these tests run the real
+script as a subprocess against a stubbed ``pnpm`` and a stubbed
+``common_prek_utils`` that points all paths into a temporary directory.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+SCRIPT_PATH = Path(__file__).resolve().parents[3] / "ci" / "prek" /
"compile_provider_assets.py"
+
+AI_WWW_RELATIVE_PATH =
Path("providers/common/ai/src/airflow/providers/common/ai/plugins/www")
+
+PNPM_STUB = textwrap.dedent(
+ """\
+ #!/usr/bin/env python3
+ import os
+ import sys
+ from pathlib import Path
+
+ with open(os.environ["PNPM_CALL_LOG"], "a") as call_log:
+ call_log.write(" ".join(sys.argv[1:]) + "\\n")
+
+ output_directory = Path("node_modules/pkg") if sys.argv[1] == "install"
else Path("dist")
+ output_directory.mkdir(parents=True, exist_ok=True)
+ (output_directory / "index.js").write_text("")
+ """
+)
+
+
+class ScriptHarness:
+ def __init__(self, tmp_path: Path):
+ airflow_root = tmp_path / "airflow_root"
+ self.www_directory = airflow_root / AI_WWW_RELATIVE_PATH
+ (self.www_directory / "src").mkdir(parents=True)
+ (self.www_directory / "src" / "main.ts").write_text("export const main
= 1;\n")
+
+ script_dir = tmp_path / "prek"
+ script_dir.mkdir()
+ self.script_path = script_dir / SCRIPT_PATH.name
+ shutil.copy(SCRIPT_PATH, self.script_path)
+ (script_dir / "common_prek_utils.py").write_text(
+ textwrap.dedent(
+ f"""\
+ from pathlib import Path
+
+ AIRFLOW_ROOT_PATH = Path({os.fspath(airflow_root)!r})
+ """
+ )
+ )
+
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ pnpm_stub = bin_dir / "pnpm"
+ pnpm_stub.write_text(PNPM_STUB)
+ pnpm_stub.chmod(0o755)
+ self.call_log = tmp_path / "pnpm_calls.txt"
+ self.env = {
+ **os.environ,
+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+ "PNPM_CALL_LOG": os.fspath(self.call_log),
+ }
+
+ def run_script(self) -> subprocess.CompletedProcess:
+ return subprocess.run(
+ [sys.executable, os.fspath(self.script_path), "ai"],
+ env=self.env,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+
+ def read_pnpm_calls(self) -> list[str]:
+ return self.call_log.read_text().splitlines() if
self.call_log.exists() else []
+
+
[email protected]
+def harness(tmp_path):
+ return ScriptHarness(tmp_path)
+
+
[email protected](
+ "changed_file",
+ [
+ pytest.param(Path(".pnpm-store/v10/files/00/abc"), id="pnpm-store"),
+ pytest.param(Path("node_modules/pkg/index.js"), id="node-modules"),
+ ],
+)
+def test_ignores_dependency_store_changes(harness, changed_file):
+ harness.run_script()
+ calls_after_first_build = harness.read_pnpm_calls()
+
+ (harness.www_directory / changed_file).parent.mkdir(parents=True,
exist_ok=True)
+ (harness.www_directory / changed_file).write_text("changed")
+ result = harness.run_script()
+
+ assert harness.read_pnpm_calls() == calls_after_first_build
+ assert "has not changed! Skip regeneration." in result.stdout
+
+
+def test_rebuilds_when_sources_change(harness):
+ harness.run_script()
+ calls_after_first_build = harness.read_pnpm_calls()
+
+ (harness.www_directory / "src" / "main.ts").write_text("export const main
= 2;\n")
+ harness.run_script()
+
+ assert harness.read_pnpm_calls() == [*calls_after_first_build, "install
--frozen-lockfile", "build"]
