This is an automated email from the ASF dual-hosted git repository.
hainenber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new c1bb129c218 fix(change-detector): classify changed files by language,
not directory (#44895)
c1bb129c218 is described below
commit c1bb129c218645f53fdf9eb6c94e598cde69f82d
Author: Krishna kumar singh <[email protected]>
AuthorDate: Fri Oct 2 22:11:05 2026 +0530
fix(change-detector): classify changed files by language, not directory
(#44895)
---
.github/actions/change-detector/action.yml | 5 ++
scripts/change_detector.py | 60 ++++++++++++++++++
tests/unit_tests/scripts/change_detector_test.py | 77 ++++++++++++++++++++++++
3 files changed, 142 insertions(+)
diff --git a/.github/actions/change-detector/action.yml
b/.github/actions/change-detector/action.yml
index da19bea6a4c..decc74fc339 100644
--- a/.github/actions/change-detector/action.yml
+++ b/.github/actions/change-detector/action.yml
@@ -20,6 +20,11 @@ outputs:
superset-extensions-cli:
description: Whether superset-extensions-cli package-related files were
changed
value: ${{ steps.change-detector.outputs.superset-extensions-cli }}
+ languages:
+ description: >-
+ JSON array of CodeQL languages touched, classified by file extension
+ regardless of directory (e.g. ["javascript", "python"])
+ value: ${{ steps.change-detector.outputs.languages }}
runs:
using: composite
steps:
diff --git a/scripts/change_detector.py b/scripts/change_detector.py
index d943edb857b..a457e2f35a3 100755
--- a/scripts/change_detector.py
+++ b/scripts/change_detector.py
@@ -74,6 +74,54 @@ PATTERNS = {
}
GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
+# Maps each PATTERNS group to the CodeQL language(s) its files are written
+# in, by file extension rather than directory. PATTERNS above groups files by
+# project area (e.g. "frontend" means "under superset-frontend/"), which is
+# right for gating CI jobs by area but wrong for picking which CodeQL
+# languages to scan: a .js file outside superset-frontend/ (e.g. under
+# scripts/ or superset/mcp_service/) is grouped under "python" by PATTERNS,
+# so a consumer that reused the "frontend" group as a stand-in for
+# "javascript changed" would silently skip scanning it. This reuses the same
+# group names as PATTERNS rather than introducing new ones, so a group's
+# language(s) can be looked up directly instead of via a separate mapping.
+# See https://github.com/apache/superset/issues/44822.
+GROUP_LANGUAGE_EXTENSIONS: dict[str, dict[str, str]] = {
+ "python": {".py": "python"},
+ "frontend": {
+ ".js": "javascript",
+ ".jsx": "javascript",
+ ".ts": "javascript",
+ ".tsx": "javascript",
+ ".mjs": "javascript",
+ ".cjs": "javascript",
+ },
+}
+# Flattened for a direct extension -> language lookup, independent of which
+# PATTERNS group (i.e. directory) the file lives in.
+LANGUAGE_EXTENSIONS: dict[str, str] = {
+ ext: lang
+ for extensions in GROUP_LANGUAGE_EXTENSIONS.values()
+ for ext, lang in extensions.items()
+}
+
+
+def detect_languages(files: Optional[List[str]]) -> List[str]:
+ """Returns the CodeQL languages actually touched, by file extension.
+
+ Unlike the PATTERNS groups above, this ignores which directory a file
+ lives in: a .js file is "javascript" whether it's under
+ superset-frontend/ or not. `files is None` (workflow_dispatch/schedule)
+ means "assume everything changed", so every known language is returned.
+ """
+ if files is None:
+ return sorted(set(LANGUAGE_EXTENSIONS.values()))
+ languages = set()
+ for file in files:
+ _, ext = os.path.splitext(file)
+ if lang := LANGUAGE_EXTENSIONS.get(ext):
+ languages.add(lang)
+ return sorted(languages)
+
def _is_rate_limited(err: HTTPError) -> bool:
"""Whether a 403 is GitHub throttling rather than a missing token scope."""
@@ -217,6 +265,16 @@ def main(event_type: str, sha: str, repo: str) -> None:
files, patterns_compiled
)
+ # The 100-file API cap below treats a push/PR touching that many files as
+ # "everything changed" for the PATTERNS groups; language detection
+ # honors the same assumption so a consumer combining both outputs never
+ # sees a language silently excluded by the cap.
+ languages = (
+ sorted(set(LANGUAGE_EXTENSIONS.values()))
+ if files is not None and len(files) >= 99
+ else detect_languages(files)
+ )
+
# Output results
output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt" #
noqa: S108
with open(output_path, "a") as f:
@@ -227,6 +285,8 @@ def main(event_type: str, sha: str, repo: str) -> None:
if changed or (files is not None and len(files) >= 99):
print(f"{check}=true", file=f)
print(f"Triggering group: {check}")
+ print(f"languages={json.dumps(languages)}", file=f)
+ print(f"Languages detected: {languages}")
def get_git_sha() -> str:
diff --git a/tests/unit_tests/scripts/change_detector_test.py
b/tests/unit_tests/scripts/change_detector_test.py
index 360f8007475..d8c2885e5fe 100644
--- a/tests/unit_tests/scripts/change_detector_test.py
+++ b/tests/unit_tests/scripts/change_detector_test.py
@@ -186,3 +186,80 @@ def test_grype_config_changes_trigger_docker_build() ->
None:
[".grype.yaml"],
change_detector.PATTERNS["docker"],
)
+
+
+def test_detect_languages_classifies_js_outside_frontend_dir() -> None:
+ """A .js file outside superset-frontend/ is still "javascript", even
+ though PATTERNS groups it under "python" by directory -- the gap
+ https://github.com/apache/superset/issues/44822 tracks."""
+ files = ["superset/mcp_service/index.js"]
+ assert change_detector.detect_languages(files) == ["javascript"]
+ assert change_detector.detect_changes(files,
change_detector.PATTERNS["python"])
+ assert not change_detector.detect_changes(
+ files, change_detector.PATTERNS["frontend"]
+ )
+
+
+def test_detect_languages_mixed_py_and_ts() -> None:
+ files = ["superset/foo.py", "superset-frontend/src/bar.tsx"]
+ assert change_detector.detect_languages(files) == ["javascript", "python"]
+
+
+def test_detect_languages_ignores_unmapped_extensions() -> None:
+ assert change_detector.detect_languages(["docs/intro.md", "Dockerfile"])
== []
+
+
+def test_detect_languages_none_means_every_language() -> None:
+ """workflow_dispatch/schedule runs assume everything changed."""
+ assert change_detector.detect_languages(None) == ["javascript", "python"]
+
+
+def test_group_language_extensions_reuses_pattern_group_names() -> None:
+ """The language map is keyed by the same group names as PATTERNS,
+ rather than introducing its own, so a group's language(s) can be looked
+ up directly."""
+ assert set(change_detector.GROUP_LANGUAGE_EXTENSIONS) <= set(
+ change_detector.PATTERNS
+ )
+ assert change_detector.GROUP_LANGUAGE_EXTENSIONS["python"] == {".py":
"python"}
+ assert change_detector.GROUP_LANGUAGE_EXTENSIONS["frontend"][".tsx"] == (
+ "javascript"
+ )
+
+
+def test_main_writes_languages_to_github_output(tmp_path, monkeypatch) -> None:
+ """`main()` must write `languages` to $GITHUB_OUTPUT, not just compute it
+ -- a value that's assigned but never written is invisible to any
+ consuming workflow step."""
+ output_file = tmp_path / "github_output.txt"
+ monkeypatch.setenv("GITHUB_OUTPUT", str(output_file))
+ monkeypatch.setattr(
+ change_detector,
+ "fetch_changed_files_push",
+ lambda repo, sha: ["superset/foo.py", "superset-frontend/src/bar.tsx"],
+ )
+
+ change_detector.main("push", "deadbeef", "apache/superset")
+
+ output = output_file.read_text()
+ assert 'languages=["javascript", "python"]' in output
+ assert "python=true" in output
+ assert "frontend=true" in output
+
+
+def test_main_languages_respects_the_99_file_cap(tmp_path, monkeypatch) ->
None:
+ """A push/PR touching >= 99 files is treated as "everything changed" for
+ the PATTERNS groups; languages must honor the same cap so a consumer
+ combining both outputs never sees a language silently excluded by it."""
+ output_file = tmp_path / "github_output.txt"
+ monkeypatch.setenv("GITHUB_OUTPUT", str(output_file))
+ monkeypatch.setattr(
+ change_detector,
+ "fetch_changed_files_push",
+ lambda repo, sha: [f"docs/page{i}.md" for i in range(99)],
+ )
+
+ change_detector.main("push", "deadbeef", "apache/superset")
+
+ output = output_file.read_text()
+ assert 'languages=["javascript", "python"]' in output
