This is an automated email from the ASF dual-hosted git repository. rusackas pushed a commit to branch fix/extensions-dev-hot-reload in repository https://gitbox.apache.org/repos/asf/superset.git
commit 4381fd97ea86257d862e5146329ea51dbc48303c Author: Amin Ghadersohi <[email protected]> AuthorDate: Tue May 12 21:19:57 2026 -0700 fix(extensions): make LOCAL_EXTENSIONS hot reload reliable in Docker The local-extensions watcher previously surfaced spurious reloads under Docker on macOS (VirtioFS / osxfs generates inotify events on every read, not just writes), and rebuilds that wrote many files at once produced one restart per file. It also touched superset/__init__.py to trigger Flask's reloader, which then meant any Python read of that file also looked like a change and recursed. Changes: - Watcher only acts on FileCreated/Modified/Moved events, verifies the file content actually changed via SHA-256, and debounces to one trigger per second. - Use a dedicated sentinel file (superset/extensions/.reload_trigger) that Python never reads, registered with Flask via --extra-files. The watcher ensures the sentinel exists on startup. - Bootstrap excludes superset/__init__.py from the file watcher so the old trigger path can't reintroduce the loop. - docker-compose mounts ./local_extensions:/app/local_extensions so the bind path matches LOCAL_EXTENSIONS in the dev superset_config. - Extension content endpoint and FRONTEND_REGEX accept nested paths inside frontend/dist so extensions can serve worker / WASM / chunk subfolders. Co-Authored-By: Claude Opus 4.7 <[email protected]> --- docker-compose.yml | 1 + docker/docker-bootstrap.sh | 4 +- superset/extensions/api.py | 2 +- superset/extensions/local_extensions_watcher.py | 102 +++++++++++++++++++++--- superset/extensions/utils.py | 2 +- 5 files changed, 99 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bd474a83ef4..583c1b13cc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,7 @@ x-superset-volumes: &superset-volumes - superset_home:/app/superset_home - ./tests:/app/tests - superset_data:/app/data + - ./local_extensions:/app/local_extensions x-common-build: &common-build context: . target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean` diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh index d20f28007f1..12375bec92e 100755 --- a/docker/docker-bootstrap.sh +++ b/docker/docker-bootstrap.sh @@ -80,7 +80,9 @@ case "${1}" in ;; app) echo "Starting web app (using development server)..." - flask run -p $PORT --reload --debugger --host=0.0.0.0 --exclude-patterns "*/node_modules/*:*/.venv/*:*/build/*:*/__pycache__/*:*/superset-frontend/*" + flask run -p $PORT --reload --debugger --host=0.0.0.0 \ + --extra-files "/app/superset/extensions/.reload_trigger" \ + --exclude-patterns "*/node_modules/*:*/.venv/*:*/build/*:*/__pycache__/*:*/superset-frontend/*:*/superset/__init__.py" ;; app-gunicorn) echo "Starting web app..." diff --git a/superset/extensions/api.py b/superset/extensions/api.py index fc386d60bf2..fbaa490cb1e 100644 --- a/superset/extensions/api.py +++ b/superset/extensions/api.py @@ -169,7 +169,7 @@ class ExtensionsRestApi(BaseApi): @protect() @safe - @expose("/<publisher>/<name>/<file>", methods=("GET",)) + @expose("/<publisher>/<name>/<path:file>", methods=("GET",)) def content(self, publisher: str, name: str, file: str) -> Response: """Get a frontend chunk of an extension. --- diff --git a/superset/extensions/local_extensions_watcher.py b/superset/extensions/local_extensions_watcher.py index 6233f91fe5c..4a6e3ad8f66 100644 --- a/superset/extensions/local_extensions_watcher.py +++ b/superset/extensions/local_extensions_watcher.py @@ -29,37 +29,113 @@ from flask import Flask logger = logging.getLogger(__name__) +# Sentinel file Flask watches via --extra-files. Touching it on a real change +# triggers a server reload without depending on cwd or the location of any +# Python source file. +RELOAD_TRIGGER = Path(__file__).resolve().parent / ".reload_trigger" + # Guard to prevent multiple initializations _watcher_initialized = False _watcher_lock = threading.Lock() -def _get_file_handler_class() -> Any: +def _get_file_handler_class() -> Any: # noqa: C901 """Get the file handler class, importing watchdog only when needed.""" try: - from watchdog.events import FileSystemEventHandler + import hashlib + + from watchdog.events import ( + FileCreatedEvent, + FileModifiedEvent, + FileMovedEvent, + FileSystemEventHandler, + ) class LocalExtensionFileHandler(FileSystemEventHandler): - """Custom file system event handler for LOCAL_EXTENSIONS directories.""" + """Custom file system event handler for LOCAL_EXTENSIONS directories. + + Only reacts to genuine content changes (create / modify / move) in the + dist directory, verified by comparing a SHA-256 of the file's content. + This avoids the Docker VirtioFS / osxfs problem where reading a file + generates inotify events that watchdog surfaces as modifications. + """ + + def __init__(self) -> None: + super().__init__() + # sha256 of last-seen content, keyed by absolute path + self._file_hashes: dict[str, str] = {} + # Deduplicate: only trigger once per second across all files + self._last_trigger: float = 0.0 + self._lock = threading.Lock() + + # ── helpers ────────────────────────────────────────────────────── + + @staticmethod + def _sha256(path: str) -> str | None: + try: + with open(path, "rb") as fh: + return hashlib.sha256(fh.read()).hexdigest() + except OSError: + return None + + def _content_changed(self, path: str) -> bool: + """Return True only when the file's content differs from last seen. + + The first time a path is observed its hash is stored as the baseline + and False is returned — that event is a 'first-seen', not a change. + Only a subsequent event where the digest differs from the baseline + is treated as a genuine content change. + """ + digest = self._sha256(path) + if digest is None: + return False + old_digest = self._file_hashes.get(path) + self._file_hashes[path] = digest + if old_digest is None: + # First observation — record baseline, do not trigger restart. + return False + return old_digest != digest + + # ── event handler ───────────────────────────────────────────────── def on_any_event(self, event: Any) -> None: - """Handle any file system event in the watched directories.""" + """Handle file system events in the watched directories.""" if event.is_directory: return - # Only trigger on changes to files in `dist` directory + # Only react to true write events; skip access / close / open etc. + if not isinstance( + event, (FileCreatedEvent, FileModifiedEvent, FileMovedEvent) + ): + return + + # Only care about files inside a `dist` directory src = getattr(event, "src_path", None) if not isinstance(src, str) or "dist" not in Path(src).parts: return + # Verify the file content actually changed to ignore spurious + # inotify events generated by Docker bind-mount reads. + if not self._content_changed(src): + return + + # Debounce: one restart per second max, regardless of how many + # files webpack writes simultaneously. + now = time.monotonic() + with self._lock: + if now - self._last_trigger < 1.0: + return + self._last_trigger = now + logger.info( "File change detected in LOCAL_EXTENSIONS: %s", event.src_path ) - # Touch superset/__init__.py to trigger Flask's file watcher - superset_init = Path("superset/__init__.py") - logger.info("Triggering restart by touching %s", superset_init) - os.utime(superset_init, (time.time(), time.time())) + # Touch the dedicated reload-trigger sentinel file. + # Flask watches this via --extra-files; it is never read by Python + # so Docker VirtioFS will not generate spurious inotify events on it. + logger.info("Triggering restart by touching %s", RELOAD_TRIGGER) + os.utime(RELOAD_TRIGGER, (time.time(), time.time())) return LocalExtensionFileHandler except ImportError: @@ -130,6 +206,14 @@ def setup_local_extensions_watcher(app: Flask) -> None: # noqa: C901 if not watch_dirs: return + # Ensure the sentinel exists so os.utime() and Flask's --extra-files watcher + # both have a real path to operate on. + try: + RELOAD_TRIGGER.touch(exist_ok=True) + except OSError as e: + logger.warning("Could not create reload trigger %s: %s", RELOAD_TRIGGER, e) + return + try: from watchdog.observers import Observer diff --git a/superset/extensions/utils.py b/superset/extensions/utils.py index 6332c97435d..29814d2eb1a 100644 --- a/superset/extensions/utils.py +++ b/superset/extensions/utils.py @@ -35,7 +35,7 @@ from superset.utils.core import check_is_safe_zip logger = logging.getLogger(__name__) -FRONTEND_REGEX = re.compile(r"^frontend/dist/([^/]+)$") +FRONTEND_REGEX = re.compile(r"^frontend/dist/(.+)$") BACKEND_REGEX = re.compile(r"^backend/src/(.+)$")
