potiuk commented on code in PR #73477:
URL: https://github.com/apache/airflow/pull/73477#discussion_r4068666384
##########
devel-common/src/sphinx_exts/docs_build/docs_builder.py:
##########
@@ -39,6 +45,121 @@
console = Console(force_terminal=True, color_system="standard",
width=CONSOLE_WIDTH)
+# Sphinx is run in the current process (one worker builds many packages in a
row) so that the fixed
+# start-up cost of a build is paid once per worker instead of once per
package. For a small provider
+# that fixed cost is ~20s out of ~22s: importing airflow, parsing and
validating every provider.yaml,
+# and - the largest part - autoapi's astroid parsing of the airflow modules
every provider imports.
+# The astroid parse cache in particular stays warm across the packages a
worker builds.
+
+
+def _forget_sphinx_conf_modules() -> None:
+ """
+ Drop the shared Sphinx configuration modules so the next conf.py
re-executes them from scratch.
+
+ The per-package ``conf.py`` files do ``from docs.provider_conf import *``
(or import
+ ``docs.utils.conf_constants``) and then mutate the lists they get -
``extensions.append(...)``,
+ ``autoapi_ignore.extend(...)``. Reusing the cached module would leak one
package's additions into
+ the next build, so everything under ``devel-common/src/docs`` except the
build script is forgotten.
+ """
+ docs_sources_prefix = DOCS_SOURCES_PATH.as_posix()
+ for name, module in list(sys.modules.items()):
+ if name == "docs.build_docs" or not name.startswith("docs."):
+ continue
+ module_file = getattr(module, "__file__", None) or ""
+ if module_file.startswith(docs_sources_prefix):
+ del sys.modules[name]
+
+
+class _RedirectableStream:
Review Comment:
Very clever :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
