This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-atr-experiments.git
The following commit(s) were added to refs/heads/main by this push:
new f08d224 Add the ability to use blockbuster and remove some blocking
calls
f08d224 is described below
commit f08d224058001daae29e9f0e9c32a849607b96e9
Author: Sean B. Palmer <[email protected]>
AuthorDate: Thu Feb 27 17:24:15 2025 +0200
Add the ability to use blockbuster and remove some blocking calls
---
atr/config.py | 6 +++--
atr/preload.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
atr/routes.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
atr/server.py | 24 ++++++++++++++++--
poetry.lock | 28 ++++++++++++++++++++-
pyproject.toml | 1 +
uv.lock | 20 +++++++++++++++
7 files changed, 229 insertions(+), 6 deletions(-)
diff --git a/atr/config.py b/atr/config.py
index e2d48f2..28b2c2c 100644
--- a/atr/config.py
+++ b/atr/config.py
@@ -28,6 +28,8 @@ GB = 1024 * MB
class AppConfig:
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
STATE_DIR = os.path.join(PROJECT_ROOT, "state")
+ DEBUG = False
+ USE_BLOCKBUSTER = False
RELEASE_STORAGE_DIR = os.path.join(STATE_DIR, "releases")
DATA_MODELS_FILE = data_models_file
@@ -54,13 +56,13 @@ class AppConfig:
)
-class ProductionConfig(AppConfig):
- DEBUG = False
+class ProductionConfig(AppConfig): ...
class DebugConfig(AppConfig):
DEBUG = True
TEMPLATES_AUTO_RELOAD = True
+ USE_BLOCKBUSTER = False
# Load all possible configurations
diff --git a/atr/preload.py b/atr/preload.py
new file mode 100644
index 0000000..dffee8b
--- /dev/null
+++ b/atr/preload.py
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Prevent blocking file operations from affecting the event loop.
+
+This module prevents blocking file operations in Jinja2 template loading from
+being run in an asynchronous context.
+"""
+
+import asyncio
+import os
+import pathlib
+
+from asfquart.base import QuartApp
+
+
+def preload_templates(app: QuartApp) -> None:
+ """Preload all templates in the templates directory."""
+ # We must disable automatic reload otherwise Jinja will check for
modifications
+ # Checking for modifications means that Jinja will call os.stat() in an
asynchronous context
+ app.jinja_env.auto_reload = False
+
+ template_dir = pathlib.Path(os.path.join(os.path.dirname(os.getcwd()),
"atr", "templates"))
+
+ if not template_dir.exists():
+ print(f"Warning: Template directory {template_dir} does not exist")
+ return
+
+ # Find all template files
+ template_files: list[pathlib.Path] = []
+ for extension in [".html", ".jinja", ".j2", ".txt"]:
+ template_files.extend(template_dir.glob(f"**/*{extension}"))
+
+ # For each template file, get its path relative to the template directory
+ for template_file in template_files:
+ try:
+ relative_path = template_file.relative_to(template_dir)
+ template_name = str(relative_path).replace("\\", "/")
+
+ # Access the template to make Jinja load and cache it
+ app.jinja_env.get_template(template_name)
+ except Exception as e:
+ print(f"Error preloading template {template_file}: {e}")
+ print(f"Preloaded {len(template_files)} templates")
+
+
+def setup_template_preloading(app: QuartApp) -> None:
+ """Register the template preloading to happen before the async loop
starts."""
+ # Store the original before_serving functions
+ original_before_serving = app.before_serving_funcs.copy()
+ app.before_serving_funcs = []
+
+ @app.before_serving
+ async def preload_before_blockbuster() -> None:
+ # Preload all templates
+ # This doesn't really need to be asynchronous
+ # We do this anyway to avoid being ironic
+ await asyncio.to_thread(preload_templates, app)
+
+ # Run all the original before_serving functions
+ for func in original_before_serving:
+ await func()
diff --git a/atr/routes.py b/atr/routes.py
index be5c5e9..339bfba 100644
--- a/atr/routes.py
+++ b/atr/routes.py
@@ -111,11 +111,84 @@ class MicrosecondsFormatter(logging.Formatter):
default_msec_format = "%s.%03d"
+class AsyncFileHandler(logging.Handler):
+ """A logging handler that writes logs asynchronously using aiofiles."""
+
+ def __init__(self, filename, mode="w", encoding=None):
+ super().__init__()
+ self.filename = filename
+
+ if mode != "w":
+ raise RuntimeError("Only write mode is supported")
+
+ self.encoding = encoding
+ self.queue = asyncio.Queue()
+ self.our_worker_task = None
+
+ def our_worker_task_ensure(self):
+ """Lazily create the worker task if it doesn't exist and there's an
event loop."""
+ if self.our_worker_task is None:
+ try:
+ loop = asyncio.get_running_loop()
+ self.our_worker_task = loop.create_task(self.our_worker())
+ except RuntimeError:
+ # No event loop running yet, try again on next emit
+ ...
+
+ async def our_worker(self):
+ """Background task that writes queued log messages to file."""
+ while True:
+ record = await self.queue.get()
+ if record is None:
+ break
+
+ try:
+ # Format the log record first
+ formatted_message = self.format(record) + "\n"
+ message_bytes = formatted_message.encode(self.encoding or
"utf-8")
+
+ # Use a binary mode literal with aiofiles.open
+ #
https://github.com/Tinche/aiofiles/blob/main/src/aiofiles/threadpool/__init__.py
+ # We should be able to use any mode, but pyright requires a
binary mode
+ async with aiofiles.open(self.filename, "wb+") as f:
+ await f.write(message_bytes)
+ except Exception:
+ self.handleError(record)
+ finally:
+ self.queue.task_done()
+
+ def emit(self, record):
+ """Queue the record for writing by the worker task."""
+ try:
+ # Ensure worker task is running
+ self.our_worker_task_ensure()
+
+ # Queue the record, but handle the case where no event loop is
running yet
+ try:
+ self.queue.put_nowait(record)
+ except RuntimeError:
+ # If there's no event loop, log synchronously as fallback
+ with open(self.filename, "w", encoding=self.encoding) as f:
+ f.write(self.format(record) + "\n")
+ except Exception:
+ self.handleError(record)
+
+ def close(self):
+ """Shut down the worker task cleanly."""
+ if self.our_worker_task is not None and not
self.our_worker_task.done():
+ try:
+ self.queue.put_nowait(None)
+ except RuntimeError:
+ # No running event loop, no need to clean up
+ ...
+ super().close()
+
+
# Setup a dedicated logger for route performance metrics
route_logger = logging.getLogger("route.performance")
# Use custom formatter that properly includes microseconds
# TODO: Is this actually UTC?
-route_logger_handler = logging.FileHandler("route-performance.log")
+route_logger_handler = AsyncFileHandler("route-performance.log")
route_logger_handler.setFormatter(MicrosecondsFormatter("%(asctime)s -
%(message)s"))
route_logger.addHandler(route_logger_handler)
route_logger.setLevel(logging.INFO)
@@ -547,6 +620,10 @@ async def package_add_session_process(
async def package_add_validate(
request: Request,
) -> tuple[str, FileStorage, FileStorage | None, FileStorage | None, str]:
+ # This calls quart.wrappers.request.form _load_form_data
+ # Which calls quart.formparser parse and parse_func and parser.parse
+ # Which calls _write which calls tempfile, which is synchronous
+ # It's getting a tempfile back from some prior call
form = await request.form
# TODO: Check that the submitter is a committer of the project
diff --git a/atr/server.py b/atr/server.py
index 0bc01d7..988d701 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -22,6 +22,7 @@ import os
from collections.abc import Iterable
from typing import Any
+from blockbuster import BlockBuster
from decouple import config
from quart_schema import OpenAPIProvider, QuartSchema
from werkzeug.routing import Rule
@@ -31,14 +32,17 @@ import asfquart.generics
import asfquart.session
from asfquart.base import QuartApp
from atr.blueprints import register_blueprints
-from atr.config import AppConfig, config_dict
+from atr.config import AppConfig, DebugConfig, config_dict
from atr.db import create_database
from atr.manager import get_worker_manager
+from atr.preload import setup_template_preloading
# WARNING: Don't run with debug turned on in production!
-DEBUG: bool = config("DEBUG", default=True, cast=bool)
+# TODO: Need to ask @tn how he wants this to work
+DEBUG: bool = config("DEBUG", default=DebugConfig.DEBUG, cast=bool)
# Determine which configuration to use
config_mode = "Debug" if DEBUG else "Production"
+use_blockbuster: bool = config("USE_BLOCKBUSTER",
default=DebugConfig.USE_BLOCKBUSTER, cast=bool)
# Avoid OIDC
asfquart.generics.OAUTH_URL_INIT =
"https://oauth.apache.org/auth?state=%s&redirect_uri=%s";
@@ -160,6 +164,22 @@ def create_app(app_config: type[AppConfig]) -> QuartApp:
app_setup_lifecycle(app)
app_setup_logging(app, config_mode, app_config)
+ # "I'll have a P, please, Bob."
+ blockbuster = BlockBuster()
+ setup_template_preloading(app)
+
+ @app.before_serving
+ async def start_blockbuster() -> None:
+ if DEBUG and use_blockbuster:
+ blockbuster.activate()
+ app.logger.info("Blockbuster activated to detect blocking calls")
+
+ @app.after_serving
+ async def stop_blockbuster() -> None:
+ if DEBUG and use_blockbuster:
+ blockbuster.deactivate()
+ app.logger.info("Blockbuster deactivated")
+
return app
diff --git a/poetry.lock b/poetry.lock
index 6e3bdaf..0460708 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -322,6 +322,21 @@ files = [
{file = "blinker-1.9.0.tar.gz", hash =
"sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf"},
]
+[[package]]
+name = "blockbuster"
+version = "1.5.23"
+description = "Utility to detect blocking calls in the async event loop"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+ {file = "blockbuster-1.5.23-py3-none-any.whl", hash =
"sha256:cf4d9df51d0ba5ac9b0f14594a456e42b7a49dcc35819c6b36805ac285b1f6fe"},
+ {file = "blockbuster-1.5.23.tar.gz", hash =
"sha256:ede6302307e700a60518c99caccfea159485382648e0158131e3506d4ff7b49c"},
+]
+
+[package.dependencies]
+forbiddenfruit = ">=0.1.4"
+
[[package]]
name = "boolean-py"
version = "4.0"
@@ -770,6 +785,17 @@ Werkzeug = ">=3.1"
async = ["asgiref (>=3.2)"]
dotenv = ["python-dotenv"]
+[[package]]
+name = "forbiddenfruit"
+version = "0.1.4"
+description = "Patch python built-in objects"
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+ {file = "forbiddenfruit-0.1.4.tar.gz", hash =
"sha256:e3f7e66561a29ae129aac139a85d610dbf3dd896128187ed5454b6421f624253"},
+]
+
[[package]]
name = "frozenlist"
version = "1.5.0"
@@ -2746,4 +2772,4 @@ propcache = ">=0.2.0"
[metadata]
lock-version = "2.1"
python-versions = "~=3.13"
-content-hash =
"1a93f827762f5267884f42f4ecb06249d951e8d897e242abd9d85ad4ff6b07c0"
+content-hash =
"79260867a4d15fb6f9c7eb985dae7cca7f425f6011baca8dc805615cb1b780f1"
diff --git a/pyproject.toml b/pyproject.toml
index 7658304..be7896c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,6 +22,7 @@ dependencies = [
"quart-schema~=0.21",
"spdx-tools>=0.8.3,<0.9.0",
"sqlmodel~=0.0",
+ "blockbuster>=1.5.23,<2.0.0",
]
[dependency-groups]
diff --git a/uv.lock b/uv.lock
index d39f6ff..1a3154c 100644
--- a/uv.lock
+++ b/uv.lock
@@ -189,6 +189,18 @@ wheels = [
{ url =
"https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl";,
hash =
"sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size
= 8458 },
]
+[[package]]
+name = "blockbuster"
+version = "1.5.23"
+source = { registry = "https://pypi.org/simple"; }
+dependencies = [
+ { name = "forbiddenfruit" },
+]
+sdist = { url =
"https://files.pythonhosted.org/packages/07/cd/19a5c7cef4ded0d5cab07c3df2195d8f599c33e7c18a362ce059d87ea79a/blockbuster-1.5.23.tar.gz";,
hash =
"sha256:ede6302307e700a60518c99caccfea159485382648e0158131e3506d4ff7b49c", size
= 51198 }
+wheels = [
+ { url =
"https://files.pythonhosted.org/packages/35/5f/7991d1f3b8d91eddabf883bc52f88f7d8c8f556be0d50ee8b0fa078be8a9/blockbuster-1.5.23-py3-none-any.whl";,
hash =
"sha256:cf4d9df51d0ba5ac9b0f14594a456e42b7a49dcc35819c6b36805ac285b1f6fe", size
= 13199 },
+]
+
[[package]]
name = "boolean-py"
version = "4.0"
@@ -412,6 +424,12 @@ wheels = [
{ url =
"https://files.pythonhosted.org/packages/af/47/93213ee66ef8fae3b93b3e29206f6b251e65c97bd91d8e1c5596ef15af0a/flask-3.1.0-py3-none-any.whl";,
hash =
"sha256:d667207822eb83f1c4b50949b1623c8fc8d51f2341d65f72e1a1815397551136", size
= 102979 },
]
+[[package]]
+name = "forbiddenfruit"
+version = "0.1.4"
+source = { registry = "https://pypi.org/simple"; }
+sdist = { url =
"https://files.pythonhosted.org/packages/e6/79/d4f20e91327c98096d605646bdc6a5ffedae820f38d378d3515c42ec5e60/forbiddenfruit-0.1.4.tar.gz";,
hash =
"sha256:e3f7e66561a29ae129aac139a85d610dbf3dd896128187ed5454b6421f624253", size
= 43756 }
+
[[package]]
name = "frozenlist"
version = "1.5.0"
@@ -1176,6 +1194,7 @@ dependencies = [
{ name = "aiosqlite" },
{ name = "alembic" },
{ name = "asfquart" },
+ { name = "blockbuster" },
{ name = "cryptography" },
{ name = "greenlet" },
{ name = "httpx" },
@@ -1207,6 +1226,7 @@ requires-dist = [
{ name = "aiosqlite", specifier = ">=0.21.0,<0.22.0" },
{ name = "alembic", specifier = "~=1.14" },
{ name = "asfquart", editable = "asfquart" },
+ { name = "blockbuster", specifier = ">=1.5.23,<2.0.0" },
{ name = "cryptography", specifier = "~=44.0" },
{ name = "greenlet", specifier = ">=3.1.1,<4.0.0" },
{ name = "httpx", specifier = "~=0.27" },
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
