This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new b9bb0ef Allow users to specify a download path for finished releases
b9bb0ef is described below
commit b9bb0ef343750daf6975d36ead612e94a0ab8a91
Author: Sean B. Palmer <[email protected]>
AuthorDate: Wed Jun 11 17:11:57 2025 +0100
Allow users to specify a download path for finished releases
---
atr/blueprints/admin/admin.py | 32 ++++++++++++-
atr/config.py | 2 +
atr/construct.py | 1 +
atr/routes/announce.py | 92 +++++++++++++++++++++++++++---------
atr/server.py | 3 +-
atr/templates/announce-selected.html | 53 ++++++++++++++++++---
atr/util.py | 22 +++++----
playwright/test.py | 1 +
8 files changed, 167 insertions(+), 39 deletions(-)
diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py
index 8bfbc40..ebb79f8 100644
--- a/atr/blueprints/admin/admin.py
+++ b/atr/blueprints/admin/admin.py
@@ -550,7 +550,7 @@ async def ongoing_tasks(project_name: str, version_name:
str, revision: str) ->
async def _delete_release_data(release_name: str) -> None:
"""Handle the deletion of database records and filesystem data for a
release."""
async with db.session() as data:
- release = await data.release(name=release_name).demand(
+ release = await data.release(name=release_name, _project=True).demand(
base.ASFQuartException(f"Release '{release_name}' not found.", 404)
)
release_dir = util.release_directory_base(release)
@@ -572,6 +572,36 @@ async def _delete_release_data(release_name: str) -> None:
_LOGGER.info("Deleted release record: %s", release_name)
await data.commit()
+ await _delete_release_data_downloads(release)
+ await _delete_release_data_filesystem(release_dir, release_name)
+
+
+async def _delete_release_data_downloads(release: models.Release) -> None:
+ # Delete hard links from the downloads directory
+ finished_dir = util.release_directory(release)
+ if await aiofiles.os.path.isdir(finished_dir):
+ release_inodes = set()
+ async for file_path in util.paths_recursive(finished_dir):
+ try:
+ stat_result = await aiofiles.os.stat(finished_dir / file_path)
+ release_inodes.add(stat_result.st_ino)
+ except FileNotFoundError:
+ continue
+
+ if release_inodes:
+ downloads_dir = util.get_downloads_dir()
+ async for link_path in util.paths_recursive(downloads_dir):
+ full_link_path = downloads_dir / link_path
+ try:
+ link_stat = await aiofiles.os.stat(full_link_path)
+ if link_stat.st_ino in release_inodes:
+ await aiofiles.os.remove(full_link_path)
+ _LOGGER.info(f"Deleted hard link: {full_link_path}")
+ except FileNotFoundError:
+ continue
+
+
+async def _delete_release_data_filesystem(release_dir: pathlib.Path,
release_name: str) -> None:
# Delete from the filesystem
try:
if await aiofiles.os.path.isdir(release_dir):
diff --git a/atr/config.py b/atr/config.py
index 6cd84e9..4e4e641 100644
--- a/atr/config.py
+++ b/atr/config.py
@@ -50,6 +50,7 @@ class AppConfig:
USE_BLOCKBUSTER = False
SECRET_KEY = decouple.config("SECRET_KEY", default=secrets.token_hex(128
// 8))
WTF_CSRF_ENABLED = decouple.config("WTF_CSRF_ENABLED", default=True,
cast=bool)
+ DOWNLOADS_STORAGE_DIR = os.path.join(STATE_DIR, "downloads")
FINISHED_STORAGE_DIR = os.path.join(STATE_DIR, "finished")
UNFINISHED_STORAGE_DIR = os.path.join(STATE_DIR, "unfinished")
SQLITE_DB_PATH = decouple.config("SQLITE_DB_PATH", default="atr.db")
@@ -117,6 +118,7 @@ def get() -> type[AppConfig]:
absolute_paths = [
(config.PROJECT_ROOT, "PROJECT_ROOT"),
(config.STATE_DIR, "STATE_DIR"),
+ (config.DOWNLOADS_STORAGE_DIR, "DOWNLOADS_STORAGE_DIR"),
(config.FINISHED_STORAGE_DIR, "FINISHED_STORAGE_DIR"),
(config.UNFINISHED_STORAGE_DIR, "UNFINISHED_STORAGE_DIR"),
]
diff --git a/atr/construct.py b/atr/construct.py
index 7701394..413773c 100644
--- a/atr/construct.py
+++ b/atr/construct.py
@@ -67,6 +67,7 @@ async def announce_release_body(body: str, options:
AnnounceReleaseOptions) -> s
download_path = util.as_url(
routes_release_view, project_name=options.project_name,
version_name=options.version_name
)
+ # TODO: This download_url should probably be for the proxy download
directory, not the ATR view
download_url = f"https://{host}{download_path}";
# Perform substitutions in the body
diff --git a/atr/routes/announce.py b/atr/routes/announce.py
index 4974787..7363f0b 100644
--- a/atr/routes/announce.py
+++ b/atr/routes/announce.py
@@ -18,7 +18,8 @@
import asyncio
import datetime
import logging
-from typing import TYPE_CHECKING, Any, Protocol
+import pathlib
+from typing import Any, Protocol
import aiofiles.os
import aioshutil
@@ -27,6 +28,7 @@ import sqlmodel
import werkzeug.wrappers.response as response
import wtforms
+import atr.config as config
import atr.construct as construct
import atr.db as db
import atr.db.models as models
@@ -38,9 +40,6 @@ import atr.tasks.message as message
import atr.template as template
import atr.util as util
-if TYPE_CHECKING:
- import pathlib
-
class AnnounceFormProtocol(Protocol):
"""Protocol for the dynamically generated AnnounceForm."""
@@ -49,6 +48,7 @@ class AnnounceFormProtocol(Protocol):
preview_revision: wtforms.HiddenField
mailing_list: wtforms.RadioField
confirm_announce: wtforms.BooleanField
+ download_path_suffix: wtforms.StringField
subject: wtforms.StringField
body: wtforms.TextAreaField
submit: wtforms.SubmitField
@@ -96,12 +96,22 @@ async def selected(session: routes.CommitterSession,
project_name: str, version_
announce_form.subject.data = f"[ANNOUNCE] {project_display_name}
{version_name} released"
# The body can be changed, either from VoteTemplate or from the form
announce_form.body.data = await
construct.announce_release_default(project_name)
+ # The download path suffix can be changed
+ # The defaults depend on whether the project is top level or not
+ if (committee := release.project.committee) is None:
+ raise ValueError("Release has no committee")
+ top_level_project = release.project.name == util.unwrap(committee.name)
+ # These defaults are as per #136, but we allow the user to change the
result
+ announce_form.download_path_suffix.data = (
+ "/" if top_level_project else
f"/{release.project.name}-{release.version}/"
+ )
+ description_download_prefix =
f"https://{config.get().APP_HOST}/downloads/{committee.name}"
+ announce_form.download_path_suffix.description = f"The URL will be
{description_download_prefix} plus this suffix"
- eventual_path = util.release_directory_eventual(release)
- finished_path = util.get_finished_dir()
- url_path = f"/downloads/{eventual_path.relative_to(finished_path)}/"
return await template.render(
- "announce-selected.html", release=release,
announce_form=announce_form, url_path=url_path
+ "announce-selected.html",
+ release=release,
+ announce_form=announce_form,
)
@@ -131,10 +141,10 @@ async def selected_post(
subject = str(announce_form.subject.data)
body = str(announce_form.body.data)
preview_revision_number = str(announce_form.preview_revision.data)
+ download_path_suffix = _download_path_suffix_validated(announce_form)
- source: str = ""
- target: str = ""
- source_base: pathlib.Path | None = None
+ unfinished_dir: str = ""
+ finished_dir: str = ""
async with db.session(log_queries=True) as data:
try:
@@ -146,6 +156,8 @@ async def selected_post(
with_revisions=True,
data=data,
)
+ if (committee := release.project.committee) is None:
+ raise ValueError("Release has no committee")
test_list = "user-tests"
recipient = f"{test_list}@tooling.apache.org"
@@ -182,11 +194,11 @@ async def selected_post(
data.add(task)
# Prepare paths for file operations
- source_base = util.release_directory_base(release)
- source_path = source_base / release.unwrap_revision_number
- source = str(source_path)
+ unfinished_revisions_path = util.release_directory_base(release)
+ unfinished_path = unfinished_revisions_path /
release.unwrap_revision_number
+ unfinished_dir = str(unfinished_path)
- await _promote(release, data, preview_revision_number)
+ await _promote_in_database(release, data, preview_revision_number)
await data.commit()
except (routes.FlashError, Exception) as e:
@@ -203,18 +215,22 @@ async def selected_post(
# Do not put it in the data block after data.commit()
# Otherwise util.release_directory() will not work
release = await
data.release(name=release.name).demand(RuntimeError(f"Release {release.name}
does not exist"))
- target_path = util.release_directory(release)
- target = str(target_path)
- if await aiofiles.os.path.exists(target):
+ finished_path = util.release_directory(release)
+ finished_dir = str(finished_path)
+ if await aiofiles.os.path.exists(finished_dir):
raise routes.FlashError("Release already exists")
# Ensure that the permissions of every directory are 755
- await asyncio.to_thread(util.chmod_directories, source_path)
+ await asyncio.to_thread(util.chmod_directories, unfinished_path)
try:
- await aioshutil.move(source, target)
- if source_base:
- await aioshutil.rmtree(str(source_base)) # type: ignore[call-arg]
+ # Move the release files from somewhere in unfinished to somewhere in
finished
+ # The whole finished hierarchy is write once for each directory, and
then read only
+ # TODO: Set permissions to help enforce this, or find alternative
methods
+ await aioshutil.move(unfinished_dir, finished_dir)
+ if unfinished_revisions_path:
+ # This removes all of the prior revisions
+ await aioshutil.rmtree(str(unfinished_revisions_path)) # type:
ignore[call-arg]
except Exception as e:
logging.exception("Error during release announcement, file system
phase:")
return await session.redirect(
@@ -224,6 +240,8 @@ async def selected_post(
version_name=version_name,
)
+ await _hard_link_downloads(committee, finished_path, download_path_suffix)
+
routes_release_finished = routes_release.finished # type: ignore[has-type]
return await session.redirect(
routes_release_finished,
@@ -248,6 +266,7 @@ async def _create_announce_form_instance(
validators=[wtforms.validators.InputRequired("Mailing list
selection is required")],
default="[email protected]",
)
+ download_path_suffix = wtforms.StringField("Download path suffix",
validators=[wtforms.validators.Optional()])
confirm_announce = wtforms.BooleanField(
"Confirm",
validators=[wtforms.validators.DataRequired("You must confirm to
proceed with announcement")],
@@ -260,7 +279,34 @@ async def _create_announce_form_instance(
return form_instance
-async def _promote(release: models.Release, data: db.Session,
preview_revision_number: str) -> None:
+def _download_path_suffix_validated(announce_form: AnnounceFormProtocol) ->
str:
+ download_path_suffix = str(announce_form.download_path_suffix.data)
+ if (".." in download_path_suffix) or ("//" in download_path_suffix):
+ raise ValueError("Download path suffix must not contain .. or //")
+ if download_path_suffix.startswith("./"):
+ download_path_suffix = download_path_suffix[1:]
+ elif download_path_suffix == ".":
+ download_path_suffix = "/"
+ if not download_path_suffix.startswith("/"):
+ download_path_suffix = "/" + download_path_suffix
+ if not download_path_suffix.endswith("/"):
+ download_path_suffix = download_path_suffix + "/"
+ if "/." in download_path_suffix:
+ raise ValueError("Download path suffix must not contain /.")
+ return download_path_suffix
+
+
+async def _hard_link_downloads(
+ committee: models.Committee, unfinished_path: pathlib.Path,
download_path_suffix: str
+) -> None:
+ """Hard link the release files to the downloads directory."""
+ # TODO: Rename *_dir functions to _path functions
+ downloads_base_path = util.get_downloads_dir()
+ downloads_path = downloads_base_path / committee.name /
download_path_suffix.removeprefix("/")
+ await util.create_hard_link_clone(unfinished_path, downloads_path,
exist_ok=True)
+
+
+async def _promote_in_database(release: models.Release, data: db.Session,
preview_revision_number: str) -> None:
"""Promote a release preview to a release and delete its old revisions."""
via = models.validate_instrumented_attribute
diff --git a/atr/server.py b/atr/server.py
index c3fe38c..61fda26 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -77,9 +77,10 @@ def app_dirs_setup(app_config: type[config.AppConfig]) ->
None:
raise RuntimeError(f"State directory not found:
{app_config.STATE_DIR}")
os.chdir(app_config.STATE_DIR)
print(f"Working directory changed to: {os.getcwd()}")
- util.get_unfinished_dir().mkdir(parents=True, exist_ok=True)
+ util.get_downloads_dir().mkdir(parents=True, exist_ok=True)
util.get_finished_dir().mkdir(parents=True, exist_ok=True)
util.get_tmp_dir().mkdir(parents=True, exist_ok=True)
+ util.get_unfinished_dir().mkdir(parents=True, exist_ok=True)
def app_setup_api_docs(app: base.QuartApp) -> None:
diff --git a/atr/templates/announce-selected.html
b/atr/templates/announce-selected.html
index 871a25b..67ef1c0 100644
--- a/atr/templates/announce-selected.html
+++ b/atr/templates/announce-selected.html
@@ -144,12 +144,11 @@
</div>
</div>
<div class="row mb-3 pb-3 border-bottom">
- <label class="col-sm-3 col-form-label text-sm-end">Download path</label>
- <div class="col-md-9 mb-3">
- <pre class="p-2 px-3 bg-light border rounded font-monospace
overflow-auto">{{ url_path }}</pre>
- </div>
- <div class="col-md-9 offset-md-3">
- <p>This value is read-only.</p>
+ {{ forms.label(announce_form.download_path_suffix, col="sm3") }}
+ <div class="col-md-9">
+ {{ forms.widget(announce_form.download_path_suffix) }}
+ {{ forms.errors(announce_form.download_path_suffix) }}
+ {{ forms.description(announce_form.download_path_suffix) }}
</div>
</div>
<div class="row mb-3">
@@ -229,6 +228,48 @@
});
fetchAndUpdateAnnouncePreview();
+
+ const pathInput = document.getElementById("download_path_suffix");
+ const pathHelpText =
document.getElementById("download_path_suffix-help");
+
+ if (pathInput && pathHelpText) {
+ const initialText = pathHelpText.textContent;
+ if (initialText.includes(" plus this suffix")) {
+ const baseText = initialText.substring(0,
initialText.indexOf(" plus this suffix"));
+ let pathDebounce;
+
+ // This must match the validation code in announce.py
+ function updatePathHelpText() {
+ let suffix = pathInput.value;
+ if (suffix.includes("..") || suffix.includes("//")) {
+ pathHelpText.textContent = "Download path suffix
must not contain .. or //";
+ return;
+ }
+ if (suffix.startsWith("./")) {
+ suffix = suffix.substring(1);
+ } else if (suffix === ".") {
+ suffix = "/";
+ }
+ if (!suffix.startsWith("/")) {
+ suffix = "/" + suffix;
+ }
+ if (!suffix.endsWith("/")) {
+ suffix = suffix + "/";
+ }
+ if (suffix.includes("/.")) {
+ pathHelpText.textContent = "Download path suffix
must not contain /.";
+ return;
+ }
+ pathHelpText.textContent = baseText + suffix;
+ }
+
+ pathInput.addEventListener("input", () => {
+ clearTimeout(pathDebounce);
+ pathDebounce = setTimeout(updatePathHelpText, 10);
+ });
+ updatePathHelpText();
+ }
+ }
});
</script>
{% endblock javascripts %}
diff --git a/atr/util.py b/atr/util.py
index 4b23687..25aec42 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -191,17 +191,19 @@ async def content_list(
async def create_hard_link_clone(
- source_dir: pathlib.Path, dest_dir: pathlib.Path, do_not_create_dest_dir:
bool = False
+ source_dir: pathlib.Path,
+ dest_dir: pathlib.Path,
+ do_not_create_dest_dir: bool = False,
+ exist_ok: bool = False,
) -> None:
"""Recursively create a clone of source_dir in dest_dir using hard links
for files."""
- # TODO: We're currently using cp -al instead
# Ensure source exists and is a directory
if not await aiofiles.os.path.isdir(source_dir):
raise ValueError(f"Source path is not a directory or does not exist:
{source_dir}")
# Create destination directory
if do_not_create_dest_dir is False:
- await aiofiles.os.makedirs(dest_dir, exist_ok=False)
+ await aiofiles.os.makedirs(dest_dir, exist_ok=exist_ok)
async def _clone_recursive(current_source: pathlib.Path, current_dest:
pathlib.Path) -> None:
for entry in await aiofiles.os.scandir(current_source):
@@ -321,6 +323,10 @@ async def get_asf_id_or_die() -> str:
return web_session.uid
+def get_downloads_dir() -> pathlib.Path:
+ return pathlib.Path(config.get().DOWNLOADS_STORAGE_DIR)
+
+
def get_finished_dir() -> pathlib.Path:
return pathlib.Path(config.get().FINISHED_STORAGE_DIR)
@@ -532,11 +538,11 @@ def release_directory_base(release: models.Release) ->
pathlib.Path:
return base_dir / project_name / version_name
-def release_directory_eventual(release: models.Release) -> pathlib.Path:
- """Return the path to the eventual destination of the release files."""
- path_project = release.project.name
- path_version = release.version
- return get_finished_dir() / path_project / path_version
+# def release_directory_eventual(release: models.Release) -> pathlib.Path:
+# """Return the path to the eventual destination of the release files."""
+# path_project = release.project.name
+# path_version = release.version
+# return get_finished_dir() / path_project / path_version
def release_directory_revision(release: models.Release) -> pathlib.Path | None:
diff --git a/playwright/test.py b/playwright/test.py
index 22c9a9d..bfb11de 100644
--- a/playwright/test.py
+++ b/playwright/test.py
@@ -1340,6 +1340,7 @@ def wait_for_path(page: sync_api.Page, path: str) -> None:
parsed_url = urllib.parse.urlparse(page.url)
if parsed_url.path != path:
logging.error(f"Expected URL path '{path}', but got
'{parsed_url.path}'")
+ logging.error(f"Page content:\\n{page.content()}")
raise RuntimeError(f"Expected URL path '{path}', but got
'{parsed_url.path}'")
logging.info(f"Current URL: {page.url}")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
