This is an automated email from the ASF dual-hosted git repository. sbp pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 02e2cdb05fb651b00cce6de2bd2bcffb84458c96 Author: Sean B. Palmer <[email protected]> AuthorDate: Thu Dec 18 21:16:53 2025 +0000 Add progress bars to indicate the status of uploads --- atr/get/upload.py | 22 ++- atr/post/upload.py | 106 +++++++++++++- atr/static/js/src/upload-progress-ui.js | 147 +++++++++++++++++++ atr/static/js/src/upload-progress.js | 243 ++++++++++++++++++++++++++++++++ atr/util.py | 6 + playwright/test.py | 7 +- 6 files changed, 527 insertions(+), 4 deletions(-) diff --git a/atr/get/upload.py b/atr/get/upload.py index b60c6ef..4a86c1b 100644 --- a/atr/get/upload.py +++ b/atr/get/upload.py @@ -15,8 +15,11 @@ # specific language governing permissions and limitations # under the License. +import secrets from collections.abc import Sequence +import htpy + import atr.blueprints.get as get import atr.db as db import atr.form as form @@ -63,6 +66,18 @@ async def selected(session: web.Committer, project_name: str, version_name: str) block.h2(id="file-upload")["File upload"] block.p["Use this form to add files to this candidate draft."] + upload_session_token = secrets.token_hex(16) + stage_url = f"/upload/stage/{project_name}/{version_name}/{upload_session_token}" + finalise_url = f"/upload/finalise/{project_name}/{version_name}/{upload_session_token}" + + block.append( + htpy.div( + "#upload-config.atr-hide", + data_stage_url=stage_url, + data_finalise_url=finalise_url, + ) + ) + form.render_block( block, model_cls=shared.upload.AddFilesForm, @@ -70,6 +85,8 @@ async def selected(session: web.Committer, project_name: str, version_name: str) form_classes=".atr-canary.py-4.px-5", ) + block.append(htpy.div("#upload-progress-container.d-none")) + block.h2(id="svn-upload")["SVN upload"] block.p["Import files from this project's ASF Subversion repository into this draft."] block.p[ @@ -114,6 +131,7 @@ async def selected(session: web.Committer, project_name: str, version_name: str) return await template.blank( f"Upload files to {release.short_display_name}", content=block.collect(), + javascripts=["upload-progress-ui", "upload-progress"], ) @@ -136,7 +154,7 @@ def _render_ssh_keys_info(block: htm.Block, user_ssh_keys: Sequence[sql.SSHKey]) if key_count == 1: key = user_ssh_keys[0] key_parts = key.key.split(" ", 2) - key_comment = key_parts[2].strip() if len(key_parts) > 2 else "key" + key_comment = key_parts[2].strip() if (len(key_parts) > 2) else "key" block.p[ "We have the SSH key ", htm.a( @@ -152,7 +170,7 @@ def _render_ssh_keys_info(block: htm.Block, user_ssh_keys: Sequence[sql.SSHKey]) key_items = [] for key in user_ssh_keys: key_parts = key.key.split(" ", 2) - key_comment = key_parts[2].strip() if len(key_parts) > 2 else "key" + key_comment = key_parts[2].strip() if (len(key_parts) > 2) else "key" key_items.append( htm.li[ htm.a( diff --git a/atr/post/upload.py b/atr/post/upload.py index f0dc590..26cfb88 100644 --- a/atr/post/upload.py +++ b/atr/post/upload.py @@ -15,10 +15,16 @@ # specific language governing permissions and limitations # under the License. - +import asyncio +import json +import pathlib from typing import Final +import aiofiles +import aiofiles.os +import aioshutil import quart +import werkzeug.wrappers.response as response import atr.blueprints.post as post import atr.db as db @@ -26,11 +32,61 @@ import atr.get as get import atr.log as log import atr.shared as shared import atr.storage as storage +import atr.util as util import atr.web as web _SVN_BASE_URL: Final[str] = "https://dist.apache.org/repos/dist"; [email protected]("/upload/finalise/<project_name>/<version_name>/<upload_session>") +async def finalise( + session: web.Committer, project_name: str, version_name: str, upload_session: str +) -> web.WerkzeugResponse: + await session.check_access(project_name) + + try: + staging_dir = util.get_upload_staging_dir(upload_session) + except ValueError: + return _json_error("Invalid session token", 400) + + if not await aiofiles.os.path.isdir(staging_dir): + return _json_error("No staged files found", 400) + + try: + staged_files = await aiofiles.os.listdir(staging_dir) + except OSError: + return _json_error("Error reading staging directory", 500) + + staged_files = [f for f in staged_files if f not in (".", "..")] + if not staged_files: + return _json_error("No staged files found", 400) + + try: + async with storage.write(session) as write: + wacp = await write.as_project_committee_participant(project_name) + number_of_files = len(staged_files) + plural = "s" if number_of_files != 1 else "" + description = f"Upload of {number_of_files} file{plural} through web interface" + + async with wacp.release.create_and_manage_revision(project_name, version_name, description) as creating: + for filename in staged_files: + src = staging_dir / filename + dst = creating.interim_path / filename + await aioshutil.move(str(src), str(dst)) + + await aioshutil.rmtree(staging_dir) + + return await session.redirect( + get.compose.selected, + success=f"{number_of_files} file{plural} added successfully", + project_name=project_name, + version_name=version_name, + ) + except Exception as e: + log.exception("Error finalising upload:") + return _json_error(f"Error finalising upload: {e!s}", 500) + + @post.committer("/upload/<project_name>/<version_name>") @post.form(shared.upload.UploadForm) async def selected( @@ -46,6 +102,46 @@ async def selected( return await _svn_import(session, svn_form, project_name, version_name) [email protected]("/upload/stage/<project_name>/<version_name>/<upload_session>") +async def stage( + session: web.Committer, project_name: str, version_name: str, upload_session: str +) -> web.WerkzeugResponse: + await session.check_access(project_name) + + try: + staging_dir = util.get_upload_staging_dir(upload_session) + except ValueError: + return _json_error("Invalid session token", 400) + + files = await quart.request.files + file = files.get("file") + if (not file) or (not file.filename): + return _json_error("No file provided", 400) + + filename = pathlib.Path(file.filename).name + if (not filename) or (filename in (".", "..")): + return _json_error("Invalid filename", 400) + + await aiofiles.os.makedirs(staging_dir, exist_ok=True) + + target_path = staging_dir / filename + if await aiofiles.os.path.exists(target_path): + return _json_error("File already exists in staging", 409) + + try: + async with aiofiles.open(target_path, "wb") as f: + # 1 MiB chunks + while chunk := await asyncio.to_thread(file.stream.read, 1024 * 1024): + await f.write(chunk) + except Exception as e: + log.exception("Error staging file:") + if await aiofiles.os.path.exists(target_path): + await aiofiles.os.remove(target_path) + return _json_error(f"Error staging file: {e!s}", 500) + + return _json_success({"status": "staged", "filename": filename}) + + async def _add_files( session: web.Committer, add_form: shared.upload.AddFilesForm, project_name: str, version_name: str ) -> web.WerkzeugResponse: @@ -80,6 +176,14 @@ def _construct_svn_url(project_name: str, area: shared.upload.SvnArea, path: str return f"{_SVN_BASE_URL}/{area.value}/{project_name}/{path}" +def _json_error(message: str, status: int) -> web.WerkzeugResponse: + return response.Response(json.dumps({"error": message}), status=status, mimetype="application/json") + + +def _json_success(data: dict[str, str], status: int = 200) -> web.WerkzeugResponse: + return response.Response(json.dumps(data), status=status, mimetype="application/json") + + async def _svn_import( session: web.Committer, svn_form: shared.upload.SvnImportForm, project_name: str, version_name: str ) -> web.WerkzeugResponse: diff --git a/atr/static/js/src/upload-progress-ui.js b/atr/static/js/src/upload-progress-ui.js new file mode 100644 index 0000000..7648e80 --- /dev/null +++ b/atr/static/js/src/upload-progress-ui.js @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +window.UploadUI = { + formatBytes(b) { + if (b < 1024) return `${b} B`; + if (b < 1048576) return `${(b / 1024).toFixed(1)} KB`; + if (b < 1073741824) return `${(b / 1048576).toFixed(1)} MB`; + return `${(b / 1073741824).toFixed(2)} GB`; + }, + + createFileProgressUI(file, index, activeUploads) { + const div = document.createElement("div"); + div.className = "card mb-3"; + div.id = `upload-file-${index}`; + div.innerHTML = `<div class="card-body"> + <div class="d-flex justify-content-between align-items-start mb-2"> + <div><strong class="file-name"></strong> + <small class="text-muted ms-2">(${this.formatBytes(file.size)})</small></div> + <button type="button" class="btn btn-sm btn-outline-secondary cancel-btn">Cancel</button> + </div> + <progress class="w-100 mb-1" value="0" max="100"></progress> + <div class="d-flex justify-content-between"> + <small class="upload-status text-muted">Preparing...</small> + <small class="upload-percent">0%</small> + </div></div>`; + div.querySelector(".file-name").textContent = file.name; + div.querySelector(".cancel-btn").addEventListener("click", () => { + const xhr = activeUploads.get(index); + if (xhr) xhr.abort(); + }); + return div; + }, + + markUploadDone(ui, success, msg) { + const statusEl = ui.querySelector(".upload-status"); + statusEl.textContent = msg; + statusEl.className = `upload-status text-${success ? "success" : "danger"}`; + const border = success ? "border-success" : "border-danger"; + ui.querySelector(".card-body").classList.add( + "border-start", + border, + "border-3", + ); + if (success) { + ui.querySelector("progress").value = 100; + ui.querySelector(".upload-percent").textContent = "100%"; + } + ui.querySelector(".cancel-btn").classList.add("d-none"); + }, + + resetUploadUI(ui) { + ui.querySelector(".card-body").classList.remove( + "border-start", + "border-danger", + "border-3", + ); + const statusEl = ui.querySelector(".upload-status"); + statusEl.textContent = "Preparing..."; + statusEl.className = "upload-status text-muted"; + ui.querySelector("progress").value = 0; + ui.querySelector(".upload-percent").textContent = "0%"; + ui.querySelector(".cancel-btn").classList.remove("d-none"); + }, + + showAllFailedSummary(container) { + const summary = document.createElement("div"); + summary.className = "alert alert-danger mt-3"; + summary.innerHTML = `<strong>All uploads failed.</strong> + <button type="button" class="btn btn-outline-primary ms-3" id="retry-all-btn">Try again</button>`; + container.append(summary); + document + .getElementById("retry-all-btn") + .addEventListener("click", () => location.reload()); + }, + + showPartialSuccessSummary(container, completedCount, totalFiles, onFinalise) { + const summary = document.createElement("div"); + summary.className = "alert alert-warning mt-3"; + summary.id = "upload-summary"; + const totalWord = totalFiles === 1 ? "file" : "files"; + const stagedWord = completedCount === 1 ? "file" : "files"; + summary.innerHTML = `<strong>${completedCount} of ${totalFiles} ${totalWord} staged.</strong> + <span class="ms-2">You can finalise with the staged files or retry failed uploads.</span> + <button type="button" class="btn btn-primary ms-3" id="finalise-btn">Finalise ${completedCount} ${stagedWord}</button>`; + container.append(summary); + document + .getElementById("finalise-btn") + .addEventListener("click", onFinalise); + }, + + doFinalise(container, finaliseUrl, csrfToken) { + const existingSummary = document.getElementById("upload-summary"); + if (existingSummary) existingSummary.remove(); + const summary = document.createElement("div"); + summary.className = "alert alert-info mt-3"; + summary.innerHTML = `<strong>Finalising upload...</strong>`; + container.append(summary); + const form = document.createElement("form"); + form.method = "POST"; + form.action = finaliseUrl; + form.style.display = "none"; + const csrfInput = document.createElement("input"); + csrfInput.type = "hidden"; + csrfInput.name = "csrf_token"; + csrfInput.value = csrfToken; + form.append(csrfInput); + document.body.append(form); + form.submit(); + }, + + handleUploadProgress(e, progressBar, statusEl, percentEl, cancelBtn) { + if (e.lengthComputable) { + const pct = Math.round((e.loaded / e.total) * 100); + progressBar.value = pct; + delete progressBar.dataset.indeterminate; + percentEl.textContent = `${pct}%`; + if (pct >= 100) { + statusEl.textContent = "Processing..."; + cancelBtn.classList.add("d-none"); + } else { + statusEl.textContent = `${this.formatBytes(e.loaded)} / ${this.formatBytes(e.total)}`; + } + } else { + progressBar.removeAttribute("value"); + progressBar.dataset.indeterminate = "true"; + percentEl.textContent = ""; + statusEl.textContent = `${this.formatBytes(e.loaded)} uploaded`; + } + }, +}; diff --git a/atr/static/js/src/upload-progress.js b/atr/static/js/src/upload-progress.js new file mode 100644 index 0000000..a9ef76a --- /dev/null +++ b/atr/static/js/src/upload-progress.js @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +function handleXhrLoad(xhr, ctx) { + ctx.activeUploads.delete(ctx.index); + if (xhr.status >= 200 && xhr.status < 300) { + UploadUI.markUploadDone(ctx.ui, true, "Staged"); + ctx.state.completedCount++; + } else { + let msg = "Upload failed"; + try { + msg = JSON.parse(xhr.responseText).error || msg; + } catch { + /* ignore */ + } + UploadUI.markUploadDone(ctx.ui, false, `Failed: ${msg}`); + ctx.state.failedCount++; + ctx.addRetryButton(ctx.ui, ctx.index); + } + ctx.checkAllComplete(); +} + +function handleXhrFailure(ctx, reason, isCancelled) { + ctx.activeUploads.delete(ctx.index); + if (isCancelled) { + ctx.statusEl.textContent = "Cancelled"; + ctx.statusEl.className = "upload-status text-secondary"; + ctx.cancelBtn.classList.add("d-none"); + } else { + UploadUI.markUploadDone(ctx.ui, false, `Failed: ${reason}`); + ctx.addRetryButton(ctx.ui, ctx.index); + } + ctx.state.failedCount++; + ctx.checkAllComplete(); +} + +function setupXhrHandlers( + xhr, + ui, + index, + state, + addRetryButton, + checkAllComplete, +) { + const ctx = { + ui, + index, + state, + addRetryButton, + checkAllComplete, + activeUploads: state.activeUploads, + statusEl: ui.querySelector(".upload-status"), + cancelBtn: ui.querySelector(".cancel-btn"), + }; + const progressBar = ui.querySelector("progress"); + const percentEl = ui.querySelector(".upload-percent"); + + xhr.upload.addEventListener("progress", (e) => + UploadUI.handleUploadProgress( + e, + progressBar, + ctx.statusEl, + percentEl, + ctx.cancelBtn, + ), + ); + xhr.addEventListener("load", () => handleXhrLoad(xhr, ctx)); + xhr.addEventListener("error", () => + handleXhrFailure(ctx, "Network error", false), + ); + xhr.addEventListener("abort", () => handleXhrFailure(ctx, null, true)); +} + +function checkAllComplete(state, container, finaliseUrl, csrfToken) { + if (state.activeUploads.size > 0) return; + if (state.completedCount === 0) { + UploadUI.showAllFailedSummary(container); + } else if (state.failedCount > 0) { + UploadUI.showPartialSuccessSummary( + container, + state.completedCount, + state.totalFiles, + () => UploadUI.doFinalise(container, finaliseUrl, csrfToken), + ); + } else { + UploadUI.doFinalise(container, finaliseUrl, csrfToken); + } +} + +function startUpload( + file, + index, + state, + stageUrl, + csrfToken, + addRetryButton, + checkComplete, +) { + const ui = document.getElementById(`upload-file-${index}`); + const xhr = new XMLHttpRequest(); + state.activeUploads.set(index, xhr); + setupXhrHandlers(xhr, ui, index, state, addRetryButton, checkComplete); + const formData = new FormData(); + formData.append("csrf_token", csrfToken); + formData.append("file", file, file.name); + xhr.open("POST", stageUrl, true); + xhr.send(formData); + ui.querySelector(".upload-status").textContent = "Uploading..."; +} + +function addRetryButton( + ui, + index, + state, + fileInput, + stageUrl, + csrfToken, + checkComplete, +) { + const retryBtn = document.createElement("button"); + retryBtn.type = "button"; + retryBtn.className = "btn btn-sm btn-outline-primary retry-btn ms-2"; + retryBtn.textContent = "Retry"; + retryBtn.addEventListener("click", () => { + UploadUI.resetUploadUI(ui); + retryBtn.remove(); + state.failedCount--; + const file = Array.from(fileInput.files || [])[index]; + startUpload( + file, + index, + state, + stageUrl, + csrfToken, + addRetryButton, + checkComplete, + ); + }); + ui.querySelector(".upload-status").parentNode.append(retryBtn); +} + +function handleFormSubmit( + e, + form, + fileInput, + container, + state, + stageUrl, + csrfToken, + checkComplete, +) { + e.preventDefault(); + const files = Array.from(fileInput.files || []); + if (files.length === 0) { + alert("Please select files to upload."); + return; + } + state.totalFiles = files.length; + state.completedCount = 0; + state.failedCount = 0; + form.classList.add("d-none"); + container.classList.remove("d-none"); + container.innerHTML = ""; + files.forEach((file, index) => { + container.append( + UploadUI.createFileProgressUI(file, index, state.activeUploads), + ); + startUpload( + file, + index, + state, + stageUrl, + csrfToken, + (ui, idx) => + addRetryButton( + ui, + idx, + state, + fileInput, + stageUrl, + csrfToken, + checkComplete, + ), + checkComplete, + ); + }); +} + +(() => { + const config = document.getElementById("upload-config"); + if (!config) return; + + const stageUrl = config.dataset.stageUrl; + const finaliseUrl = config.dataset.finaliseUrl; + if (!stageUrl || !finaliseUrl) return; + + const form = document + .querySelector('input[name="variant"][value="add_files"]') + ?.closest("form"); + if (!form) return; + + const fileInput = form.querySelector('input[type="file"]'); + const container = document.getElementById("upload-progress-container"); + const csrfToken = form.querySelector('input[name="csrf_token"]')?.value; + const state = { + activeUploads: new Map(), + completedCount: 0, + failedCount: 0, + totalFiles: 0, + }; + + const checkComplete = () => + checkAllComplete(state, container, finaliseUrl, csrfToken); + + form.addEventListener("submit", (e) => + handleFormSubmit( + e, + form, + fileInput, + container, + state, + stageUrl, + csrfToken, + checkComplete, + ), + ); +})(); diff --git a/atr/util.py b/atr/util.py index 0c6ebff..e3fbdf0 100644 --- a/atr/util.py +++ b/atr/util.py @@ -482,6 +482,12 @@ def get_unfinished_dir() -> pathlib.Path: return pathlib.Path(config.get().UNFINISHED_STORAGE_DIR) +def get_upload_staging_dir(session_token: str) -> pathlib.Path: + if not session_token.isalnum(): + raise ValueError("Invalid session token") + return get_tmp_dir() / "upload-staging" / session_token + + async def get_urls_as_completed(urls: Sequence[str]) -> AsyncGenerator[tuple[str, int | str | None, bytes]]: """GET a list of URLs in parallel and yield (url, status, content_bytes) as they become available.""" async with aiohttp.ClientSession() as session: diff --git a/playwright/test.py b/playwright/test.py index 1dca1cf..bbca524 100755 --- a/playwright/test.py +++ b/playwright/test.py @@ -150,7 +150,12 @@ def lifecycle_03_add_file(page: Page, credentials: Credentials, version_name: st expect(submit_button_locator).to_be_enabled() submit_button_locator.click() - logging.info(f"Waiting for navigation to /compose/{TEST_PROJECT}/{version_name} after adding file") + logging.info("Waiting for upload progress UI to appear") + progress_container = page.locator("#upload-progress-container") + expect(progress_container).to_be_visible() + + logging.info("Waiting for upload to complete and redirect to compose page") + page.wait_for_url(f"**/compose/{TEST_PROJECT}/{version_name}*", timeout=30000) wait_for_path(page, f"/compose/{TEST_PROJECT}/{version_name}") logging.info("Add file actions completed successfully") --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
