This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new aca2188 Allow admins to start the key import script, and improve
logging
aca2188 is described below
commit aca2188365983dcfbaaeff13b19eb5a6fbc8daa9
Author: Sean B. Palmer <[email protected]>
AuthorDate: Tue Jun 17 14:53:35 2025 +0100
Allow admins to start the key import script, and improve logging
---
atr/blueprints/admin/admin.py | 102 ++++++++++++------------
atr/blueprints/admin/templates/update-keys.html | 7 +-
scripts/keys_import.py | 57 ++++++++++---
3 files changed, 105 insertions(+), 61 deletions(-)
diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py
index 14f769e..eaf36f8 100644
--- a/atr/blueprints/admin/admin.py
+++ b/atr/blueprints/admin/admin.py
@@ -21,6 +21,7 @@ import logging
import os
import pathlib
import statistics
+import sys
import time
from collections.abc import Callable, Mapping
from typing import Any, Final
@@ -359,27 +360,29 @@ async def admin_env() -> quart.wrappers.response.Response:
@admin.BLUEPRINT.route("/keys/update", methods=["GET", "POST"])
async def admin_keys_update() -> str | response.Response | tuple[Mapping[str,
Any], int]:
"""Update keys from remote data."""
- if quart.request.method == "POST":
- try:
- added_count, updated_count = await _update_keys()
- return {
- "message": f"Successfully added {added_count} and updated
{updated_count} keys",
- "category": "success",
- }, 200
- except httpx.RequestError as e:
- return {
- "message": f"Failed to fetch data: {e!s}",
- "category": "error",
- }, 200
- except Exception as e:
- return {
- "message": f"Failed to update projects: {e!s}",
- "category": "error",
- }, 200
+ if quart.request.method != "POST":
+ empty_form = await util.EmptyForm.create_form()
+ # Get the previous output from the log file
+ log_path = pathlib.Path("keys_import.log")
+ if not await aiofiles.os.path.exists(log_path):
+ previous_output = None
+ else:
+ async with aiofiles.open(log_path) as f:
+ previous_output = await f.read()
+ return await template.render("update-keys.html",
empty_form=empty_form, previous_output=previous_output)
- # For GET requests, show the update form
- empty_form = await util.EmptyForm.create_form()
- return await template.render("update-keys.html", empty_form=empty_form)
+ try:
+ pid = await _update_keys()
+ return {
+ "message": f"Successfully started key update process with PID
{pid}",
+ "category": "success",
+ }, 200
+ except Exception as e:
+ _LOGGER.exception("Failed to start key update process")
+ return {
+ "message": f"Failed to update keys: {e!s}",
+ "category": "error",
+ }, 200
@admin.BLUEPRINT.route("/performance")
@@ -797,38 +800,37 @@ async def _update_committees(
return added_count, updated_count
-async def _update_keys() -> tuple[int, int]:
- import httpx
+async def _update_keys() -> int:
+ async def _log_process(process: asyncio.subprocess.Process) -> None:
+ try:
+ stdout, stderr = await process.communicate()
+ if stdout:
+ _LOGGER.info(f"keys_import.py
stdout:\n{stdout.decode('utf-8')[:1000]}")
+ if stderr:
+ _LOGGER.error(f"keys_import.py
stderr:\n{stderr.decode('utf-8')[:1000]}")
+ except Exception:
+ _LOGGER.exception("Error reading from subprocess for
keys_import.py")
- successes = 0
- failures = 0
- async with db.session() as data:
- # Get all committees
- committees = await data.committee().all()
- for committee in committees:
- # Get the KEYS file
- async with httpx.AsyncClient() as client:
- response = await
client.get(f"https://downloads.apache.org/{committee.name}/KEYS";)
- try:
- response.raise_for_status()
- except httpx.HTTPStatusError as e:
- _LOGGER.error(f"Failed to fetch KEYS file for
{committee.name}: {e!s}")
- continue
- keys_data = await response.aread()
- keys_text = keys_data.decode("utf-8", errors="replace")
- # results, success_count, error_count, submitted_committees
- try:
- _result, yes, no, _committees = await interaction.upload_keys(
- [committee.name], keys_text, [committee.name]
- )
- except interaction.InteractionError as e:
- _LOGGER.error(f"Failed to update keys for {committee.name}:
{e!s}")
- continue
- _LOGGER.info(f"Updated keys for {committee.name}: {yes} successes,
{no} failures")
- successes += yes
- failures += no
+ app = asfquart.APP
+ if not hasattr(app, "background_tasks"):
+ app.background_tasks = set()
+
+ if await aiofiles.os.path.exists("../Dockerfile.alpine"):
+ # Not in a container, developing locally
+ command = ["poetry", "run", "python3", "scripts/keys_import.py"]
+ else:
+ # In a container
+ command = [sys.executable, "scripts/keys_import.py"]
+
+ process = await asyncio.create_subprocess_exec(
+ *command, stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE, cwd=".."
+ )
+
+ task = asyncio.create_task(_log_process(process))
+ app.background_tasks.add(task)
+ task.add_done_callback(app.background_tasks.discard)
- return successes, failures
+ return process.pid
async def _update_metadata() -> tuple[int, int]:
diff --git a/atr/blueprints/admin/templates/update-keys.html
b/atr/blueprints/admin/templates/update-keys.html
index 6fa65fb..4c23eed 100644
--- a/atr/blueprints/admin/templates/update-keys.html
+++ b/atr/blueprints/admin/templates/update-keys.html
@@ -88,9 +88,14 @@
<form action="javascript:submitForm().then(_ => { return false; })">
{{ empty_form.hidden_tag() }}
- <button type="submit" id="submitButton">Update projects</button>
+ <button type="submit" id="submitButton">Update keys</button>
</form>
+ {% if previous_output %}
+ <h2>Previous output</h2>
+ <pre>{{ previous_output }}</pre>
+ {% endif %}
+
<script>
const submitForm = async () => {
const button = document.getElementById("submitButton");
diff --git a/scripts/keys_import.py b/scripts/keys_import.py
index a86ed74..7e3f73c 100644
--- a/scripts/keys_import.py
+++ b/scripts/keys_import.py
@@ -2,6 +2,8 @@
# Usage: poetry run python3 scripts/keys_import.py
import asyncio
+import contextlib
+import os
import sys
import time
@@ -15,7 +17,7 @@ import atr.ldap as ldap
import atr.util as util
-def get(entry, prop):
+def get(entry: dict, prop: str) -> str | None:
if prop in entry:
values = entry[prop]
if values:
@@ -23,9 +25,35 @@ def get(entry, prop):
return None
-async def amain():
+def write(message: str) -> None:
+ print(message)
+ sys.stdout.flush()
+
+
[email protected]
+def log_to_file(conf: config.AppConfig):
+ log_file_path = os.path.join(conf.STATE_DIR, "keys_import.log")
+ # This should not be required
+ os.makedirs(conf.STATE_DIR, exist_ok=True)
+
+ original_stdout = sys.stdout
+ original_stderr = sys.stderr
+ with open(log_file_path, "a") as f:
+ sys.stdout = f
+ sys.stderr = f
+ try:
+ yield
+ finally:
+ sys.stdout = original_stdout
+ sys.stderr = original_stderr
+
+
+async def keys_import(conf: config.AppConfig) -> None:
# Runs as a standalone script, so we need a worker style database
connection
await db.init_database_for_worker()
+ # Print the time and current PID
+ print(f"--- {time.strftime('%Y-%m-%d %H:%M:%S')} by pid {os.getpid()} ---")
+ sys.stdout.flush()
# Get all email addresses in LDAP
# We'll discard them when we're finished
@@ -41,7 +69,7 @@ async def amain():
start = time.perf_counter_ns()
await asyncio.to_thread(ldap.search, ldap_params)
end = time.perf_counter_ns()
- print("LDAP search took", (end - start) / 1000000, "ms")
+ write(f"LDAP search took {(end - start) / 1000000} ms")
# Map the LDAP addresses to Apache UIDs
email_to_uid = {}
@@ -53,7 +81,7 @@ async def amain():
email_to_uid[alt_email] = uid
if committer_email := get(entry, "asf-committer-email"):
email_to_uid[committer_email] = uid
- print("Email addresses from LDAP:", len(email_to_uid))
+ write(f"Email addresses from LDAP: {len(email_to_uid)}")
# Open an ATR database connection
async with db.session() as data:
@@ -68,7 +96,7 @@ async def amain():
# For each remote KEYS file, check that it responded 200 OK
committee_name = url.rsplit("/", 2)[-2]
if status != 200:
- print(committee_name, "error:", status)
+ write(f"{committee_name} error: {status}")
continue
# Parse the KEYS file and add it to the database
@@ -79,18 +107,27 @@ async def amain():
[committee_name], content, [committee_name],
ldap_data=email_to_uid
)
except Exception as e:
- print(committee_name, "error:", e)
+ write(f"{committee_name} error: {e}")
continue
# Print and record the number of keys that were okay and failed
- print(committee_name, yes, no)
+ write(f"{committee_name} {yes} {no}")
total_yes += yes
total_no += no
- print("Total okay:", total_yes)
- print("Total failed:", total_no)
+ write(f"Total okay: {total_yes}")
+ write(f"Total failed: {total_no}")
+ end = time.perf_counter_ns()
+ write(f"Script took {(end - start) / 1000000} ms")
+ write("")
+
+
+async def amain() -> None:
+ conf = config.AppConfig()
+ with log_to_file(conf):
+ await keys_import(conf)
-def main():
+def main() -> None:
asyncio.run(amain())
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
