This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new c9fdfb8 Allow wildcard LDAP queries for admins, and support LDAP
cache for key upload
c9fdfb8 is described below
commit c9fdfb8bb7ba6dc4905c594de69968364456d2ba
Author: Sean B. Palmer <[email protected]>
AuthorDate: Mon Jun 16 20:50:39 2025 +0100
Allow wildcard LDAP queries for admins, and support LDAP cache for key
upload
---
atr/blueprints/admin/admin.py | 8 +++-
atr/blueprints/admin/templates/ldap-lookup.html | 49 ++++++++++++++++++-------
atr/db/__init__.py | 19 ++++++++++
atr/db/interaction.py | 48 +++++++++++++++++++++---
atr/ldap.py | 10 ++++-
atr/util.py | 20 +++++++---
6 files changed, 126 insertions(+), 28 deletions(-)
diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py
index 9698726..14f769e 100644
--- a/atr/blueprints/admin/admin.py
+++ b/atr/blueprints/admin/admin.py
@@ -21,6 +21,7 @@ import logging
import os
import pathlib
import statistics
+import time
from collections.abc import Callable, Mapping
from typing import Any, Final
@@ -88,7 +89,7 @@ class DeleteReleaseForm(util.QuartFormTyped):
class LdapLookupForm(util.QuartFormTyped):
uid = wtforms.StringField(
"ASF UID (optional)",
- render_kw={"placeholder": "Enter ASF UID, e.g. johnsmith"},
+ render_kw={"placeholder": "Enter ASF UID, e.g. johnsmith, or * for
all"},
)
email = wtforms.StringField(
"Email address (optional)",
@@ -546,13 +547,17 @@ async def ldap_search() -> str:
bind_dn = quart.current_app.config.get("LDAP_BIND_DN")
bind_password = quart.current_app.config.get("LDAP_BIND_PASSWORD")
+ start = time.perf_counter_ns()
ldap_params = ldap.SearchParameters(
uid_query=uid_query,
email_query=email_query,
bind_dn_from_config=bind_dn,
bind_password_from_config=bind_password,
+ email_only=True,
)
await asyncio.to_thread(ldap.search, ldap_params)
+ end = time.perf_counter_ns()
+ _LOGGER.info("LDAP search took %d ms", (end - start) / 1000000)
return await template.render(
"ldap-lookup.html",
@@ -560,6 +565,7 @@ async def ldap_search() -> str:
ldap_params=ldap_params,
asf_id=asf_id_for_template,
ldap_query_performed=ldap_params is not None,
+ uid_query=uid_query,
)
diff --git a/atr/blueprints/admin/templates/ldap-lookup.html
b/atr/blueprints/admin/templates/ldap-lookup.html
index 91c1e89..cbecd9a 100644
--- a/atr/blueprints/admin/templates/ldap-lookup.html
+++ b/atr/blueprints/admin/templates/ldap-lookup.html
@@ -53,30 +53,51 @@
{% endif %}
</div>
{% elif ldap_params and ldap_params.results_list %}
- {% for result in ldap_params.results_list %}
- <table class="table table-sm table-striped">
+ {% if uid_query == '*' %}
+ <table class="table table-sm table-striped table-bordered">
<thead>
<tr>
- <th class="w-25">Attribute</th>
- <th>Value</th>
+ <th>UID</th>
+ <th>Full Name</th>
+ <th>Email</th>
</tr>
</thead>
<tbody>
- {% for key, value in result.items()|sort %}
+ {% for result in ldap_params.results_list %}
<tr>
- <td><strong>{{ key }}</strong></td>
- <td>
- {% if (value is iterable) and (value is not string) %}
- {{ value|join(", ") }}
- {% else %}
- {{ value }}
- {% endif %}
- </td>
+ <td>{{ result.get('uid', [''])[0] }}</td>
+ <td>{{ result.get('cn', [''])[0] }}</td>
+ <td>{{ result.get('mail', [''])[0] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
- {% endfor %}
+ {% else %}
+ {% for result in ldap_params.results_list %}
+ <table class="table table-sm table-striped">
+ <thead>
+ <tr>
+ <th class="w-25">Attribute</th>
+ <th>Value</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for key, value in result.items()|sort %}
+ <tr>
+ <td><strong>{{ key }}</strong></td>
+ <td>
+ {% if (value is iterable) and (value is not string) %}
+ {{ value|join(", ") }}
+ {% else %}
+ {{ value }}
+ {% endif %}
+ </td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ {% endfor %}
+ {% endif %}
{% else %}
<div class="alert alert-info" role="alert">No results found for the
given criteria.</div>
{% endif %}
diff --git a/atr/db/__init__.py b/atr/db/__init__.py
index 0e48e90..1af7501 100644
--- a/atr/db/__init__.py
+++ b/atr/db/__init__.py
@@ -34,6 +34,7 @@ import sqlmodel.sql.expression as expression
import atr.config as config
import atr.db.models as models
+import atr.schema as schema
import atr.util as util
if TYPE_CHECKING:
@@ -115,6 +116,24 @@ class Query(Generic[T]):
result = await self.session.execute(self.query)
return result.scalars().all()
+ async def bulk_upsert(self, items: list[schema.Strict], log_query: bool =
False) -> None:
+ if not items:
+ return
+
+ self.log_query("bulk_upsert", log_query)
+ model_class = self.query.column_descriptions[0]["type"]
+ stmt =
sqlalchemy.dialects.sqlite.insert(model_class).values([item.model_dump() for
item in items])
+ # TODO: The primary key might not be the index element
+ # For example, we might have a unique constraint on other columns
+ primary_keys = [key.name for key in
sqlalchemy.inspect(model_class).primary_key]
+ update_cols = {
+ col.name: getattr(stmt.excluded, col.name)
+ for col in sqlalchemy.inspect(model_class).c
+ if col.name not in primary_keys
+ }
+ stmt = stmt.on_conflict_do_update(index_elements=primary_keys,
set_=update_cols)
+ await self.session.execute(stmt)
+
# async def execute(self) -> sqlalchemy.Result[tuple[T]]:
# return await self.session.execute(self.query)
diff --git a/atr/db/interaction.py b/atr/db/interaction.py
index 7360a37..e4a4e1a 100644
--- a/atr/db/interaction.py
+++ b/atr/db/interaction.py
@@ -68,7 +68,9 @@ async def ephemeral_gpg_home() -> AsyncGenerator[str]:
yield str(temp_dir)
-async def key_user_add(asf_uid: str | None, public_key: str,
selected_committees: list[str]) -> list[dict]:
+async def key_user_add(
+ asf_uid: str | None, public_key: str, selected_committees: list[str],
ldap_data: dict[str, str] | None = None
+) -> list[dict]:
if not public_key:
raise PublicKeyError("Public key is required")
@@ -77,7 +79,7 @@ async def key_user_add(asf_uid: str | None, public_key: str,
selected_committees
added_keys = []
for key in keys:
- asf_uid = await util.asf_uid_from_uids(key.get("uids", []))
+ asf_uid = await util.asf_uid_from_uids(key.get("uids", []),
ldap_data=ldap_data)
# Store key in database
async with db.session() as data:
added = await key_user_session_add(asf_uid, public_key, key,
selected_committees, data)
@@ -279,7 +281,10 @@ async def unfinished_releases(asfuid: str) -> dict[str,
list[models.Release]]:
async def upload_keys(
- user_committees: list[str], keys_text: str, selected_committees: list[str]
+ user_committees: list[str],
+ keys_text: str,
+ selected_committees: list[str],
+ ldap_data: dict[str, str] | None = None,
) -> tuple[list[dict], int, int, list[str]]:
key_blocks = util.parse_key_blocks(keys_text)
if not key_blocks:
@@ -294,7 +299,36 @@ async def upload_keys(
submitted_committees = selected_committees[:]
# Process each key block
- results = await _upload_process_key_blocks(key_blocks, selected_committees)
+ results = await _upload_process_key_blocks(key_blocks,
selected_committees, ldap_data=ldap_data)
+ # if not results:
+ # raise InteractionError("No keys were added")
+
+ success_count = sum(1 for result in results if result["status"] ==
"success")
+ error_count = len(results) - success_count
+
+ return results, success_count, error_count, submitted_committees
+
+
+async def upload_keys_bytes(
+ user_committees: list[str],
+ keys_bytes: bytes,
+ selected_committees: list[str],
+ ldap_data: dict[str, str] | None = None,
+) -> tuple[list[dict], int, int, list[str]]:
+ key_blocks = util.parse_key_blocks_bytes(keys_bytes)
+ if not key_blocks:
+ raise InteractionError("No valid GPG keys found in the uploaded file")
+
+ # Ensure that the selected committees are ones of which the user is
actually a member
+ invalid_committees = [committee for committee in selected_committees if
(committee not in user_committees)]
+ if invalid_committees:
+ raise InteractionError(f"Invalid committee selection: {',
'.join(invalid_committees)}")
+
+ # TODO: Do we modify this? Store a copy just in case, for the template to
use
+ submitted_committees = selected_committees[:]
+
+ # Process each key block
+ results = await _upload_process_key_blocks(key_blocks,
selected_committees, ldap_data=ldap_data)
# if not results:
# raise InteractionError("No keys were added")
@@ -397,14 +431,16 @@ async def _successes_errors_warnings(
info.errors.setdefault(pathlib.Path(primary_rel_path),
[]).append(error)
-async def _upload_process_key_blocks(key_blocks: list[str],
selected_committees: list[str]) -> list[dict]:
+async def _upload_process_key_blocks(
+ key_blocks: list[str], selected_committees: list[str], ldap_data:
dict[str, str] | None = None
+) -> list[dict]:
"""Process GPG key blocks and add them to the user's account."""
results: list[dict] = []
# Process each key block
for i, key_block in enumerate(key_blocks):
try:
- added_keys = await key_user_add(None, key_block,
selected_committees)
+ added_keys = await key_user_add(None, key_block,
selected_committees, ldap_data=ldap_data)
for key_info in added_keys:
key_info["status"] = key_info.get("status", "success")
key_info["email"] = key_info.get("email", "Unknown")
diff --git a/atr/ldap.py b/atr/ldap.py
index 2e9b1cc..127bc60 100644
--- a/atr/ldap.py
+++ b/atr/ldap.py
@@ -39,6 +39,7 @@ class SearchParameters:
srv_info: str | None = None
detail_err: str | None = None
connection: ldap3.Connection | None = None
+ email_only: bool = False
def parse_dn(dn_string: str) -> dict[str, list[str]]:
@@ -83,7 +84,10 @@ def _search_core(params: SearchParameters) -> None:
filters: list[str] = []
if params.uid_query:
- filters.append(f"(uid={conv.escape_filter_chars(params.uid_query)})")
+ if params.uid_query == "*":
+ filters.append("(uid=*)")
+ else:
+
filters.append(f"(uid={conv.escape_filter_chars(params.uid_query)})")
if params.email_query:
escaped_email = conv.escape_filter_chars(params.email_query)
@@ -102,10 +106,12 @@ def _search_core(params: SearchParameters) -> None:
params.err_msg = "LDAP Connection object not established or auto_bind
failed."
return
+ email_attributes = ["uid", "mail", "asf-altEmail", "asf-committer-email"]
+ attributes = email_attributes if params.email_only else
ldap3.ALL_ATTRIBUTES
params.connection.search(
search_base=LDAP_SEARCH_BASE,
search_filter=search_filter,
- attributes=ldap3.ALL_ATTRIBUTES,
+ attributes=attributes,
)
for entry in params.connection.entries:
result_item: dict[str, str | list[str]] = {"dn": entry.entry_dn}
diff --git a/atr/util.py b/atr/util.py
index d96f380..39cba67 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -152,7 +152,9 @@ def asf_uid_from_email(email: str) -> str | None:
return ldap_uid_val[0] if isinstance(ldap_uid_val, list) else ldap_uid_val
-async def asf_uid_from_uids(uids: list[str]) -> str | None:
+async def asf_uid_from_uids(
+ uids: list[str], use_ldap: bool = True, ldap_data: dict[str, str] | None =
None
+) -> str | None:
# Determine ASF UID if not provided
emails = []
for uid_str in uids:
@@ -162,10 +164,18 @@ async def asf_uid_from_uids(uids: list[str]) -> str |
None:
return email.removesuffix("@apache.org")
emails.append(email)
# We did not find a direct @apache.org email address
- # Therefore, search LDAP
- for email in emails:
- if asf_uid := await asyncio.to_thread(asf_uid_from_email, email):
- return asf_uid
+ # Therefore, search LDAP data, either cached or directly
+ if ldap_data is not None:
+ # Use cached LDAP data
+ for email in emails:
+ if email in ldap_data:
+ return ldap_data[email]
+ return None
+ if use_ldap:
+ # Search LDAP directly
+ for email in emails:
+ if asf_uid := await asyncio.to_thread(asf_uid_from_email, email):
+ return asf_uid
return None
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
