This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new 9106048 Use exclusion files in RAT checks
9106048 is described below
commit 91060481163985aee42b13aefca3f80726324c85
Author: Sean B. Palmer <[email protected]>
AuthorDate: Tue Jun 24 17:08:33 2025 +0100
Use exclusion files in RAT checks
- Moves the .gitignore matcher to the util module
- Fixes some problems with extracting archives
- Adds path tracking in archives for efficient file search
- Searches for .rat-excludes or rat-excludes.txt in archives
- Ensures that RAT checks also fail on unrecognised licenses
- Improves the RAT check message
- Updates the test key for the browser tests
- Adds a script to automatically re-sign the browser test files
---
atr/archives.py | 77 +++++---
atr/db/interaction.py | 10 +-
atr/tasks/checks/__init__.py | 19 +-
atr/tasks/checks/license.py | 2 +-
atr/tasks/checks/paths.py | 4 +-
atr/tasks/checks/rat.py | 214 +++++++++++++--------
atr/tasks/sbom.py | 2 +-
atr/util.py | 17 +-
.../0924BA4B875A1A472D147C1EB11EB02801756B9E.asc | 42 ++++
.../E35604DD9E2892E5465B3D8A203F105A7B33A64F.asc | 64 ------
.../apache-tooling-test-example-0.2.tar.gz | Bin 4408 -> 4432 bytes
.../apache-tooling-test-example-0.2.tar.gz.asc | 18 +-
.../apache-tooling-test-example-0.2.tar.gz.sha512 | 2 +-
playwright/mk.sh | 43 +++++
playwright/test.py | 10 +-
15 files changed, 314 insertions(+), 210 deletions(-)
diff --git a/atr/archives.py b/atr/archives.py
index 0175fb7..1a4a490 100644
--- a/atr/archives.py
+++ b/atr/archives.py
@@ -36,37 +36,41 @@ def extract(
extract_dir: str,
max_size: int,
chunk_size: int,
-) -> int:
+ track_files: bool | set[str] = False,
+) -> tuple[int, list[str]]:
+ _LOGGER.info(f"Extracting {archive_path} to {extract_dir}")
+
total_extracted = 0
+ extracted_paths = []
try:
with tarzip.open_archive(archive_path) as archive:
match archive.specific():
case tarfile.TarFile() as tf:
for member in tf:
- keep_going, total_extracted = archive_extract_member(
- tf, member, extract_dir, total_extracted,
max_size, chunk_size
+ total_extracted, extracted_paths =
_archive_extract_member(
+ tf, member, extract_dir, total_extracted,
max_size, chunk_size, track_files, extracted_paths
)
- if not keep_going:
- break
case zipfile.ZipFile():
for member in archive:
if not isinstance(member, tarzip.ZipMember):
continue
- keep_going, total_extracted =
_zip_archive_extract_member(
- archive, member, extract_dir, total_extracted,
max_size, chunk_size
+ total_extracted, extracted_paths =
_zip_archive_extract_member(
+ archive,
+ member,
+ extract_dir,
+ total_extracted,
+ max_size,
+ chunk_size,
+ track_files,
+ extracted_paths,
)
- if not keep_going:
- break
-
- case _:
- raise ExtractionError("Unsupported archive type",
{"archive_path": archive_path})
except (tarfile.TarError, zipfile.BadZipFile, ValueError) as e:
raise ExtractionError(f"Failed to read archive: {e}", {"archive_path":
archive_path}) from e
- return total_extracted
+ return total_extracted, extracted_paths
def total_size(tgz_path: str, chunk_size: int = 4096) -> int:
@@ -125,16 +129,27 @@ def _archive_extract_safe_process_file(
return extracted_file_size
-def archive_extract_member(
- tf: tarfile.TarFile, member: tarfile.TarInfo, extract_dir: str,
total_extracted: int, max_size: int, chunk_size: int
-) -> tuple[bool, int]:
- if member.name and member.name.split("/")[-1].startswith("._"):
+def _archive_extract_member(
+ tf: tarfile.TarFile,
+ member: tarfile.TarInfo,
+ extract_dir: str,
+ total_extracted: int,
+ max_size: int,
+ chunk_size: int,
+ track_files: bool | set[str] = False,
+ extracted_paths: list[str] = [],
+) -> tuple[int, list[str]]:
+ member_basename = os.path.basename(member.name)
+ if member_basename.startswith("._"):
# Metadata convention
- return False, 0
+ return 0, extracted_paths
# Skip any character device, block device, or FIFO
if member.isdev():
- return False, 0
+ return 0, extracted_paths
+
+ if track_files and isinstance(track_files, set) and (member_basename in
track_files):
+ extracted_paths.append(member.name)
# Check whether extraction would exceed the size limit
if member.isreg() and ((total_extracted + member.size) > max_size):
@@ -149,7 +164,7 @@ def archive_extract_member(
target_path = os.path.join(extract_dir, member.name)
if not
os.path.abspath(target_path).startswith(os.path.abspath(extract_dir)):
_LOGGER.warning(f"Skipping potentially unsafe path: {member.name}")
- return False, 0
+ return 0, extracted_paths
tf.extract(member, extract_dir, numeric_owner=True)
elif member.isreg():
@@ -164,7 +179,7 @@ def archive_extract_member(
elif member.islnk():
_archive_extract_safe_process_hardlink(member, extract_dir)
- return True, total_extracted
+ return total_extracted, extracted_paths
def _archive_extract_safe_process_hardlink(member: tarfile.TarInfo,
extract_dir: str) -> None:
@@ -261,9 +276,15 @@ def _zip_archive_extract_member(
total_extracted: int,
max_size: int,
chunk_size: int,
-) -> tuple[bool, int]:
- if member.name.split("/")[-1].startswith("._"):
- return False, 0
+ track_files: bool | set[str] = False,
+ extracted_paths: list[str] = [],
+) -> tuple[int, list[str]]:
+ member_basename = os.path.basename(member.name)
+ if track_files and (isinstance(track_files, set) and (member_basename in
track_files)):
+ extracted_paths.append(member.name)
+
+ if member_basename.startswith("._"):
+ return 0, extracted_paths
if member.isfile() and (total_extracted + member.size) > max_size:
raise ExtractionError(
@@ -275,17 +296,17 @@ def _zip_archive_extract_member(
target_path = os.path.join(extract_dir, member.name)
if not
os.path.abspath(target_path).startswith(os.path.abspath(extract_dir)):
_LOGGER.warning("Skipping potentially unsafe path: %s",
member.name)
- return False, 0
+ return 0, extracted_paths
os.makedirs(target_path, exist_ok=True)
- return True, total_extracted
+ return total_extracted, extracted_paths
if member.isfile():
extracted_size = _zip_extract_safe_process_file(
archive, member, extract_dir, total_extracted, max_size, chunk_size
)
- return True, total_extracted + extracted_size
+ return total_extracted + extracted_size, extracted_paths
- return False, total_extracted
+ return total_extracted, extracted_paths
def _zip_extract_safe_process_file(
diff --git a/atr/db/interaction.py b/atr/db/interaction.py
index cff67ae..01d0abf 100644
--- a/atr/db/interaction.py
+++ b/atr/db/interaction.py
@@ -85,11 +85,15 @@ async def key_user_add(
added_keys = []
for key in keys:
- asf_uid = await util.asf_uid_from_uids(key.get("uids", []),
ldap_data=ldap_data)
- if (key.get("fingerprint") or "").upper() ==
"E35604DD9E2892E5465B3D8A203F105A7B33A64F":
+ uids = key.get("uids", [])
+ asf_uid = await util.asf_uid_from_uids(uids, ldap_data=ldap_data)
+ test_key_uids = ["Apache Tooling (For test use only)
<[email protected]>"]
+ is_admin = user.is_admin(session_asf_uid)
+ if (uids == test_key_uids) and is_admin:
# Allow the test key
# TODO: We should fix the test key, not add an exception for it
- pass
+ # But the admin check probably makes this safe enough
+ asf_uid = session_asf_uid
elif session_asf_uid and (asf_uid != session_asf_uid):
# TODO: Give a more detailed error message about why and what to do
raise InteractionError(f"Key {key.get('fingerprint', '').upper()}
is not associated with your ASF account")
diff --git a/atr/tasks/checks/__init__.py b/atr/tasks/checks/__init__.py
index 7e07d0c..f08e445 100644
--- a/atr/tasks/checks/__init__.py
+++ b/atr/tasks/checks/__init__.py
@@ -23,7 +23,6 @@ import functools
import pathlib
from typing import TYPE_CHECKING, Any
-import gitignore_parser
import sqlmodel
if TYPE_CHECKING:
@@ -163,7 +162,7 @@ class Recorder:
project = await self.project()
if not project.policy_binary_artifact_paths:
return False
- matches = _create_path_matcher(
+ matches = util.create_path_matcher(
project.policy_binary_artifact_paths, self.abs_path_base() /
".ignore", self.abs_path_base()
)
abs_path = await self.abs_path()
@@ -175,7 +174,7 @@ class Recorder:
project = await self.project()
if not project.policy_source_artifact_paths:
return False
- matches = _create_path_matcher(
+ matches = util.create_path_matcher(
project.policy_source_artifact_paths, self.abs_path_base() /
".ignore", self.abs_path_base()
)
abs_path = await self.abs_path()
@@ -254,17 +253,3 @@ def with_model(cls: type[schema.Strict]) ->
Callable[[Callable[..., Any]], Calla
return wrapper
return decorator
-
-
-def _create_path_matcher(lines: list[str], full_path: pathlib.Path, base_dir:
pathlib.Path) -> Callable[[str], bool]:
- rules = []
- negation = False
- for line_no, line in enumerate(lines, start=1):
- rule = gitignore_parser.rule_from_pattern(line.rstrip("\n"),
base_path=base_dir, source=(full_path, line_no))
- if rule:
- rules.append(rule)
- if rule.negation:
- negation = True
- if not negation:
- return lambda file_path: any(r.match(file_path) for r in rules)
- return lambda file_path: gitignore_parser.handle_negation(file_path, rules)
diff --git a/atr/tasks/checks/license.py b/atr/tasks/checks/license.py
index cf4c6fa..834c2ae 100644
--- a/atr/tasks/checks/license.py
+++ b/atr/tasks/checks/license.py
@@ -205,7 +205,7 @@ def _files_check_core_logic(artifact_path: str) ->
Iterator[Result]:
# Check for license files in the root directory
with tarzip.open_archive(artifact_path) as archive:
for member in archive:
- _LOGGER.warning(f"Checking member: {member.name}")
+ # _LOGGER.warning(f"Checking member: {member.name}")
if member.name and member.name.split("/")[-1].startswith("._"):
# Metadata convention
continue
diff --git a/atr/tasks/checks/paths.py b/atr/tasks/checks/paths.py
index daf5a9c..3d7e4f4 100644
--- a/atr/tasks/checks/paths.py
+++ b/atr/tasks/checks/paths.py
@@ -26,6 +26,7 @@ import atr.analysis as analysis
import atr.tasks.checks as checks
import atr.util as util
+_ALLOWED_TOP_LEVEL = {"CHANGES", "LICENSE", "NOTICE", "README"}
_LOGGER: Final = logging.getLogger(__name__)
@@ -179,8 +180,7 @@ async def _check_path_process_single(
await _check_metadata_rules(base_path, relative_path, relative_paths,
ext_metadata, errors, warnings)
else:
_LOGGER.info("Checking general rules for %s", full_path)
- allowed_top_level = {"CHANGES", "LICENSE", "NOTICE", "README"}
- if (relative_path.parent == pathlib.Path(".")) and (relative_path.name
not in allowed_top_level):
+ if (relative_path.parent == pathlib.Path(".")) and (relative_path.name
not in _ALLOWED_TOP_LEVEL):
warnings.append(f"Unknown top level file: {relative_path.name}")
for error in errors:
diff --git a/atr/tasks/checks/rat.py b/atr/tasks/checks/rat.py
index 716a9e1..4c30693 100644
--- a/atr/tasks/checks/rat.py
+++ b/atr/tasks/checks/rat.py
@@ -18,6 +18,7 @@
import asyncio
import logging
import os
+import pathlib
import subprocess
import tempfile
import xml.etree.ElementTree as ElementTree
@@ -26,6 +27,7 @@ from typing import Any, Final
import atr.archives as archives
import atr.config as config
import atr.tasks.checks as checks
+import atr.util as util
_CONFIG: Final = config.get()
_JAVA_MEMORY_ARGS: Final[list[str]] = []
@@ -39,6 +41,7 @@ _JAVA_MEMORY_ARGS: Final[list[str]] = []
# "-XX:CompressedClassSpaceSize=16m"
# ]
_LOGGER: Final = logging.getLogger(__name__)
+_RAT_EXCLUDES_FILENAMES: Final[set[str]] = {".rat-excludes",
"rat-excludes.txt"}
async def check(args: checks.FunctionArguments) -> str | None:
@@ -128,7 +131,13 @@ def _check_core_logic(
# Extract the archive to the temporary directory
_LOGGER.info(f"Extracting {artifact_path} to {temp_dir}")
- extracted_size = archives.extract(artifact_path, temp_dir,
max_size=max_extract_size, chunk_size=chunk_size)
+ extracted_size, extracted_paths = archives.extract(
+ artifact_path,
+ temp_dir,
+ max_size=max_extract_size,
+ chunk_size=chunk_size,
+ track_files=_RAT_EXCLUDES_FILENAMES,
+ )
_LOGGER.info(f"Extracted {extracted_size} bytes")
# Find the root directory
@@ -143,7 +152,9 @@ def _check_core_logic(
_LOGGER.info(f"Using root directory: {extract_dir}")
# Execute RAT and get results or error
- error_result, xml_output_path =
_check_core_logic_execute_rat(rat_jar_path, extract_dir, temp_dir)
+ error_result, xml_output_path = _check_core_logic_execute_rat(
+ rat_jar_path, extract_dir, temp_dir, extracted_paths
+ )
if error_result:
return error_result
@@ -175,7 +186,7 @@ def _check_core_logic(
def _check_core_logic_execute_rat(
- rat_jar_path: str, extract_dir: str, temp_dir: str
+ rat_jar_path: str, extract_dir: str, temp_dir: str, excluded_paths:
list[str]
) -> tuple[dict[str, Any] | None, str | None]:
"""Execute Apache RAT and process its output."""
# Define output file path
@@ -191,21 +202,20 @@ def _check_core_logic_execute_rat(
*_JAVA_MEMORY_ARGS,
"-jar",
rat_jar_path,
- "-d",
- extract_dir,
"-x",
"-o",
xml_output_path,
- "--exclude",
- "LICENSE",
- "--exclude",
- "NOTICE",
+ "-d",
+ "--",
+ ".",
]
+ if excluded_paths:
+ _rat_apply_exclusions(extract_dir, excluded_paths, temp_dir)
_LOGGER.info(f"Running Apache RAT: {' '.join(command)}")
- # Change working directory to temp_dir when running the process
+ # Change working directory to extract_dir when running the process
current_dir = os.getcwd()
- os.chdir(temp_dir)
+ os.chdir(extract_dir)
_LOGGER.info(f"Executing Apache RAT from directory: {os.getcwd()}")
@@ -283,10 +293,7 @@ def _check_core_logic_execute_rat(
os.chdir(current_dir)
# Check that the output file exists
- if os.path.exists(xml_output_path):
- _LOGGER.info(f"Found XML output at: {xml_output_path} (size:
{os.path.getsize(xml_output_path)} bytes)")
- return None, xml_output_path
- else:
+ if not os.path.exists(xml_output_path):
_LOGGER.error(f"XML output file not found at: {xml_output_path}")
# List files in the temporary directory
_LOGGER.info(f"Files in {temp_dir}: {os.listdir(temp_dir)}")
@@ -304,6 +311,10 @@ def _check_core_logic_execute_rat(
"errors": [f"Missing output file: {xml_output_path}"],
}, None
+ # The XML was found correctly
+ _LOGGER.info(f"Found XML output at: {xml_output_path} (size:
{os.path.getsize(xml_output_path)} bytes)")
+ return None, xml_output_path
+
def _check_core_logic_jar_exists(rat_jar_path: str) -> tuple[str, dict[str,
Any] | None]:
"""Verify that the Apache RAT JAR file exists and is accessible."""
@@ -356,68 +367,9 @@ def _check_core_logic_jar_exists(rat_jar_path: str) ->
tuple[str, dict[str, Any]
def _check_core_logic_parse_output(xml_file: str, base_dir: str) -> dict[str,
Any]:
- """Parse the XML output from Apache RAT."""
+ """Parse the XML output from Apache RAT safely."""
try:
- tree = ElementTree.parse(xml_file)
- root = tree.getroot()
-
- total_files = 0
- approved_licenses = 0
- unapproved_licenses = 0
- unknown_licenses = 0
-
- unapproved_files = []
- unknown_license_files = []
-
- # Process each resource
- for resource in root.findall(".//resource"):
- total_files += 1
-
- # Get the name attribute value
- name = resource.get("name", "")
-
- # Remove base_dir prefix for cleaner display
- if name.startswith(base_dir):
- name = name[len(base_dir) :].lstrip("/")
-
- # Get license information
- license_approval = resource.find("license-approval")
- license_family = resource.find("license-family")
-
- is_approved = license_approval is not None and
license_approval.get("name") == "true"
- license_name = license_family.get("name") if license_family is not
None else "Unknown"
-
- # Update counters and lists
- if is_approved:
- approved_licenses += 1
- elif license_name == "Unknown license":
- unknown_licenses += 1
- unknown_license_files.append({"name": name, "license":
license_name})
- else:
- unapproved_licenses += 1
- unapproved_files.append({"name": name, "license":
license_name})
-
- # Calculate overall validity
- valid = unapproved_licenses == 0
-
- # Prepare awkwardly long summary message
- message = f"""\
-Found {approved_licenses} files with approved licenses, {unapproved_licenses} \
-with unapproved licenses, and {unknown_licenses} with unknown licenses"""
-
- # We limit the number of files we report to 100
- return {
- "valid": valid,
- "message": message,
- "total_files": total_files,
- "approved_licenses": approved_licenses,
- "unapproved_licenses": unapproved_licenses,
- "unknown_licenses": unknown_licenses,
- "unapproved_files": unapproved_files[:100],
- "unknown_license_files": unknown_license_files[:100],
- "errors": [],
- }
-
+ return _check_core_logic_parse_output_core(xml_file, base_dir)
except Exception as e:
_LOGGER.error(f"Error parsing RAT output: {e}")
return {
@@ -431,6 +383,70 @@ with unapproved licenses, and {unknown_licenses} with
unknown licenses"""
}
+def _check_core_logic_parse_output_core(xml_file: str, base_dir: str) ->
dict[str, Any]:
+ """Parse the XML output from Apache RAT."""
+ tree = ElementTree.parse(xml_file)
+ root = tree.getroot()
+
+ total_files = 0
+ approved_licenses = 0
+ unapproved_licenses = 0
+ unknown_licenses = 0
+
+ unapproved_files = []
+ unknown_license_files = []
+
+ # Process each resource
+ for resource in root.findall(".//resource"):
+ total_files += 1
+
+ # Get the name attribute value
+ name = resource.get("name", "")
+
+ # Remove base_dir prefix for cleaner display
+ if name.startswith(base_dir):
+ name = name[len(base_dir) :].lstrip("/")
+
+ # Get license information
+ license_approval = resource.find("license-approval")
+ license_family = resource.find("license-family")
+
+ is_approved = True
+ if license_approval is not None:
+ if license_approval.get("name") == "false":
+ is_approved = False
+ license_name = license_family.get("name") if (license_family is not
None) else "Unknown"
+
+ # Update counters and lists
+ if is_approved:
+ approved_licenses += 1
+ elif license_name == "Unknown license":
+ unknown_licenses += 1
+ unknown_license_files.append({"name": name, "license":
license_name})
+ else:
+ unapproved_licenses += 1
+ unapproved_files.append({"name": name, "license": license_name})
+
+ # Calculate overall validity
+ valid = (unapproved_licenses == 0) and (unknown_licenses == 0)
+
+ # Prepare a summary message of just the right length
+ message = _summary_message(valid, unapproved_licenses, unknown_licenses)
+
+ # We limit the number of files we report to 100
+ return {
+ "valid": valid,
+ "message": message,
+ "total_files": total_files,
+ "approved_licenses": approved_licenses,
+ "unapproved_licenses": unapproved_licenses,
+ "unknown_licenses": unknown_licenses,
+ "unapproved_files": unapproved_files[:100],
+ "unknown_license_files": unknown_license_files[:100],
+ "errors": [],
+ }
+
+
def _check_java_installed() -> dict[str, Any] | None:
# Check that Java is installed
# TODO: Run this only once, when the server starts
@@ -478,6 +494,7 @@ def _check_java_installed() -> dict[str, Any] | None:
def _extracted_dir(temp_dir: str) -> str | None:
# Loop through all the dirs in temp_dir
extract_dir = None
+ _LOGGER.info(f"Checking directories in {temp_dir}: {os.listdir(temp_dir)}")
for dir_name in os.listdir(temp_dir):
if dir_name.startswith("."):
continue
@@ -489,3 +506,48 @@ def _extracted_dir(temp_dir: str) -> str | None:
else:
raise ValueError(f"Multiple root directories found: {extract_dir},
{dir_path}")
return extract_dir
+
+
+def _rat_apply_exclusions(extract_dir: str, excluded_paths: list[str],
temp_dir: str) -> None:
+ """Apply exclusions to the extracted directory."""
+ # Exclusions are difficult using the command line version of RAT
+ # Each line is interpreted as a literal AND a glob AND a regex
+ # Then, if ANY of those three match a filename, the file is excluded
+ # You cannot specify which syntax to use; all three are always tried
+ # You cannot specify that you want to match against the whole path
+ # Therefore, we take a different approach
+ # We interpret the exclusion file as a glob file in .gitignore format
+ # Then, we simply remove any files that match the glob
+ exclusion_lines = []
+ for excluded_path in excluded_paths:
+ abs_excluded_path = os.path.join(temp_dir, excluded_path)
+ if not os.path.exists(abs_excluded_path):
+ _LOGGER.error(f"Exclusion file not found: {abs_excluded_path}")
+ continue
+ if not os.path.isfile(abs_excluded_path):
+ _LOGGER.error(f"Exclusion file is not a file: {abs_excluded_path}")
+ continue
+ with open(abs_excluded_path, encoding="utf-8") as f:
+ exclusion_lines.extend(f.readlines())
+ matcher = util.create_path_matcher(
+ exclusion_lines, pathlib.Path(extract_dir) / ".ignore",
pathlib.Path(extract_dir)
+ )
+ for root, _dirs, files in os.walk(extract_dir):
+ for file in files:
+ abs_path = os.path.join(root, file)
+ if matcher(abs_path):
+ _LOGGER.info(f"Removing {abs_path} because it matches the
exclusion")
+ os.remove(abs_path)
+
+
+def _summary_message(valid: bool, unapproved_licenses: int, unknown_licenses:
int) -> str:
+ message = "All files have approved licenses"
+ if not valid:
+ message = "Found "
+ if unapproved_licenses > 0:
+ message += f"{unapproved_licenses} files with unapproved licenses"
+ if unknown_licenses > 0:
+ message += " and "
+ if unknown_licenses > 0:
+ message += f"{unknown_licenses} files with unknown licenses"
+ return message
diff --git a/atr/tasks/sbom.py b/atr/tasks/sbom.py
index 7992ac1..4f86c10 100644
--- a/atr/tasks/sbom.py
+++ b/atr/tasks/sbom.py
@@ -86,7 +86,7 @@ async def _generate_cyclonedx_core(artifact_path: str,
output_path: str) -> dict
# Extract the archive to the temporary directory
# TODO: Ideally we'd have task dependencies or archive caching
_LOGGER.info(f"Extracting {artifact_path} to {temp_dir}")
- extracted_size = await asyncio.to_thread(
+ extracted_size, _extracted_paths = await asyncio.to_thread(
archives.extract,
artifact_path,
str(temp_dir),
diff --git a/atr/util.py b/atr/util.py
index ad1878f..0aa3549 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -29,7 +29,7 @@ import tarfile
import tempfile
import uuid
import zipfile
-from collections.abc import AsyncGenerator, Callable, Sequence
+from collections.abc import AsyncGenerator, Callable, Iterable, Sequence
from typing import Any, Final, TypeVar
import aiofiles.os
@@ -37,6 +37,7 @@ import aioshutil
import asfquart
import asfquart.base as base
import asfquart.session as session
+import gitignore_parser
import httpx
import jinja2
import quart
@@ -275,6 +276,20 @@ async def create_hard_link_clone(
await _clone_recursive(source_dir, dest_dir)
+def create_path_matcher(lines: Iterable[str], full_path: pathlib.Path,
base_dir: pathlib.Path) -> Callable[[str], bool]:
+ rules = []
+ negation = False
+ for line_no, line in enumerate(lines, start=1):
+ rule = gitignore_parser.rule_from_pattern(line.rstrip("\n"),
base_path=base_dir, source=(full_path, line_no))
+ if rule:
+ rules.append(rule)
+ if rule.negation:
+ negation = True
+ if not negation:
+ return lambda file_path: any(r.match(file_path) for r in rules)
+ return lambda file_path: gitignore_parser.handle_negation(file_path, rules)
+
+
def email_from_uid(uid: str) -> str | None:
if m := re.search(r"<([^>]+)>", uid):
return m.group(1).lower()
diff --git a/playwright/0924BA4B875A1A472D147C1EB11EB02801756B9E.asc
b/playwright/0924BA4B875A1A472D147C1EB11EB02801756B9E.asc
new file mode 100644
index 0000000..1a0bd14
--- /dev/null
+++ b/playwright/0924BA4B875A1A472D147C1EB11EB02801756B9E.asc
@@ -0,0 +1,42 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: 0924 BA4B 875A 1A47 2D14 7C1E B11E B028 0175 6B9E
+Comment: Apache Tooling (For test use only) <apache-tooling@exam
+
+xjMEaFrKuRYJKwYBBAHaRw8BAQdAf8ene7BE8ixAZMhJJWePxETQjKBHuJL3qAsJ
+c/HGTZ3CwBEEHxYKAIMFgmhayrkFiQWkj70DCwkHCRCxHrAoAXVrnkcUAAAAAAAe
+ACBzYWx0QG5vdGF0aW9ucy5zZXF1b2lhLXBncC5vcmcjaDIP1oXI5qfuRF9d97kf
+3xx/CqciqcHFPU3ZVHpvcAMVCggCm4ECHgEWIQQJJLpLh1oaRy0UfB6xHrAoAXVr
+ngAAAa8A/jB81dDt+cyOj+VTQ3tbrgFU3TlYezjzs/JEqJrTfE9cAQCHb3jK+tZd
+K0C+5YQopDMo/MhkWPjGLvaauBwC8pbbCM1DQXBhY2hlIFRvb2xpbmcgKEZvciB0
+ZXN0IHVzZSBvbmx5KSA8YXBhY2hlLXRvb2xpbmdAZXhhbXBsZS5pbnZhbGlkPsLA
+EwQTFgoAhgWCaFrKuQWJBaSPvQMLCQcJELEesCgBdWueRxQAAAAAAB4AIHNhbHRA
+bm90YXRpb25zLnNlcXVvaWEtcGdwLm9yZ4BMML8pZTlkBSgj3d0nKZ068tOSAQ8E
+v5FIx+WvrCMCAxUKCAKZAQKbgQIeARYhBAkkukuHWhpHLRR8HrEesCgBdWueAACd
+fgD3YpYcOQAaku/qukVgiJ0n0eux0mMIHNMWLbz40X1ouAEAoylhe7ySr47009Oy
+BoDDDlbChPiPjVHkJvfBPumXhwvOMwRoWsq5FgkrBgEEAdpHDwEBB0AGXtYCNilT
+I1M3MUzF3eGK9M7Ax/mSJdjyRgiFy1+5w8LAxQQYFgoBNwWCaFrKuQWJBaSPvQkQ
+sR6wKAF1a55HFAAAAAAAHgAgc2FsdEBub3RhdGlvbnMuc2VxdW9pYS1wZ3Aub3Jn
+VBjBlT1Lse4x/eNxTYapJwSoYHM+L8dnRvMtP49ZYyICm6C+oAQZFgoAbwWCaFrK
+uQkQIUeYql9BqjZHFAAAAAAAHgAgc2FsdEBub3RhdGlvbnMuc2VxdW9pYS1wZ3Au
+b3Jn5xIUllSf3Y6dQlEMi2JeTlTfa+SW+wQCnRwEAKi/yLoWIQTl6SWaidYoB0L8
+2tIhR5iqX0GqNgAATikA/3L0VT44NY7SN3GFtsmso6fFznA5PdesAKYhjtMhpEyV
+AP9hs0bzkb2xK+MxLViv6TqvF4HHqsFhOBFGd2f5Pj7KDhYhBAkkukuHWhpHLRR8
+HrEesCgBdWueAAANngD/Z9PeSYcwlQINT8qGo6r5LyYrr23M3bCTOHRH0viSSrEA
+/2kEz8NUAtCJ00s+j+tNZV1mTR4aX/xL3mNNN8sOLrQNzjMEaFrKuRYJKwYBBAHa
+Rw8BAQdAOIRVNk+dBcuy+ldmuWU4DcA3hQvyl4QAEYFpyqZXYYjCwMUEGBYKATcF
+gmhayrkFiQWkj70JELEesCgBdWueRxQAAAAAAB4AIHNhbHRAbm90YXRpb25zLnNl
+cXVvaWEtcGdwLm9yZ7a10MOsaWwzj2i3UY5A2DdamQmIqlOs6rxXoEia4qnvApuC
+vqAEGRYKAG8FgmhayrkJEE9m+vmdYrj0RxQAAAAAAB4AIHNhbHRAbm90YXRpb25z
+LnNlcXVvaWEtcGdwLm9yZ53GaCuxxCI4KHoMH3+etvJ3bqxnjMJWTOBuJPejPA4J
+FiEEb6YmD+8n2tqPMh+RT2b6+Z1iuPQAAMhWAQDRuD8PGzzv/NZva5wpFfyxrasI
+saKIJ5Hzu+XVPTbV0AEAmgucr3c/NX5ekIN4G1euNHUW8DsDR+oTwJQyshV+Mw8W
+IQQJJLpLh1oaRy0UfB6xHrAoAXVrngAA0wkA/Rl6LcJoz87AvtaD2cTmO2xZwJR0
+jxvw5Rg+KhVBTLLgAP9U6VRz37AAi4i0K2WPRl7lRI0wkL5KDt+w6DihwCsxCs44
+BGhayrkSCisGAQQBl1UBBQEBB0AcxzOd403UVstZNkdaxK466P/GzuPLqwEdnYWI
+SmXuTQMBCAfCwAYEGBYKAHgFgmhayrkFiQWkj70JELEesCgBdWueRxQAAAAAAB4A
+IHNhbHRAbm90YXRpb25zLnNlcXVvaWEtcGdwLm9yZ3spMkAbPj9q1WRlb0Zql/eV
+FPcx6aLEuPfPBXETMvWbApuMFiEECSS6S4daGkctFHwesR6wKAF1a54AAO+SAP4z
+g/bCHhg+q2ZbZMFFupWcNL8ziFbI0PDSNuoaSzB4GwEA8zKo1Ft6uJ7cHvQzp+qX
+sm3HEBYzuBdcSKzUMFOgsQg=
+=1Ulz
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/playwright/E35604DD9E2892E5465B3D8A203F105A7B33A64F.asc
b/playwright/E35604DD9E2892E5465B3D8A203F105A7B33A64F.asc
deleted file mode 100644
index 97ab288..0000000
--- a/playwright/E35604DD9E2892E5465B3D8A203F105A7B33A64F.asc
+++ /dev/null
@@ -1,64 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-
-mQINBGf+gWoBEACy4rPsTxiWX1CpPAg23yOKFGEz759KOJ2Hd+J81V2/Lx1CRTmu
-/zqVY3wUmd7qQXAHfwHSQkpgSv7Gu15VVp7VTWc0ro3DSWVbF9/3p/uOu4b/jjAD
-+FW4gQX/5tQogQPEley2EQ8IiSC99PMxSSbiSmKN/nGjuK9jzrX2YsFcVUr6046n
-IkEnDPsvGmJRuvO5YDBIBw9psxLnE4M3WQCGTehyRk3VsUxbofpJE2P+XLGjmzgT
-3t8is3fql/rr6Gf02o5ioU7Wc4S4/9FbaubL07Ctzrm+PDHjPXiRmzonYNManDIT
-xsb/+YHDqGrL7RNp6dWlwCPiXaqDbh66oURq76uVViZF7e0Lc2005wrxljmKmcbY
-iCmro0CXxqZZhQdk9+ai6CHC1Yq5ejBzuxWtzzmF2IrbPWTU0+Isg00cFs1j0hM2
-bjhSFFzWikeZmj6lUr6xf1T24sPqNtyeUbR43iMhHn6LPYeBdSshTKWYg8ZiQ5WF
-Q60syTqHyWdTqDMmGFETr1mtnPNCADK6jW5Kp95YQbf+xHdj58pTYmM8enEELd0M
-fgm/agJtnVSUZWjNH+WvKIQzZNazMrvsEeSmKK24SgHqNEL5MqzOs1M/Oq6rmTlZ
-LEnbuSg6leaBMNrMYBeSGnSA3unTO8yVz56FWczzU3IW8efO8T4y+xBxcQARAQAB
-tENBcGFjaGUgVG9vbGluZyAoRm9yIHRlc3QgdXNlIG9ubHkpIDxhcGFjaGUtdG9v
-bGluZ0BleGFtcGxlLmludmFsaWQ+iQJSBBMBCgA8FiEE41YE3Z4okuVGWz2KID8Q
-Wnszpk8FAmf+gWoDGy8EBQsJCAcCAiICBhUKCQgLAgQWAgMBAh4HAheAAAoJECA/
-EFp7M6ZPhEwQAI5Ad+w3PgQO6R4U2oOdWDNnk5sTwngQKd1V4TTEJCpNLyvQAABF
-vNNTRuzX9jXpbnsCxYltCofsn8xCn0F/tXnnKSLt8oP+t9j7j2DpT2uvb57zIWfR
-K259CcpxaIm84oJ5ynPY64RnxmMaMsCua8vG3+Ee489D6iLvd/DTSOQV6C++jMfg
-jupwZamWkp0aNbOtHhTXxSLRMXdaszJTTnrJgMB2WdZFN1NtlIcSvfuT1jlDDANB
-IdlKc8h2zrcrCHgBHm9FsYK11FIetZffan+hzFR5if4XdSQfnnIC/x/dXsW0cs3y
-i2SHHStsosHHQ/QiUpg8bNJCEFRPsVgVFNz8MubLD4xCG7MHZSdBM8cYKMu7FMG6
-WW8ha+at42+sAy7vSlnzqI3ccvEI25QgtBPUjzae29p7hXMLLeQwbNNuh5dqBDJY
-pb3mFMFiDkOq3HtE15ln92NIjl0kI0oaaTvLX7szTFaAVOwOhZIYvF7Oyzx1MRPS
-tLrz2C7eNxBojVOzHdrRXSUqPbJKBSS/JA+KMb2e3dKmtCcbJ7esOkgNoGAXkGMQ
-0CW8+Y8yn0w6sXCl1g97rDc8UOARDOvxCqn5J/9kpPB0rvStf2OvoBdGcs2LA3NS
-YTtICW8D+deB04YVQlgaCYbsAFR3oUudBrmwKDzhFO7VZj4lJ0BgXNpfuQINBGf+
-gWoBEADJxvrtglSzvdC+OA8ZYTnlqs3zrx3ohHW3jFMJJDOnRmsbqiidMTODAb3r
-Q7GwWqAAk4PVYyxRs07w6VyXO8iMD/N70nFvWJB7vJpOv1xIk746xnyg1wV6lyuX
-4ry4caIJAIg2d4RrJ3tAypIqOrY8iIvk9DRKnzW2jVju+CBtkuMCKLsKJOihRLUM
-9Ps7MLKHkpD4VpzVuzOgr0p88ovivCpbBVSN5b6dRxAzNsRtT4Jkz2+fBGztVsUO
-GswCN0TwkbxjTsz6JA4MZg4UqQKl9WEREobNMcIdO4rYcfKvUMPfzRF1nwRCAWb3
-zvOx1yjl+p5KxmpxKfx1nl2gOalLb0BaupJdz2HzBoPoWOI3pKFlk3bl8C053XLl
-cVYfvrgY6le4rgyoD1lXw1XAXse0ivncniPFOHtNnN1tlLu/LNbHwJBs+1WKWlBs
-NGRcfamVqbYIE/eL4lZj6IRgLtt+WHsau+KTa8/YJgHhTVVydr2ovb6VIZRg5H3Y
-WfMt16IYDjTjioB36qF2jh4vGSVOmpUoedQM4yuMPnSwyH0GiK2xqgQ4sgDAFKY0
-1D3ZjKhayVnPn8QmPgB5RTI17nupj4q3k07QrZG9JI+tyz5w4aV+SjoecLOHLVk9
-qO5IJRg57Z+A5J9KC4HecFaXQAij2W4I3HnI4xNQhCcYf0DtRwARAQABiQRsBBgB
-CgAgFiEE41YE3Z4okuVGWz2KID8QWnszpk8FAmf+gWoCGy4CQAkQID8QWnszpk/B
-dCAEGQEKAB0WIQQvXmhasW/E2fEg6cxWFx+pj1YlMgUCZ/6BagAKCRBWFx+pj1Yl
-MnkAD/49K5BkOCykKXxpXJ3YeNxW1K4BiQe6XTBPza7KhXZEljSfq/9Z8O3WJBgg
-YDT15rATOaz2Ao4JI4Zl8/kxPqr0j6VEvXVv7Cu9mytHvZgv4i6v1VG4oL2QjSO8
-N2WNon+fxV4niU8itBq4L/MK8CcOwikaATKXuvwPUKXAJLT79pcC6tlLZOIqV87B
-OVVy3TSnoM3zhhTAVGOQtofbCu76myVk+rRUZycF9hAxhilLwdZu+wZGYVBLC8+V
-uNI9eWOT4+zYSXbL8ZJQywNx9QNOFA8VNiGpCVRbE8utCVVzKVw/aAbRmav6Gl/R
-uI6MSx33fIIGAm05UETuEPK5sfro/8tTuCOlT5npvhq5AY6uiU82xaRZ4RNyn4eF
-5zUdUkfX93DlsBOBwgOGSRGp7szvHbq0pdMWGuR+lJM9t/iVemywBsaShUZjelDN
-Uv9mLQ8NoEUDtWIYhn/N1NlDeoJ9a16ZaGqbbxhVsCZUlaGqUS4c3WY4taFzRvT6
-d2Nwvea+U/r9k53FotSKxG9cfZOJwqYRxve1zspbW12SZdz15w4ZzzAUxdgQxpGF
-mZek95cv0I7HMY7Y2vJuN2vRV3jGRHZu2777Wo1ZgwkWB9x45baK28bb9adVUBYG
-kqYtfDO4DtKWDl4dNvp3lKxS4uoL7O4wJNHDom9WahScMq49/PV4D/9+82x+jjhs
-RpGKSj04pNswZJa7mytZx7cWQBKPU0uoWS3CPNuBl8V9UcKfMwbSXSjE2/4UWlAj
-SLNmlKHx438RE3zCVk6N5BCiDIZFhyrqaYuUmW8U0MTrL4qkvWr26lsAUxRXu3nN
-joSM7JaLpjZrLvpj/6BHRpWvLiSDNafObfZt3QABK3DJAqE9oPe8B6FDKydNgea5
-AcPLNOoTt4WgqNbjOmRzE5RyVCkgHCrXyahWY6ZatErGR4ftomKphwLVGCRznP5R
-jLpqeKnxfCR8g0TP0RsZXS2Y31Lwu9U3KJ2fn9TUo5gX/DbW+GGksD3qxIhs4R1P
-5rTppqBSLdTJAaRlk17s3lv1sJ5Px/nZD0r/3bM4rtfNImZvxJtgTedFNtE4KnAL
-2RXfL6rQqsa89HgGfjlcGcTZJGg4M6ekFHCTPX9h8fA6Y0PagWxsDEJpn2vkdIod
-rUzWGuMYxytf/u4W7ZADqDJxXFy5V8Gau+RBnHUR+GHhCbyo3s7rDnWdh3fgwIG6
-Lcbse0fVW4uSnAcqZay0RVObRcAeZpdvJPLMaobMVaMMc1zYrviBh0QeB0TtnnMS
-9ljQ/qzD/JEbTLN8KbGMwMrc1EE0K1zvbmIVD5VIzrI+U6gULQnGDHpB+jx+vtOi
-Qx20dp/Ekji4w6nAtopc4CTjL7YeRFdBKQ==
-=CTh1
------END PGP PUBLIC KEY BLOCK-----
diff --git
a/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz
b/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz
index 1450508..0512b3c 100644
Binary files
a/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz
and
b/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz
differ
diff --git
a/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.asc
b/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.asc
index 9dbc865..7d5b50b 100644
---
a/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.asc
+++
b/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.asc
@@ -1,16 +1,8 @@
-----BEGIN PGP SIGNATURE-----
-iQIzBAABCgAdFiEEL15oWrFvxNnxIOnMVhcfqY9WJTIFAmf+glMACgkQVhcfqY9W
-JTIDrQ//SHU6wH/l6iXg2ILaPdfDgM/dsX2o3OOGtoP2Xr8MoI7yt2cSDduV/xjE
-VX43/v8kIoFqZl/yhYyd3bO7oe3W22cApSwHb2csf9yH4/kx881iAP6JfcsOzKxP
-YQ4xuBrz6N3Vt6I2Fxtgic9FA2XkF86SiQS0++9/gQBOMksFXnAeivlly/81DjeK
-/SuKiL6hgxbKrausLum/e/hEbYy/Fom/demv8Psr9+3f3t1IHPi8iUAVcN1v7LFv
-nnkza7t1c4toNFHmIQYrbHccwjbjJ1uzTBt2LMFzq3QAuTehT7OGgSWM4Vsx/zzC
-UFg1g9ZXNcrv87JFJTQcopN8szY5YlTiRhjBVAGr8jU+3i2XH6EZZr5HlakKy4mG
-vftAekP209pnfK5WIYFbmuSlMsLYBdCj4kVcY0S6sjdEHKb8UEfmtn40vdvH5/QY
-loghG/t10FxPxaVIyaEHY2Z2QZCMEUrYVwLVlDFolSZb0y9ungnyRXzaWFAV0TGm
-SEYRjDxP+w6fxKYSaB37xbLcufO6BOK2UyDjGG1oLj/8gzDeLdYdqbs9YHV/e/xc
-GBDapYpb7BVN3WJBzmvMfRq1P0SrN3W/Mqo8OJjZ8PXyAOag+EcRHcjAlB7sujV6
-Sv2GFEo2Il67JtPYbFPXAg2V91SHDbMYib1lsc3kU6AyBPJN/Gg=
-=aros
+wr0EABYKAG8FgmhayrkJEE9m+vmdYrj0RxQAAAAAAB4AIHNhbHRAbm90YXRpb25z
+LnNlcXVvaWEtcGdwLm9yZ3vpwgIRTsF5adxcqdebGCCmSGKvgOzuVzTFzEdfkCIW
+FiEEb6YmD+8n2tqPMh+RT2b6+Z1iuPQAAPZAAQC2wpFu44Z10oaQHvXAoWdHWH1r
+rh0ypJtdf8m8Z3fU7gD9EbDlFqqr1g3VE+52ftONNAaOOCNFIRUAcajGjphXjws=
+=8tSd
-----END PGP SIGNATURE-----
diff --git
a/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.sha512
b/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.sha512
index d57f986..06bb2c0 100644
---
a/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.sha512
+++
b/playwright/apache-tooling-test-example-0.2/apache-tooling-test-example-0.2.tar.gz.sha512
@@ -1 +1 @@
-9973b48f7e2424621d865a2f9a139fd0da3b63cd2ec240a86e1ede914d524c4d5c95c79bc84c1da2b3d7e41463bb133188fe45b34fc095dd3e1c08a508ccd101
apache-tooling-test-example-0.2.tar.gz
+82709c7c20145bff4a62728a305d93c1a07f3082e209b147dd174dc31ddd151f0164a37a62dddb68c20eb8e21404199cd1863ea946438aecf509ced57d583a8c
apache-tooling-test-example-0.2.tar.gz
diff --git a/playwright/mk.sh b/playwright/mk.sh
new file mode 100755
index 0000000..99d7ff4
--- /dev/null
+++ b/playwright/mk.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+# This script requires Alpine Linux
+# Usage: sh mk.sh
+
+# Use strict mode in POSIX sh
+set -eux
+
+# Add Sequoia to system packages
+apk add cmd:sq
+
+# Set the UID for the key pair
+_uid='Apache Tooling (For test use only) <[email protected]>'
+
+# Generate the secret key
+sq key generate --userid "$_uid" --rev-cert rev-cert.tmp --shared-key \
+ --output tmp.secret.asc --without-password
+
+# Remove the rev cert
+rm rev-cert.tmp
+
+# Extract the fingerprint of the secret key
+_fp=$(sq inspect tmp.secret.asc | awk '/Fingerprint:/ {print $2}')
+
+# Move the secret key to a file name containing its fingerprint
+mv tmp.secret.asc "${_fp}.secret.asc"
+
+# Compute the public key, with a file name containing its fingerprint
+sq key delete --cert-file "${_fp}.secret.asc" --output "${_fp}.asc"
+
+# Enter the directory containing the artifact
+cd apache-tooling-test-example-0.2/
+
+# Generate the SHA-2-512 hash
+sha512sum apache-tooling-test-example-0.2.tar.gz > \
+ apache-tooling-test-example-0.2.tar.gz.sha512
+
+# Generate the signature
+sq sign --signer-file "../${_fp}.secret.asc" \
+ --signature-file apache-tooling-test-example-0.2.tar.gz > \
+ apache-tooling-test-example-0.2.tar.gz.asc
+
+# Remove the secret key
+rm "../${_fp}.secret.asc"
diff --git a/playwright/test.py b/playwright/test.py
index 371e269..8e111be 100644
--- a/playwright/test.py
+++ b/playwright/test.py
@@ -20,6 +20,7 @@
import argparse
import dataclasses
import getpass
+import glob
import logging
import os
import re
@@ -693,9 +694,12 @@ def test_checks_06_targz(page: sync_api.Page, credentials:
Credentials) -> None:
def test_openpgp_01_upload(page: sync_api.Page, credentials: Credentials) ->
None:
- key_fingerprint_lower = "e35604dd9e2892e5465b3d8a203f105a7b33a64f"
- key_fingerprint_upper = key_fingerprint_lower.upper()
- key_path = f"/run/tests/{key_fingerprint_lower.upper()}.asc"
+ for key_path in glob.glob("/run/tests/*.asc"):
+ key_fingerprint_lower =
os.path.basename(key_path).split(".")[0].lower()
+ key_fingerprint_upper = key_fingerprint_lower.upper()
+ break
+ else:
+ raise RuntimeError("No test key found")
logging.info("Starting OpenPGP key upload test")
go_to_path(page, "/keys")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
