This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/sbp by this push:
new e461dacd Move archives into a top level state directory
e461dacd is described below
commit e461dacd55d51bac5a204aaa64a0393fbd2fd8b8
Author: Sean B. Palmer <[email protected]>
AuthorDate: Thu Mar 12 19:00:00 2026 +0000
Move archives into a top level state directory
---
atr/config.py | 2 ++
atr/paths.py | 4 ++--
atr/server.py | 3 +++
atr/storage/writers/release.py | 2 +-
atr/tasks/checks/__init__.py | 2 +-
atr/tasks/quarantine.py | 6 +++---
tests/unit/test_quarantine_backfill.py | 12 +++++++-----
tests/unit/test_quarantine_task.py | 8 ++++----
8 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/atr/config.py b/atr/config.py
index dcce98ff..63a70ef8 100644
--- a/atr/config.py
+++ b/atr/config.py
@@ -103,6 +103,7 @@ class AppConfig:
# TODO: By convention this is at /x1/, but we can symlink it here perhaps?
# TODO: We need to get Puppet to check SVN out initially, or do it manually
SVN_STORAGE_DIR = os.path.join(STATE_DIR, "subversion")
+ ARCHIVES_STORAGE_DIR = os.path.join(STATE_DIR, "archives")
ATTESTABLE_STORAGE_DIR = os.path.join(STATE_DIR, "attestable")
SQLITE_DB_PATH = decouple.config("SQLITE_DB_PATH",
default="database/atr.db")
STORAGE_AUDIT_LOG_FILE = os.path.join(STATE_DIR, "audit",
"storage-audit.log")
@@ -177,6 +178,7 @@ def get() -> type[AppConfig]:
(config.FINISHED_STORAGE_DIR, "FINISHED_STORAGE_DIR"),
(config.UNFINISHED_STORAGE_DIR, "UNFINISHED_STORAGE_DIR"),
(config.SVN_STORAGE_DIR, "SVN_STORAGE_DIR"),
+ (config.ARCHIVES_STORAGE_DIR, "ARCHIVES_STORAGE_DIR"),
(config.ATTESTABLE_STORAGE_DIR, "ATTESTABLE_STORAGE_DIR"),
(config.STORAGE_AUDIT_LOG_FILE, "STORAGE_AUDIT_LOG_FILE"),
(config.PERFORMANCE_LOG_FILE, "PERFORMANCE_LOG_FILE"),
diff --git a/atr/paths.py b/atr/paths.py
index 7931db1d..38ced9e8 100644
--- a/atr/paths.py
+++ b/atr/paths.py
@@ -32,8 +32,8 @@ def get_attestable_dir() -> pathlib.Path:
return pathlib.Path(config.get().ATTESTABLE_STORAGE_DIR)
-def get_cache_archives_dir() -> pathlib.Path:
- return pathlib.Path(config.get().STATE_DIR) / "cache" / "archives"
+def get_archives_dir() -> pathlib.Path:
+ return pathlib.Path(config.get().ARCHIVES_STORAGE_DIR)
def get_downloads_dir() -> pathlib.Path:
diff --git a/atr/server.py b/atr/server.py
index b31418fd..f6d6f5ce 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -73,6 +73,8 @@ app: base.QuartApp | None = None
# The order of these migrations must be checked carefully to avoid conflicts
_MIGRATIONS: Final[list[tuple[str, str]]] = [
+ # Archives
+ ("cache/archives", "archives"),
# Audit
("storage-audit.log", "audit/storage-audit.log"),
# Cache
@@ -176,6 +178,7 @@ def _app_dirs_setup(state_dir_str: str, hot_reload: bool)
-> None:
pathlib.Path(state_dir_str) / "runtime",
pathlib.Path(state_dir_str) / "secrets" / "curated",
pathlib.Path(state_dir_str) / "secrets" / "generated",
+ paths.get_archives_dir(),
paths.get_downloads_dir(),
paths.get_finished_dir(),
paths.get_quarantined_dir(),
diff --git a/atr/storage/writers/release.py b/atr/storage/writers/release.py
index 32394e4b..8503e1c3 100644
--- a/atr/storage/writers/release.py
+++ b/atr/storage/writers/release.py
@@ -108,7 +108,7 @@ class CommitteeParticipant(FoundationCommitter):
release_dirs = [
paths.release_directory_base(release),
paths.get_attestable_dir() / str(project_name) / str(version),
- paths.get_cache_archives_dir() / str(project_name) / str(version),
+ paths.get_archives_dir() / str(project_name) / str(version),
paths.get_quarantined_dir() / str(project_name) / str(version),
]
diff --git a/atr/tasks/checks/__init__.py b/atr/tasks/checks/__init__.py
index bb40eb4a..26912a0d 100644
--- a/atr/tasks/checks/__init__.py
+++ b/atr/tasks/checks/__init__.py
@@ -323,7 +323,7 @@ async def resolve_cache_dir(args: FunctionArguments) ->
pathlib.Path | None:
if content_hash is None:
return None
cache_key = hashes.filesystem_cache_archives_key(content_hash)
- cache_dir = file_paths.get_cache_archives_dir() / str(args.project_name) /
str(args.version_name) / cache_key
+ cache_dir = file_paths.get_archives_dir() / str(args.project_name) /
str(args.version_name) / cache_key
if await aiofiles.os.path.isdir(cache_dir):
return cache_dir
return None
diff --git a/atr/tasks/quarantine.py b/atr/tasks/quarantine.py
index 85f261cf..e5dd9561 100644
--- a/atr/tasks/quarantine.py
+++ b/atr/tasks/quarantine.py
@@ -68,7 +68,7 @@ def backfill_archive_cache() -> list[tuple[str, pathlib.Path,
float]]:
done_file.touch()
return []
- cache_archives_dir = paths.get_cache_archives_dir()
+ cache_archives_dir = paths.get_archives_dir()
staging_base = paths.get_tmp_dir()
staging_base.mkdir(parents=True, exist_ok=True)
extraction_cfg = _extraction_config()
@@ -143,7 +143,7 @@ async def validate(args: QuarantineValidate) ->
results.Results | None:
def _backfill_done_file() -> pathlib.Path:
- return paths.get_cache_archives_dir().parent / "archive-backfill.done"
+ return pathlib.Path(config.get().STATE_DIR) / "cache" /
"archive-backfill.done"
def _backfill_extract_archive(
@@ -220,7 +220,7 @@ async def _extract_archives_to_cache(
version_name: str,
file_entries: list[sql.QuarantineFileEntryV1],
) -> None:
- cache_base = paths.get_cache_archives_dir() / project_name / version_name
+ cache_base = paths.get_archives_dir() / project_name / version_name
staging_base = paths.get_tmp_dir()
await aiofiles.os.makedirs(cache_base, exist_ok=True)
await aiofiles.os.makedirs(staging_base, exist_ok=True)
diff --git a/tests/unit/test_quarantine_backfill.py
b/tests/unit/test_quarantine_backfill.py
index 3f6c125d..f0370441 100644
--- a/tests/unit/test_quarantine_backfill.py
+++ b/tests/unit/test_quarantine_backfill.py
@@ -184,17 +184,19 @@ def _patch_paths(
monkeypatch: pytest.MonkeyPatch,
tmp_path: pathlib.Path,
unfinished_dir: pathlib.Path,
- cache_dir: pathlib.Path,
+ archives_dir: pathlib.Path,
) -> None:
monkeypatch.setattr(quarantine.paths, "get_unfinished_dir", lambda:
unfinished_dir)
- monkeypatch.setattr(quarantine.paths, "get_cache_archives_dir", lambda:
cache_dir)
+ monkeypatch.setattr(quarantine.paths, "get_archives_dir", lambda:
archives_dir)
monkeypatch.setattr(quarantine.paths, "get_tmp_dir", lambda: tmp_path /
"temporary")
+ monkeypatch.setattr(quarantine, "_backfill_done_file", lambda: tmp_path /
"cache" / "archive-backfill.done")
def _setup_dirs(tmp_path: pathlib.Path) -> tuple[pathlib.Path, pathlib.Path]:
unfinished_dir = tmp_path / "unfinished"
- cache_dir = tmp_path / "cache" / "archives"
+ archives_dir = tmp_path / "archives"
+ cache_dir = tmp_path / "cache"
staging_dir = tmp_path / "temporary"
- for d in [unfinished_dir, cache_dir, staging_dir]:
+ for d in [unfinished_dir, archives_dir, cache_dir, staging_dir]:
d.mkdir(parents=True)
- return unfinished_dir, cache_dir
+ return unfinished_dir, archives_dir
diff --git a/tests/unit/test_quarantine_task.py
b/tests/unit/test_quarantine_task.py
index dfbe1072..05efa15c 100644
--- a/tests/unit/test_quarantine_task.py
+++ b/tests/unit/test_quarantine_task.py
@@ -87,7 +87,7 @@ async def
test_extract_archives_to_cache_discards_staging_dir_on_enotempty_colli
(dst_path / "winner.txt").write_text("winner")
raise OSError(errno.ENOTEMPTY, "Directory not empty", str(dst_path))
- monkeypatch.setattr(quarantine.paths, "get_cache_archives_dir", lambda:
cache_root)
+ monkeypatch.setattr(quarantine.paths, "get_archives_dir", lambda:
cache_root)
monkeypatch.setattr(quarantine.paths, "get_tmp_dir", lambda: tmp_root)
monkeypatch.setattr(quarantine.exarch, "extract_archive", extract_archive)
monkeypatch.setattr(quarantine.os, "rename", rename)
@@ -132,7 +132,7 @@ async def
test_extract_archives_to_cache_discards_staging_dir_when_other_worker_
(dst_path / "winner.txt").write_text("winner")
raise FileExistsError(dst)
- monkeypatch.setattr(quarantine.paths, "get_cache_archives_dir", lambda:
cache_root)
+ monkeypatch.setattr(quarantine.paths, "get_archives_dir", lambda:
cache_root)
monkeypatch.setattr(quarantine.paths, "get_tmp_dir", lambda: tmp_root)
monkeypatch.setattr(quarantine.exarch, "extract_archive", extract_archive)
monkeypatch.setattr(quarantine.os, "rename", rename)
@@ -168,7 +168,7 @@ async def
test_extract_archives_to_cache_propagates_exarch_error_to_file_entry(
def extract_archive(_archive_path: str, _extract_dir: str, _config:
object) -> None:
raise RuntimeError("unsafe zip detected")
- monkeypatch.setattr(quarantine.paths, "get_cache_archives_dir", lambda:
cache_root)
+ monkeypatch.setattr(quarantine.paths, "get_archives_dir", lambda:
cache_root)
monkeypatch.setattr(quarantine.paths, "get_tmp_dir", lambda: tmp_root)
monkeypatch.setattr(quarantine.exarch, "extract_archive", extract_archive)
@@ -205,7 +205,7 @@ async def
test_extract_archives_to_cache_stages_in_temporary_then_promotes(
extract_path = pathlib.Path(extract_dir)
(extract_path / "content.txt").write_text("cached")
- monkeypatch.setattr(quarantine.paths, "get_cache_archives_dir", lambda:
cache_root)
+ monkeypatch.setattr(quarantine.paths, "get_archives_dir", lambda:
cache_root)
monkeypatch.setattr(quarantine.paths, "get_tmp_dir", lambda: tmp_root)
monkeypatch.setattr(quarantine.exarch, "extract_archive", extract_archive)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
