Quentin Debhi has proposed merging
~ruinedyourlife/launchpad:craft-build-rust-scan into launchpad:master.
Commit message:
Scan crates inside rust build archives
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/477899
This will find `.crate` files inside `sourcecraft` outputted archives and add
it as the build output.
--
Your team Launchpad code reviewers is requested to review the proposed merge of
~ruinedyourlife/launchpad:craft-build-rust-scan into launchpad:master.
diff --git a/lib/lp/archivepublisher/artifactory.py b/lib/lp/archivepublisher/artifactory.py
index 08eb83a..e435aa7 100644
--- a/lib/lp/archivepublisher/artifactory.py
+++ b/lib/lp/archivepublisher/artifactory.py
@@ -617,7 +617,7 @@ class ArtifactoryPool:
]
elif repository_format == ArchiveRepositoryFormat.RUST:
return [
- "*.tar.xz",
+ "*.crate",
]
elif repository_format == ArchiveRepositoryFormat.GENERIC:
return ["*"]
diff --git a/lib/lp/archivepublisher/tests/test_artifactory.py b/lib/lp/archivepublisher/tests/test_artifactory.py
index 3fc2aa5..a5212b7 100644
--- a/lib/lp/archivepublisher/tests/test_artifactory.py
+++ b/lib/lp/archivepublisher/tests/test_artifactory.py
@@ -290,7 +290,7 @@ class TestArtifactoryPool(TestCase):
def test_getArtifactPatterns_rust(self):
pool = self.makePool()
self.assertEqual(
- ["*.tar.xz"],
+ ["*.crate"],
pool.getArtifactPatterns(ArchiveRepositoryFormat.RUST),
)
diff --git a/lib/lp/archiveuploader/craftrecipeupload.py b/lib/lp/archiveuploader/craftrecipeupload.py
index 5e9d285..a7ed62f 100644
--- a/lib/lp/archiveuploader/craftrecipeupload.py
+++ b/lib/lp/archiveuploader/craftrecipeupload.py
@@ -8,7 +8,10 @@ __all__ = [
]
import os
+import tarfile
+import tempfile
+import yaml
from zope.component import getUtility
from lp.archiveuploader.utils import UploadError
@@ -45,6 +48,7 @@ class CraftRecipeUpload:
# All relevant files will be in a subdirectory.
continue
for craft_file in sorted(filenames):
+ # Look for .tar.xz archives (which might contain .crate files)
if craft_file.endswith(".tar.xz"):
found_craft = True
craft_paths.append(os.path.join(dirpath, craft_file))
@@ -53,18 +57,52 @@ class CraftRecipeUpload:
raise UploadError("Build did not produce any craft files.")
for craft_path in craft_paths:
- with open(craft_path, "rb") as file:
- libraryfile = self.librarian.create(
- os.path.basename(craft_path),
- os.stat(craft_path).st_size,
- file,
- filenameToContentType(craft_path),
- restricted=build.is_private,
- )
- build.addFile(libraryfile)
+ # Extract and process .crate files from archive
+ self._process_rust_archive(build, craft_path)
# The master verifies the status to confirm successful upload.
self.logger.debug("Updating %s" % build.title)
build.updateStatus(BuildStatus.FULLYBUILT)
-
self.logger.debug("Finished upload.")
+
+ def _process_rust_archive(self, build, archive_path):
+ """Process a .tar.xz archive that may contain .crate files."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ with tarfile.open(archive_path, "r:xz") as tar:
+ tar.extractall(path=tmpdir)
+
+ # Read metadata.yaml for crate info
+ # XXX: ruinedyourlife 2024-12-06
+ # We will need this later to give the crate a name and version to
+ # artifactory. This is a placeholder for now, which will be changed
+ # when we find a way to send that information to artifactory.
+ metadata_path = os.path.join(tmpdir, "metadata.yaml")
+ if os.path.exists(metadata_path):
+ with open(metadata_path) as f:
+ try:
+ metadata = yaml.safe_load(f)
+ _ = metadata.get("name")
+ _ = metadata.get("version")
+ except Exception as e:
+ self.logger.warning(
+ "Failed to parse metadata.yaml: %s", e
+ )
+
+ # Look for .crate files in extracted contents
+ for root, _, files in os.walk(tmpdir):
+ for filename in files:
+ if filename.endswith(".crate"):
+ crate_path = os.path.join(root, filename)
+ self._process_crate_file(build, crate_path)
+
+ def _process_crate_file(self, build, crate_path):
+ """Process a single .crate file."""
+ with open(crate_path, "rb") as file:
+ libraryfile = self.librarian.create(
+ os.path.basename(crate_path),
+ os.stat(crate_path).st_size,
+ file,
+ filenameToContentType(crate_path),
+ restricted=build.is_private,
+ )
+ build.addFile(libraryfile)
diff --git a/lib/lp/archiveuploader/tests/test_craftrecipeupload.py b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
index 5f7f737..c137c78 100644
--- a/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
+++ b/lib/lp/archiveuploader/tests/test_craftrecipeupload.py
@@ -4,8 +4,12 @@
"""Tests for `CraftRecipeUpload`."""
import os
+import tarfile
+import tempfile
+import yaml
from storm.store import Store
+from zope.security.proxy import removeSecurityProxy
from lp.archiveuploader.tests.test_uploadprocessor import (
TestUploadProcessorBase,
@@ -39,16 +43,45 @@ class TestCraftRecipeUploads(TestUploadProcessorBase):
self.layer.txn, builds=True
)
+ def _createArchiveWithCrate(
+ self, upload_dir, crate_name="test-crate", crate_version="0.1.0"
+ ):
+ """Helper to create a tar.xz archive containing a crate & metadata."""
+ # Create a temporary directory to build our archive
+ with tempfile.TemporaryDirectory() as tmpdir:
+ # Create metadata.yaml
+ metadata = {
+ "name": crate_name,
+ "version": crate_version,
+ }
+ metadata_path = os.path.join(tmpdir, "metadata.yaml")
+ with open(metadata_path, "w") as f:
+ yaml.safe_dump(metadata, f)
+
+ # Create dummy crate file
+ crate_path = os.path.join(
+ tmpdir, f"{crate_name}-{crate_version}.crate"
+ )
+ with open(crate_path, "wb") as f:
+ f.write(b"dummy crate contents")
+
+ # Create tar.xz archive
+ archive_path = os.path.join(upload_dir, "output.tar.xz")
+ with tarfile.open(archive_path, "w:xz") as tar:
+ tar.add(metadata_path, arcname="metadata.yaml")
+ tar.add(crate_path, arcname=os.path.basename(crate_path))
+
+ return archive_path
+
def test_sets_build_and_state(self):
# The upload processor uploads files and sets the correct status.
self.assertFalse(self.build.verifySuccessfulUpload())
upload_dir = os.path.join(
self.incoming_folder, "test", str(self.build.id), "ubuntu"
)
- write_file(
- os.path.join(upload_dir, "foo_0_all.craft.tar.xz"), b"craft"
- )
- write_file(os.path.join(upload_dir, "foo_0_all.manifest"), b"manifest")
+ os.makedirs(upload_dir, exist_ok=True)
+ self._createArchiveWithCrate(upload_dir)
+
handler = UploadHandler.forProcessor(
self.uploadprocessor, self.incoming_folder, "test", self.build
)
@@ -61,6 +94,42 @@ class TestCraftRecipeUploads(TestUploadProcessorBase):
self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
self.assertTrue(self.build.verifySuccessfulUpload())
+ # Verify that the crate file was properly extracted and stored
+ build = removeSecurityProxy(self.build)
+ files = list(build.getFiles())
+ self.assertEqual(1, len(files))
+ stored_file = files[0][1]
+ self.assertTrue(stored_file.filename.endswith(".crate"))
+
+ def test_processes_crate_from_archive(self):
+ """Test extracting/processing crates within .tar.xz archives."""
+ upload_dir = os.path.join(
+ self.incoming_folder, "test", str(self.build.id), "ubuntu"
+ )
+ os.makedirs(upload_dir, exist_ok=True)
+
+ # Create archive with specific crate name and version
+ crate_name = "test-crate"
+ crate_version = "0.2.0"
+ self._createArchiveWithCrate(upload_dir, crate_name, crate_version)
+
+ handler = UploadHandler.forProcessor(
+ self.uploadprocessor, self.incoming_folder, "test", self.build
+ )
+ result = handler.processCraftRecipe(self.log)
+
+ # Verify upload succeeded
+ self.assertEqual(UploadStatusEnum.ACCEPTED, result)
+ self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
+
+ # Verify the crate file was properly stored
+ build = removeSecurityProxy(self.build)
+ files = list(build.getFiles())
+ self.assertEqual(1, len(files))
+ stored_file = files[0][1]
+ expected_filename = f"{crate_name}-{crate_version}.crate"
+ self.assertEqual(expected_filename, stored_file.filename)
+
def test_requires_craft(self):
# The upload processor fails if the upload does not contain any
# .craft files.
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index 100e3ba..65d2861 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -330,6 +330,9 @@ class CIBuildUploadJob(ArchiveJobDerived):
SourcePackageFileType.GO_MODULE_MOD,
SourcePackageFileType.GO_MODULE_ZIP,
},
+ # XXX: ruinedyourlife 2024-12-06
+ # Remove the Rust format and it's scanner as we don't need it from
+ # CI builds. We only care about crates in craft recipe uploads.
ArchiveRepositoryFormat.RUST: {
BinaryPackageFormat.CRATE,
},
_______________________________________________
Mailing list: https://launchpad.net/~launchpad-reviewers
Post to : launchpad-reviewers@lists.launchpad.net
Unsubscribe : https://launchpad.net/~launchpad-reviewers
More help : https://help.launchpad.net/ListHelp
