Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package obs-service-cargo_audit for
openSUSE:Factory checked in at 2023-06-06 19:56:05
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/obs-service-cargo_audit (Old)
and /work/SRC/openSUSE:Factory/.obs-service-cargo_audit.new.15902 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "obs-service-cargo_audit"
Tue Jun 6 19:56:05 2023 rev:12 rq:1090995 version:0.1.8~6
Changes:
--------
---
/work/SRC/openSUSE:Factory/obs-service-cargo_audit/obs-service-cargo_audit.changes
2022-07-06 15:42:16.526541617 +0200
+++
/work/SRC/openSUSE:Factory/.obs-service-cargo_audit.new.15902/obs-service-cargo_audit.changes
2023-06-06 19:56:48.450668286 +0200
@@ -1,0 +2,15 @@
+Tue Jun 06 00:18:25 UTC 2023 - william.br...@suse.com
+
+- Update to version 0.1.8~6:
+ * fix: include zstd decompression for tarballs (#8)
+ * Update to exclude broken packages
+ * Resolve typo in do_scan
+ * Improve do_scan
+ * improve do_scan :)
+ * Updates
+ * Fix workspace scanning
+ * Improve output to show fast/slow path updates
+ * Resolve issue with multibuild tests
+ * Improve handling when a lock is requested but not existing
+
+-------------------------------------------------------------------
Old:
----
obs-service-cargo_audit-0.1.8~1.tar.xz
New:
----
obs-service-cargo_audit-0.1.8~6.tar.xz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ obs-service-cargo_audit.spec ++++++
--- /var/tmp/diff_new_pack.Lm8Ff2/_old 2023-06-06 19:56:49.114672223 +0200
+++ /var/tmp/diff_new_pack.Lm8Ff2/_new 2023-06-06 19:56:49.114672223 +0200
@@ -1,7 +1,7 @@
#
# spec file
#
-# Copyright (c) 2022 SUSE LLC
+# Copyright (c) 2023 SUSE LLC
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -22,7 +22,7 @@
License: MPL-2.0
Group: Development/Tools/Building
URL: https://github.com/openSUSE/obs-service-%{service}
-Version: 0.1.8~1
+Version: 0.1.8~6
Release: 0
Source: %{name}-%{version}.tar.xz
Source99: obs-service-cargo_audit-rpmlintrc
++++++ obs-service-cargo_audit-0.1.8~1.tar.xz ->
obs-service-cargo_audit-0.1.8~6.tar.xz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/obs-service-cargo_audit-0.1.8~1/README.md
new/obs-service-cargo_audit-0.1.8~6/README.md
--- old/obs-service-cargo_audit-0.1.8~1/README.md 2022-04-05
09:17:15.000000000 +0200
+++ new/obs-service-cargo_audit-0.1.8~6/README.md 2023-06-05
01:13:54.000000000 +0200
@@ -128,6 +128,18 @@
python3 do_scan.py --assume-setup --rustsec-id XXXX
```
+You can scan specific packages with `--package`.
+
+```
+python3 do_scan.py --package a b c
+```
+
+To scan a different OBS instance you can set the url and repo name
+
+```
+python3 do_scan.py --api https://api.opensuse.org --repo openSUSE:Factory
+```
+
## License
MPL-2.0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/obs-service-cargo_audit-0.1.8~1/cargo_audit
new/obs-service-cargo_audit-0.1.8~6/cargo_audit
--- old/obs-service-cargo_audit-0.1.8~1/cargo_audit 2022-04-05
09:17:15.000000000 +0200
+++ new/obs-service-cargo_audit-0.1.8~6/cargo_audit 2023-06-05
01:13:54.000000000 +0200
@@ -29,6 +29,7 @@
import os
import shutil
import json
+import zstandard
from subprocess import run
from subprocess import PIPE
@@ -76,7 +77,7 @@
log.error(f" Could not generate Cargo.lock under {path}")
log.error(f"{output}")
log.error(f"{os.listdir(path)}")
- exit(1)
+ raise ValueError(f" Could not generate Cargo.lock under {path}")
def cargo_audit(lock_file):
log.debug(f"Running cargo audit against: {lock_file}")
@@ -115,7 +116,7 @@
categories = vuln["advisory"]["categories"]
log.error(f" * {vid} -> crate: {affects}, cvss: {cvss}, class:
{categories}")
return details
- log.info(f" â
No known issues detected in {lock_file}")
+ log.debug(f" â
No known issues detected in {lock_file}")
return None
def do_extract(from_dir, tgt_dir):
@@ -126,8 +127,21 @@
]
for src in maybe_src:
log.debug(f"Unpacking assumed source tar {src} to {tgt_dir}")
- with tarfile.open(f"{from_dir}/{src}", "r:*") as tar:
- tar.extractall(path=tgt_dir)
+ # Code for zst decompression is
+ # based or copied from
https://github.com/openSUSE/obs-service-cargo_vendor/blob/4780427e1640bdd31afc5adb258dc555eec33c40/cargo_vendor#L136-L139
+ if src.endswith(".tar.zst"):
+ log.debug(" zst compressed tarball sources found.")
+ with tempfile.NamedTemporaryFile() as decompressed:
+ with open(src, "rb") as compressed:
+ zstd = zstandard.ZstdDecompressor()
+ zstd.copy_stream(compressed, decompressed)
+ decompressed.seek(0)
+ # Setup the tar
+ with tarfile.open(fileobj=decompressed, mode="r:") as tar:
+ tar.extractall(path=tgt_dir)
+ else:
+ with tarfile.open(f"{from_dir}/{src}", "r:*") as tar:
+ tar.extractall(path=tgt_dir)
return [os.path.join(tgt_dir, x) for x in os.listdir(tgt_dir)]
def do_cargo_audit(workdir, lsrcdir, lockfile):
@@ -178,7 +192,7 @@
# cargo_lock_paths = [os.path.join(gsrcdir,
'Cargo.lock')]
else:
log.error(f" No Rust Cargo.toml found under {lsrcdir}")
- exit(1)
+ raise ValueError(f" No Rust Cargo.toml found under
{lsrcdir}")
# depending on the layout this could be a workspace, or a bunch
# of projects. This changes where the cargo locks MAY end up
so we
# have to rescan here.
@@ -187,7 +201,7 @@
if not cargo_lock_paths:
log.error(f" No Rust Cargo.lock found under {lsrcdir}
after generation")
- exit(1)
+ raise ValueError(f" No Rust Cargo.lock found under
{lsrcdir} after generation")
else:
log.debug(f" Detected Rust lock files: {cargo_lock_paths}")
@@ -226,14 +240,16 @@
lsrcdir = args.srcdir
lockfile = args.lockfile
- status = do_cargo_audit(workdir, lsrcdir, lockfile)
+ try:
+ status = do_cargo_audit(workdir, lsrcdir, lockfile)
- # We exit the context manager, so tempdirname is now removed.
- if len(status) > 0:
- log.error(" â ï¸ Vulnerabilities may have been found. You must
review these.")
+ # We exit the context manager, so tempdirname is now removed.
+ if len(status) > 0:
+ log.error(" â ï¸ Vulnerabilities may have been found. You must
review these.")
+ exit(1)
+ log.info(" ð ð¦ No known issues detected")
+ except Exception as e:
exit(1)
- log.info(" ð ð¦ No known issues detected")
-
if __name__ == "__main__":
main()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/obs-service-cargo_audit-0.1.8~1/do_scan.py
new/obs-service-cargo_audit-0.1.8~6/do_scan.py
--- old/obs-service-cargo_audit-0.1.8~1/do_scan.py 2022-04-05
09:17:15.000000000 +0200
+++ new/obs-service-cargo_audit-0.1.8~6/do_scan.py 2023-06-05
01:13:54.000000000 +0200
@@ -15,17 +15,11 @@
cargo_audit_module = importlib.util.module_from_spec( spec )
loader.exec_module( cargo_audit_module )
-WHATDEPENDS = ["osc", "whatdependson", "openSUSE:Factory", "rust", "standard",
"x86_64"]
-
-CHECKOUT = ["osc", "co", "openSUSE:Factory"]
-UPDATE = ["osc", "up", "openSUSE:Factory"]
-
EXCLUDE = set([
# ALready cared for
'MozillaFirefox',
'MozillaThunderbird',
'rust',
- 'rust1.53',
# Doesn't have any true rust deps.
'obs-service-cargo_audit',
'cargo-audit-advisory-db',
@@ -33,11 +27,27 @@
# Dead
'seamonkey',
'meson:test',
+
+
+ # Temp
+ 'aws-nitro-enclaves-cli',
+ # Broken
+ 'netease-cloud-music-gtk'
])
-def list_whatdepends():
+def list_whatdepends(obs_api, obs_repo):
# osc whatdependson openSUSE:Factory rust standard x86_64
- raw_depends = subprocess.check_output(WHATDEPENDS, encoding='UTF-8')
+ raw_depends = subprocess.check_output(
+ [
+ "osc",
+ "-A", obs_api,
+ "whatdependson",
+ obs_repo,
+ "rust",
+ "standard",
+ "x86_64"
+ ],
+ encoding='UTF-8')
# Split on new lines
raw_depends = raw_depends.split('\n')
@@ -59,42 +69,43 @@
return raw_depends
-def get_develproject(pkgname):
- print(f"intent to scan - openSUSE:Factory/{pkgname}")
+def get_develproject(pkgname, obs_api, obs_repo):
+ print(f"intent to scan - {obs_repo}/{pkgname}")
try:
- out = subprocess.check_output(["osc", "dp",
f"openSUSE:Factory/{pkgname}"])
+ out = subprocess.check_output(["osc", "-A", obs_api, "dp",
f"{obs_repo}/{pkgname}"])
except subprocess.CalledProcessError as e:
- print(f"Failed to retrieve develproject information for
openSUSE:Factory/{pkgname}")
+ print(f"Failed to retrieve develproject information for
{obs_repo}/{pkgname}")
print(e.stdout)
raise e
return out.decode('UTF-8').strip()
-def checkout_or_update(pkgname, should_setup):
+def checkout_or_update(pkgname, should_setup, obs_api, obs_repo):
try:
- if os.path.exists('openSUSE:Factory') and
os.path.exists(f'openSUSE:Factory/{pkgname}'):
- print(f"osc revert openSUSE:Factory/{pkgname}")
+ if os.path.exists(obs_repo) and
os.path.exists(f'{obs_repo}/{pkgname}'):
+ print(f"osc -A {obs_api} revert {obs_repo}/{pkgname}")
# Revert/cleanup if required.
- out = subprocess.check_output(["osc", "revert", "."],
cwd=f"openSUSE:Factory/{pkgname}")
- print(f"osc clean openSUSE:Factory/{pkgname}")
- out = subprocess.check_output(["osc", "clean", "."],
cwd=f"openSUSE:Factory/{pkgname}")
+ out = subprocess.check_output(["osc", "-A", obs_api, "revert",
"."], cwd=f"{obs_repo}/{pkgname}")
+ print(f"osc -A {obs_api} clean {obs_repo}/{pkgname}")
+ out = subprocess.check_output(["osc", "-A", obs_api, "clean",
"."], cwd=f"{obs_repo}/{pkgname}")
if should_setup:
- print(f"osc up openSUSE:Factory/{pkgname}")
- out = subprocess.check_output(["osc", "up",
f"openSUSE:Factory/{pkgname}"])
+ print(f"osc -A {obs_api} up {obs_repo}/{pkgname}")
+ out = subprocess.check_output(["osc", "-A", obs_api, "up",
f"{obs_repo}/{pkgname}"])
elif should_setup:
- print(f"osc co openSUSE:Factory/{pkgname}")
- out = subprocess.check_output(["osc", "co",
f"openSUSE:Factory/{pkgname}"])
+ print(f"osc -A {obs_api} co {obs_repo}/{pkgname}")
+ out = subprocess.check_output(["osc", "-A", obs_api, "co",
f"{obs_repo}/{pkgname}"])
else:
print(f"Nothing to do")
except subprocess.CalledProcessError as e:
- print(f"Failed to checkout or update openSUSE:Factory/{pkgname}")
+ print(f"Failed to checkout or update {obs_repo}/{pkgname}")
print(e.stdout)
raise e
- print(f"done")
+ # Done!
-def does_have_cargo_audit(pkgname):
- service = f"openSUSE:Factory/{pkgname}/_service"
+def does_have_cargo_audit(pkgname, obs_repo):
+ service = f"{obs_repo}/{pkgname}/_service"
if os.path.exists(service):
has_audit = False
+ has_vendor = False
has_vendor_update = False
lockfile = None
tree = ET.parse(service)
@@ -109,6 +120,7 @@
# from our internal calls.
root_node.remove(tag)
if tag.attrib['name'] == 'cargo_vendor':
+ has_vendor = True
for attr in tag:
if attr.attrib['name'] == 'update' and attr.text == 'true':
has_vendor_update = True
@@ -116,17 +128,17 @@
# on vulns.
root_node.remove(tag)
tree.write(service)
- return (True, has_audit, has_vendor_update, lockfile)
- return (False, False, False, None)
+ return (True, has_audit, has_vendor, has_vendor_update, lockfile)
+ return (False, False, False, False, None)
-def do_services(pkgname):
+def do_services(pkgname, obs_api, obs_repo):
cmd = [
"nsjail",
"--really_quiet",
"--config", "scan.cfg",
- "--cwd", f"{os.getcwd()}/openSUSE:Factory/{pkgname}",
+ "--cwd", f"{os.getcwd()}/{obs_repo}/{pkgname}",
"--bindmount", f"{os.getcwd()}:{os.getcwd()}",
- "/usr/bin/osc", "service", "ra"
+ "--", "/usr/bin/osc", "-A", obs_api, "service", "ra"
]
try:
out = subprocess.check_output(cmd, encoding='UTF-8',
stderr=subprocess.STDOUT)
@@ -138,14 +150,14 @@
print(e.stdout)
return False
-def do_unpack_scan(pkgname, lockfile, rustsec_id):
+def do_unpack_scan(pkgname, lockfile, rustsec_id, obs_repo):
try:
scan = cargo_audit_module.do_cargo_audit(
- f"{os.getcwd()}/openSUSE:Factory/{pkgname}",
+ f"{os.getcwd()}/{obs_repo}/{pkgname}",
None,
lockfile)
- except:
- print(f"ð¨ -- cargo_audit was unable to be run")
+ except Exception as e:
+ print(f"ð¨ -- cargo_audit was unable to be run - {e}")
return False
if rustsec_id is not None:
@@ -175,44 +187,50 @@
description="scan OBS gooderer",
formatter_class=argparse.RawDescriptionHelpFormatter
)
+
+ parser.add_argument('-a', '--api', dest='obs_api',
default="https://api.opensuse.org";)
+ parser.add_argument('-r', '--repo', dest='obs_repo',
default="openSUSE:Factory")
parser.add_argument('--assume-setup', dest='should_setup',
action='store_false')
parser.add_argument('--rustsec-id', dest='rustsec_id', default=None)
+ parser.add_argument('--package', dest='package', default=None, nargs='*')
args = parser.parse_args()
- depends = list_whatdepends()
-
- # For testing, we hardcode the list for dev.
- # depends = ["kanidm", "389-ds", "bottom", "helvum"]
- # depends = ["greetd"]
+ if args.package is not None:
+ depends = args.package
+ else:
+ depends = list_whatdepends(args.obs_api, args.obs_repo)
devel_projects = {}
for pkgname in depends:
- devel_projects[pkgname] = get_develproject(pkgname)
+ devel_projects[pkgname] = get_develproject(pkgname, args.obs_api,
args.obs_repo)
lockfiles = {}
# Check them out, or update if they exist.
auditable_depends = []
unpack_depends = []
- need_services = set()
+ need_vendor_services = set()
maybe_vuln = set()
for pkgname in depends:
print("---")
- checkout_or_update(pkgname, args.should_setup)
+ checkout_or_update(pkgname, args.should_setup, args.obs_api,
args.obs_repo)
# do they have cargo_audit as a service? Could we consider adding it?
- (has_services, has_audit, has_vendor_update, lockfile) =
does_have_cargo_audit(pkgname)
+ (has_services, has_audit, has_vendor, has_vendor_update, lockfile) =
does_have_cargo_audit(pkgname, args.obs_repo)
lockfiles[pkgname] = lockfile
+ if not has_vendor:
+ print(f"ð {args.obs_repo}/{pkgname} missing cargo_vendor
service + update - the maintainer should be contacted to add this")
+ need_vendor_services.add(f"{devel_projects[pkgname]}/{pkgname}")
+
if not has_vendor_update:
- print(f"ð openSUSE:Factory/{pkgname} missing cargo_vendor
service + update - the maintainer should be contacted to add this")
- need_services.add(f"{devel_projects[pkgname]}/{pkgname}")
+ print(f"ð {args.obs_repo}/{pkgname} missing cargo_vendor auto
update - the maintainer should be contacted to add this")
+
if not has_audit:
- print(f"â ï¸ openSUSE:Factory/{pkgname} missing cargo_audit
service - the maintainer should be contacted to add this")
+ print(f"â ï¸ {args.obs_repo}/{pkgname} missing cargo_audit
service - the maintainer should be contacted to add this")
# print(f"âï¸
https://build.opensuse.org/package/users/openSUSE:Factory/{pkgname}";)
# If not, we should contact the developers to add this. We can
attempt to unpack
# and run a scan still though.
unpack_depends.append((pkgname, has_services))
- # need_services.add(f"{devel_projects[pkgname]}/{pkgname}")
else:
# If they do, run services. We may not know what they need for
this to work, so we
# have to run the full stack, but at the least, the developer
probably has this
@@ -223,26 +241,26 @@
for pkgname in auditable_depends:
print("---")
print(f"ð running services for
{devel_projects[pkgname]}/{pkgname} ...")
- do_services(pkgname)
+ do_services(pkgname, args.obs_api, args.obs_repo)
for (pkgname, has_services) in unpack_depends:
print("---")
if has_services:
- print(f"ð running services for
{devel_projects[pkgname]}/{pkgname} ...")
- do_services(pkgname)
+ print(f"ð running services for
{devel_projects[pkgname]}/{pkgname} ...")
+ do_services(pkgname, args.obs_api, args.obs_repo)
# Do the thang
for pkgname in depends:
print("---")
print(f"ð¿ unpacking and scanning
{devel_projects[pkgname]}/{pkgname} ...")
lockfile = lockfiles.get(pkgname, None)
- if not do_unpack_scan(pkgname, lockfile, args.rustsec_id):
+ if not do_unpack_scan(pkgname, lockfile, args.rustsec_id,
args.obs_repo):
maybe_vuln.add(f"{devel_projects[pkgname]}/{pkgname}")
- slow_update = maybe_vuln & need_services
+ slow_update = maybe_vuln & need_vendor_services
fast_update = maybe_vuln - slow_update
# We will warn about these anyway since they are in the vuln set.
- need_services -= maybe_vuln
+ need_vendor_services -= maybe_vuln
# Remove items which items can rapid-update from the slow set
maybe_vuln -= fast_update
@@ -254,24 +272,24 @@
else:
print("- the following pkgs need SECURITY updates - svc setup")
for item in fast_update:
- print(f"osc bco {item}")
+ print(f"osc -A {args.obs_api} bco {item}")
print(f" Alternately")
- print(f" python3 do_bulk_update.py %s" % ' '.join(fast_update))
+ print(f" python3 do_bulk_update.py --yolo %s" % ' '.join(fast_update))
if len(slow_update) > 0:
if args.rustsec_id:
- print(f"- the following pkgs need SECURITY updates to address
{args.rustsec_id} - manual")
+ print(f"- the following pkgs need SECURITY updates to address
{args.rustsec_id} - manual, missing cargo_vendor")
else:
print("- the following pkgs need SECURITY updates - manual")
for item in slow_update:
- print(f"osc bco {item}")
+ print(f"osc -A {args.obs_api} bco {item}")
- if len(need_services) > 0:
- print("- the following are NOT vulnerable but SHOULD have services
updated")
- for item in need_services:
- print(f"osc bco {item}")
+ if len(need_vendor_services) > 0:
+ print("- the following are NOT vulnerable but SHOULD have services
updated to include cargo_vendor!")
+ for item in need_vendor_services:
+ print(f"osc -A {args.obs_api} bco {item}")
