This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new eb6ea6c2d5 More automation when retrieving Airlfow dependencies health
status (#41653)
eb6ea6c2d5 is described below
commit eb6ea6c2d5e0b2ca30c4f66a1154994e7730d608
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Aug 25 14:22:10 2024 +0200
More automation when retrieving Airlfow dependencies health status (#41653)
* More automation when retrieving Airlfow dependencies health status
The command to retrieve Airflow dependencies health status aims to
automate retrieval of all the information that has been so far
manually derived and to put it automatically into Google Spreadsheet.
What's done so far:
* Google Spreadsheet integration
* Filtering of relevant OPSF scorecard values
* Converting OPSF scorecard details into comments for relevant cells
* Automatically deriving "Governance", "Livecycle status" and
"Unpatched vulnerabilities" (no importance yet - that will need
Github integration.
* fixup! More automation when retrieving Airlfow dependencies health status
---
dev/breeze/README.md | 2 +-
.../output_sbom_export-dependency-information.svg | 92 +++++-
.../output_sbom_export-dependency-information.txt | 2 +-
dev/breeze/pyproject.toml | 3 +
.../src/airflow_breeze/commands/sbom_commands.py | 334 +++++++++++++++++++--
.../commands/sbom_commands_config.py | 31 +-
dev/breeze/src/airflow_breeze/utils/cdxgen.py | 173 ++++++++++-
7 files changed, 573 insertions(+), 64 deletions(-)
diff --git a/dev/breeze/README.md b/dev/breeze/README.md
index 2c38aa7c1a..15c0b66f57 100644
--- a/dev/breeze/README.md
+++ b/dev/breeze/README.md
@@ -66,6 +66,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY
UPDATED BY PRE-COMMIT.
---------------------------------------------------------------------------------------------------------
-Package config hash:
f8e8729f4236f050d4412cbbc9d53fdd4e6ddad65ce5fafd3c5b6fcdacbea5431eea760b961534a63fd5733b072b38e8167b5b0c12ee48b31c3257306ef11940
+Package config hash:
8e382ff46231b261a569886a45480104eb5436434d2845c3eb011ee9dd4da3c2fa33f561eaa36f2245a29c8719ae2e86d7ffec39463c46e0b3b4bde56a27abe6
---------------------------------------------------------------------------------------------------------
diff --git
a/dev/breeze/doc/images/output_sbom_export-dependency-information.svg
b/dev/breeze/doc/images/output_sbom_export-dependency-information.svg
index e9387591c9..0f525cd60a 100644
--- a/dev/breeze/doc/images/output_sbom_export-dependency-information.svg
+++ b/dev/breeze/doc/images/output_sbom_export-dependency-information.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 489.2"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 879.5999999999999"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -45,7 +45,7 @@
<defs>
<clipPath id="breeze-sbom-export-dependency-information-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="438.2" />
+ <rect x="0" y="0" width="1463.0" height="828.5999999999999" />
</clipPath>
<clipPath id="breeze-sbom-export-dependency-information-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -98,9 +98,57 @@
<clipPath id="breeze-sbom-export-dependency-information-line-16">
<rect x="0" y="391.9" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-17">
+ <rect x="0" y="416.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-18">
+ <rect x="0" y="440.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-19">
+ <rect x="0" y="465.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-20">
+ <rect x="0" y="489.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-21">
+ <rect x="0" y="513.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-22">
+ <rect x="0" y="538.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-23">
+ <rect x="0" y="562.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-24">
+ <rect x="0" y="587.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-25">
+ <rect x="0" y="611.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-26">
+ <rect x="0" y="635.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-27">
+ <rect x="0" y="660.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-28">
+ <rect x="0" y="684.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-29">
+ <rect x="0" y="709.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-30">
+ <rect x="0" y="733.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-31">
+ <rect x="0" y="757.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-sbom-export-dependency-information-line-32">
+ <rect x="0" y="782.3" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="487.2" rx="8"/><text
class="breeze-sbom-export-dependency-information-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: sbom export-dependency-information</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="877.6" rx="8"/><text
class="breeze-sbom-export-dependency-information-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: sbom export-dependency-information</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -116,18 +164,34 @@
</text><text class="breeze-sbom-export-dependency-information-r1" x="12.2"
y="93.2" textLength="488"
clip-path="url(#breeze-sbom-export-dependency-information-line-3)">Export dependency information from SBOM.</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="93.2"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-3)">
</text><text class="breeze-sbom-export-dependency-information-r1" x="1464"
y="117.6" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-4)">
</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="142" textLength="24.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-5)">╭─</text><text
class="breeze-sbom-export-dependency-information-r5" x="24.4" y="142"
textLength="451.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-5)"> Export dependency information flags </text><text
class="breeze-sbom-export-dependency-information-r5" x="475.8" y="142"
textLength [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="166.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-6)">│</text><text
class="breeze-sbom-export-dependency-information-r6" x="24.4" y="166.4"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-6)">*</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="166.4"
textLength="122" clip-path="url(#breeze-sbom-export-dependency-informa [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="190.8" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-7)">│</text><text
class="breeze-sbom-export-dependency-information-r6" x="24.4" y="190.8"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-7)">*</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="190.8"
textLength="207.4" clip-path="url(#breeze-sbom-export-dependency-infor [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="215.2" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-8)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="215.2"
textLength="97.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-8)">--python</text><text
class="breeze-sbom-export-dependency-information-r7" x="427" y="215.2"
textLength="24.4" clip-path="url(#breeze-sbom-export-dependency- [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="239.6" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-9)">│</text><text
class="breeze-sbom-export-dependency-information-r8" x="475.8" y="239.6"
textLength="732"
clip-path="url(#breeze-sbom-export-dependency-information-line-9)">(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)            &#
[...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="264" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-10)">│</text><text
class="breeze-sbom-export-dependency-information-r5" x="475.8" y="264"
textLength="732"
clip-path="url(#breeze-sbom-export-dependency-information-line-10)">[default: 3.8]                       
[...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="288.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-11)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="288.4"
textLength="341.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-11)">--include-open-psf-scorecard</text><text
class="breeze-sbom-export-dependency-information-r7" x="427" y="288.4"
textLength="24.4" clip-path="url(#breeze- [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="166.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-6)">│</text><text
class="breeze-sbom-export-dependency-information-r6" x="24.4" y="166.4"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-6)">*</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="166.4"
textLength="207.4" clip-path="url(#breeze-sbom-export-dependency-infor [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="190.8" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-7)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="190.8"
textLength="97.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-7)">--python</text><text
class="breeze-sbom-export-dependency-information-r7" x="427" y="190.8"
textLength="24.4" clip-path="url(#breeze-sbom-export-dependency- [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="215.2" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-8)">│</text><text
class="breeze-sbom-export-dependency-information-r8" x="475.8" y="215.2"
textLength="732"
clip-path="url(#breeze-sbom-export-dependency-information-line-8)">(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)            &#
[...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="239.6" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-9)">│</text><text
class="breeze-sbom-export-dependency-information-r5" x="475.8" y="239.6"
textLength="732"
clip-path="url(#breeze-sbom-export-dependency-information-line-9)">[default: 3.8]                      
[...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="264" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-10)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="264"
textLength="341.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-10)">--include-open-psf-scorecard</text><text
class="breeze-sbom-export-dependency-information-r7" x="427" y="264"
textLength="24.4" clip-path="url(#breeze-sbom-e [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="288.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-11)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="61" y="288.4"
textLength="268.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-11)">--include-github-stats</text><text
class="breeze-sbom-export-dependency-information-r7" x="427" y="288.4"
textLength="24.4" clip-path="url(#breeze-sbom-e [...]
</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="312.8" textLength="1464"
clip-path="url(#breeze-sbom-export-dependency-information-line-12)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="312.8"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-12)">
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="337.2" textLength="24.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-13)">╭─</text><text
class="breeze-sbom-export-dependency-information-r5" x="24.4" y="337.2"
textLength="195.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-13)"> Common options </text><text
class="breeze-sbom-export-dependency-information-r5" x="219.6" y="337.2"
textLength="1220" clip-path="url( [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-14)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="361.6"
textLength="109.8"
clip-path="url(#breeze-sbom-export-dependency-information-line-14)">--dry-run</text><text
class="breeze-sbom-export-dependency-information-r7" x="158.6" y="361.6"
textLength="24.4" clip-path="url(#breeze-sbom-export-dep [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="386" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-15)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="386"
textLength="97.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-15)">--answer</text><text
class="breeze-sbom-export-dependency-information-r7" x="158.6" y="386"
textLength="24.4" clip-path="url(#breeze-sbom-export-dependency- [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="410.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-16)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="410.4"
textLength="73.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-16)">--help</text><text
class="breeze-sbom-export-dependency-information-r7" x="158.6" y="410.4"
textLength="24.4" clip-path="url(#breeze-sbom-export-depende [...]
-</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="434.8" textLength="1464"
clip-path="url(#breeze-sbom-export-dependency-information-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="434.8"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-17)">
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="337.2" textLength="24.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-13)">╭─</text><text
class="breeze-sbom-export-dependency-information-r5" x="24.4" y="337.2"
textLength="134.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-13)"> CSV flags </text><text
class="breeze-sbom-export-dependency-information-r5" x="158.6" y="337.2"
textLength="1281" clip-path="url(#bree [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-14)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="361.6"
textLength="122"
clip-path="url(#breeze-sbom-export-dependency-information-line-14)">--csv-file</text><text
class="breeze-sbom-export-dependency-information-r7" x="170.8" y="361.6"
textLength="24.4" clip-path="url(#breeze-sbom-export-depe [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="386" textLength="1464"
clip-path="url(#breeze-sbom-export-dependency-information-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="386"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-15)">
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="410.4" textLength="24.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-16)">╭─</text><text
class="breeze-sbom-export-dependency-information-r5" x="24.4" y="410.4"
textLength="231.8"
clip-path="url(#breeze-sbom-export-dependency-information-line-16)"> Github auth flags </text><text
class="breeze-sbom-export-dependency-information-r5" x="256.2" y="410.4"
textLength="1183.4" clip- [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-17)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="434.8"
textLength="170.8"
clip-path="url(#breeze-sbom-export-dependency-information-line-17)">--github-token</text><text
class="breeze-sbom-export-dependency-information-r1" x="244" y="434.8"
textLength="500.2" clip-path="url(#breeze-sbom-export [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="459.2" textLength="1464"
clip-path="url(#breeze-sbom-export-dependency-information-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="459.2"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-18)">
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="483.6" textLength="24.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-19)">╭─</text><text
class="breeze-sbom-export-dependency-information-r5" x="24.4" y="483.6"
textLength="317.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-19)"> Google spreadsheet flags </text><text
class="breeze-sbom-export-dependency-information-r5" x="341.6" y="483.6"
textLength="1098" [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="508" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-20)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="508"
textLength="280.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-20)">--json-credentials-file</text><text
class="breeze-sbom-export-dependency-information-r1" x="378.2" y="508"
textLength="915" clip-path="url(#breeze-sbom-exp [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="532.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-21)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="532.4"
textLength="280.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-21)">--google-spreadsheet-id</text><text
class="breeze-sbom-export-dependency-information-r7" x="329.4" y="532.4"
textLength="24.4" clip-path="url(#breeze-s [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="556.8" textLength="1464"
clip-path="url(#breeze-sbom-export-dependency-information-line-22)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="556.8"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-22)">
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="581.2" textLength="24.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-23)">╭─</text><text
class="breeze-sbom-export-dependency-information-r5" x="24.4" y="581.2"
textLength="207.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-23)"> Debugging flags </text><text
class="breeze-sbom-export-dependency-information-r5" x="231.8" y="581.2"
textLength="1207.8" clip-path="u [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="605.6" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-24)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="605.6"
textLength="170.8"
clip-path="url(#breeze-sbom-export-dependency-information-line-24)">--limit-output</text><text
class="breeze-sbom-export-dependency-information-r7" x="219.6" y="605.6"
textLength="24.4" clip-path="url(#breeze-sbom-expor [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="630" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-25)">│</text><text
class="breeze-sbom-export-dependency-information-r1" x="268.4" y="630"
textLength="1171.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-25)">to output all dependencies, do not specify this option.         
[...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="654.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-26)">│</text><text
class="breeze-sbom-export-dependency-information-r8" x="268.4" y="654.4"
textLength="1171.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-26)">(INTEGER)                       &#
[...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="678.8" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-27)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="678.8"
textLength="170.8"
clip-path="url(#breeze-sbom-export-dependency-information-line-27)">--project-name</text><text
class="breeze-sbom-export-dependency-information-r1" x="268.4" y="678.8"
textLength="1012.6" clip-path="url(#breeze-sbom-exp [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="703.2" textLength="1464"
clip-path="url(#breeze-sbom-export-dependency-information-line-28)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="703.2"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-28)">
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="727.6" textLength="24.4"
clip-path="url(#breeze-sbom-export-dependency-information-line-29)">╭─</text><text
class="breeze-sbom-export-dependency-information-r5" x="24.4" y="727.6"
textLength="195.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-29)"> Common options </text><text
class="breeze-sbom-export-dependency-information-r5" x="219.6" y="727.6"
textLength="1220" clip-path="url( [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="752" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-30)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="752"
textLength="109.8"
clip-path="url(#breeze-sbom-export-dependency-information-line-30)">--dry-run</text><text
class="breeze-sbom-export-dependency-information-r7" x="158.6" y="752"
textLength="24.4" clip-path="url(#breeze-sbom-export-dependenc [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="776.4" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-31)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="776.4"
textLength="97.6"
clip-path="url(#breeze-sbom-export-dependency-information-line-31)">--answer</text><text
class="breeze-sbom-export-dependency-information-r7" x="158.6" y="776.4"
textLength="24.4" clip-path="url(#breeze-sbom-export-depen [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="800.8" textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-32)">│</text><text
class="breeze-sbom-export-dependency-information-r4" x="24.4" y="800.8"
textLength="73.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-32)">--help</text><text
class="breeze-sbom-export-dependency-information-r7" x="158.6" y="800.8"
textLength="24.4" clip-path="url(#breeze-sbom-export-depende [...]
+</text><text class="breeze-sbom-export-dependency-information-r5" x="0"
y="825.2" textLength="1464"
clip-path="url(#breeze-sbom-export-dependency-information-line-33)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-sbom-export-dependency-information-r1" x="1464" y="825.2"
textLength="12.2"
clip-path="url(#breeze-sbom-export-dependency-information-line-33)">
</text>
</g>
</g>
diff --git
a/dev/breeze/doc/images/output_sbom_export-dependency-information.txt
b/dev/breeze/doc/images/output_sbom_export-dependency-information.txt
index 30128b0e80..0097771912 100644
--- a/dev/breeze/doc/images/output_sbom_export-dependency-information.txt
+++ b/dev/breeze/doc/images/output_sbom_export-dependency-information.txt
@@ -1 +1 @@
-24dce1db728b2c24782375cd3c18600c
+04ba24ba16920575bec41722d612f0a4
diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml
index 32b3e1fbe6..7aa94df878 100644
--- a/dev/breeze/pyproject.toml
+++ b/dev/breeze/pyproject.toml
@@ -49,6 +49,9 @@ dependencies = [
"click>=8.1.7",
"filelock>=3.13.0",
"flit>=3.5.0",
+ "google-api-python-client>=2.142.0",
+ "google-auth-httplib2>=0.2.0",
+ "google-auth-oauthlib>=1.2.0",
"gitpython>=3.1.40",
"hatch==1.9.4",
# Importib_resources 6.2.0-6.3.1 break pytest_rewrite
diff --git a/dev/breeze/src/airflow_breeze/commands/sbom_commands.py
b/dev/breeze/src/airflow_breeze/commands/sbom_commands.py
index e7ecdb4d63..db17120fda 100644
--- a/dev/breeze/src/airflow_breeze/commands/sbom_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/sbom_commands.py
@@ -22,6 +22,7 @@ import json
import os
import sys
from pathlib import Path
+from typing import Any
import click
@@ -30,6 +31,7 @@ from airflow_breeze.commands.common_options import (
option_answer,
option_debug_resources,
option_dry_run,
+ option_github_token,
option_historical_python_version,
option_include_success_outputs,
option_parallelism,
@@ -45,12 +47,13 @@ from airflow_breeze.global_constants import (
PROVIDER_DEPENDENCIES,
)
from airflow_breeze.utils.cdxgen import (
+ CHECK_DOCS,
PROVIDER_REQUIREMENTS_DIR_PATH,
SbomApplicationJob,
SbomCoreJob,
SbomProviderJob,
build_all_airflow_versions_base_image,
- convert_sbom_to_csv,
+ convert_sbom_entry_to_dict,
get_cdxgen_port_mapping,
get_field_names,
get_requirements_for_provider,
@@ -70,6 +73,7 @@ from airflow_breeze.utils.parallel import (
run_with_pool,
)
from airflow_breeze.utils.path_utils import FILES_SBOM_DIR,
PROVIDER_METADATA_JSON_FILE_PATH
+from airflow_breeze.utils.recording import generating_command_images
from airflow_breeze.utils.shared_options import get_dry_run
@@ -643,24 +647,83 @@ def generate_providers_requirements(
"-f",
"--csv-file",
type=click.Path(file_okay=True, dir_okay=False, path_type=Path,
writable=True),
- help="CSV file to produce.",
+ help="CSV file to produce. Mutually exclusive with Google Spreadsheet Id.",
envvar="CSV_FILE",
- required=True,
+ required=False,
+)
[email protected](
+ "-g",
+ "--google-spreadsheet-id",
+ type=str,
+ help="Google Spreadsheet Id to produce. Mutually exclusive with CSV file.",
+ envvar="GOOGLE_SPREADSHEET_ID",
+ required=False,
+)
+@option_github_token
[email protected](
+ "--json-credentials-file",
+ type=click.Path(file_okay=True, dir_okay=False, path_type=Path,
writable=False),
+ help="Gsheet JSON credentials file (defaults to
~/.config/gsheet/credentials.json",
+ envvar="JSON_CREDENTIALS_FILE",
+ default=Path.home() / ".config" / "gsheet" / "credentials.json"
+ if not generating_command_images()
+ else "credentials.json",
+ required=False,
)
@click.option(
"-s",
"--include-open-psf-scorecard",
+ help="Include statistics from the Open PSF Scorecard",
is_flag=True,
default=False,
)
[email protected](
+ "-G",
+ "--include-github-stats",
+ help="Include statistics from GitHub",
+ is_flag=True,
+ default=False,
+)
[email protected](
+ "-l",
+ "--limit-output",
+ help="Limit the output to the first N dependencies. Default is to output
all dependencies. "
+ "If you want to output all dependencies, do not specify this option.",
+ type=int,
+ required=False,
+)
[email protected](
+ "--project-name",
+ help="Only used for debugging purposes. The name of the project to
generate the sbom for.",
+ type=str,
+ required=False,
+)
@option_dry_run
@option_answer
def export_dependency_information(
python: str,
airflow_version: str,
- csv_file: Path,
+ csv_file: Path | None,
+ google_spreadsheet_id: str | None,
+ github_token: str | None,
+ json_credentials_file: Path,
include_open_psf_scorecard: bool,
+ include_github_stats: bool,
+ limit_output: int | None,
+ project_name: str | None,
):
+ if not google_spreadsheet_id and not csv_file:
+ get_console().print("[error]You need to specify either --csv-file or
--google-spreadsheet-id")
+ sys.exit(1)
+ if google_spreadsheet_id and csv_file:
+ get_console().print("[error]You cannot specify both --csv-file and
--google-spreadsheet-id")
+ sys.exit(1)
+ if google_spreadsheet_id and not json_credentials_file.exists():
+ get_console().print(
+ f"[error]The JSON credentials file {json_credentials_file} does
not exist. "
+ "Please specify a valid path to the JSON credentials file."
+ )
+ sys.exit(1)
import requests
base_url =
f"https://airflow.apache.org/docs/apache-airflow/{airflow_version}/sbom";
@@ -673,43 +736,250 @@ def export_dependency_information(
full_sbom_r = requests.get(sbom_full_url)
full_sbom_r.raise_for_status()
- core_dependencies = set()
-
core_sbom = core_sbom_r.json()
-
full_sbom = full_sbom_r.json()
- dev_deps = set(normalize_package_name(name) for name in
DEVEL_DEPS_PATH.read_text().splitlines())
- num_deps = 0
+ all_dependency_value_dicts = convert_all_sbom_to_value_dictionaries(
+ core_sbom=core_sbom,
+ full_sbom=full_sbom,
+ include_open_psf_scorecard=include_open_psf_scorecard,
+ include_github_stats=include_github_stats,
+ limit_output=limit_output,
+ github_token=github_token,
+ project_name=project_name,
+ )
+ all_dependency_value_dicts = sorted(all_dependency_value_dicts,
key=sort_deps_key)
+
+ fieldnames = get_field_names(include_open_psf_scorecard,
include_github_stats)
+
+ if csv_file:
+ write_to_csv_file(
+ csv_file=csv_file, all_dependencies=all_dependency_value_dicts,
fieldnames=fieldnames
+ )
+ elif google_spreadsheet_id:
+ write_to_google_spreadsheet(
+ google_spreadsheet_id=google_spreadsheet_id,
+ json_credentials_file=json_credentials_file,
+ all_dependencies=all_dependency_value_dicts,
+ fieldnames=fieldnames,
+ )
+
+
+def calculate_range(num_columns: int, row: int) -> str:
+ import string
+
+ # Generate column letters
+ columns = list(string.ascii_uppercase)
+ if num_columns > 26:
+ columns += [f"{a}{b}" for a in string.ascii_uppercase for b in
string.ascii_uppercase]
+
+ # Calculate the range
+ end_column = columns[num_columns - 1]
+ return f"A{row}:{end_column}{row}"
+
+
+def convert_sbom_dict_to_spreadsheet_data(headers: list[str], value_dict:
dict[str, Any]):
+ return [value_dict.get(header, "") for header in headers]
+
+
+INTERESTING_OPSF_FIELDS = [
+ "Score",
+ "Code-Review",
+ "Maintained",
+ "Dangerous-Workflow",
+ "Security-Policy",
+ "Packaging",
+ "Vulnerabilities",
+]
+
+INTERESTING_OPSF_SCORES = ["OPSF-" + field for field in
INTERESTING_OPSF_FIELDS]
+INTERESTING_OPSF_DETAILS = ["OPSF-Details-" + field for field in
INTERESTING_OPSF_FIELDS]
+
+
+def write_to_google_spreadsheet(
+ google_spreadsheet_id: str,
+ json_credentials_file: Path,
+ all_dependencies: list[dict[str, Any]],
+ fieldnames: list[str],
+):
+ token_path = Path.home() / ".config" / "gsheet" / "token.json"
+
+ sheet = authorize_gsheet(json_credentials_file, token_path)
+
+ # Use only interesting values from the scorecard
+ cell_field_names = [
+ fieldname
+ for fieldname in fieldnames
+ if fieldname in INTERESTING_OPSF_SCORES or not
fieldname.startswith("OPSF-")
+ ]
+
+ num_rows = update_field_values(all_dependencies, cell_field_names,
google_spreadsheet_id, sheet)
+ update_opsf_detailed_comments(all_dependencies, fieldnames, num_rows,
google_spreadsheet_id, sheet)
+
+
+def update_opsf_detailed_comments(
+ all_dependencies: list[dict[str, Any]],
+ fieldnames: list[str],
+ num_rows: int,
+ google_spreadsheet_id: str,
+ sheet,
+):
+ opsf_details_field_names = [
+ fieldname for fieldname in fieldnames if fieldname in
INTERESTING_OPSF_DETAILS
+ ]
+ start_opsf_column = fieldnames.index(opsf_details_field_names[0]) - 1
+ opsf_details = []
+ opsf_details.append(
+ {
+ "values": [
+ {"note": CHECK_DOCS[check]}
+ for check in INTERESTING_OPSF_FIELDS
+ if check != INTERESTING_OPSF_FIELDS[0]
+ ]
+ }
+ )
+ for dependency in all_dependencies:
+ note_row =
convert_sbom_dict_to_spreadsheet_data(opsf_details_field_names, dependency)
+ opsf_details.append({"values": [{"note": note} for note in note_row]})
+ notes = {
+ "updateCells": {
+ "range": {
+ "startRowIndex": 1,
+ "endRowIndex": num_rows + 1,
+ "startColumnIndex": start_opsf_column,
+ "endColumnIndex": start_opsf_column +
len(opsf_details_field_names) + 1,
+ },
+ "rows": opsf_details,
+ "fields": "note",
+ },
+ }
+ update_note_body = {"requests": [notes]}
+ sheet.batchUpdate(spreadsheetId=google_spreadsheet_id,
body=update_note_body).execute()
+
+
+def simplify_field_names(fieldname: str):
+ if fieldname.startswith("OPSF-"):
+ return fieldname[5:]
+ return fieldname
+
+
+def update_field_values(
+ all_dependencies: list[dict[str, Any]], cell_field_names: list[str],
google_spreadsheet_id, sheet
+) -> int:
+ num_fields = len(cell_field_names)
+ data = []
+ top_header = []
+ for field in cell_field_names:
+ if field.startswith("OPSF-"):
+ top_header.append("Relevant OPSF Scores and details")
+ break
+ else:
+ top_header.append("")
+
+ simplified_cell_field_names = [simplify_field_names(field) for field in
cell_field_names]
+
+ data.append({"range": calculate_range(num_fields, 1), "values":
[top_header]})
+ data.append({"range": calculate_range(num_fields, 2), "values":
[simplified_cell_field_names]})
+ row = 3
+ for dependency in all_dependencies:
+ spreadsheet_row =
convert_sbom_dict_to_spreadsheet_data(cell_field_names, dependency)
+ data.append({"range": calculate_range(num_fields, row), "values":
[spreadsheet_row]})
+ row += 1
+ body = {"valueInputOption": "RAW", "data": data}
+ result = sheet.values().batchUpdate(spreadsheetId=google_spreadsheet_id,
body=body).execute()
+ get_console().print(f"{result.get('totalUpdatedCells')} cells values set
in the Google spreadsheet.")
+ return row
+
+
+def authorize_gsheet(json_credentials_file: Path, token_path: Path):
+ from google.auth.transport.requests import Request
+ from google.oauth2.credentials import Credentials
+ from google_auth_oauthlib.flow import InstalledAppFlow
+ from googleapiclient.discovery import build
+
+ SCOPES = ["https://www.googleapis.com/auth/spreadsheets";]
+ creds = None
+ if token_path.exists():
+ creds = Credentials.from_authorized_user_file(token_path.as_posix(),
SCOPES)
+ if not creds or not creds.valid:
+ if creds and creds.expired and creds.refresh_token:
+ creds.refresh(Request())
+ else:
+ flow =
InstalledAppFlow.from_client_secrets_file(json_credentials_file.as_posix(),
SCOPES)
+ creds = flow.run_local_server(port=0)
+ # Save the credentials for the next run
+ token_path.write_text(creds.to_json())
+ service = build("sheets", "v4", credentials=creds)
+ sheet = service.spreadsheets()
+ return sheet
+
+
+def write_to_csv_file(csv_file: Path, all_dependencies: list[dict[str, Any]],
fieldnames: list[str]):
with csv_file.open("w") as csvfile:
- fieldnames = get_field_names(include_open_psf_scorecard)
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
- for dependency in core_sbom["components"]:
- name = dependency["name"]
- normalized_name = normalize_package_name(name)
- core_dependencies.add(normalized_name)
+ for dependency_value_dict in all_dependencies:
+ writer.writerow(dependency_value_dict)
+ get_console().print(f"[info]Exported {len(all_dependencies)} dependencies
to {csv_file}")
+
+
+def sort_deps_key(dependency: dict[str, Any]) -> str:
+ if dependency.get("Vcs"):
+ return "0:" + dependency["Name"]
+ else:
+ return "1:" + dependency["Name"]
+
+
+def convert_all_sbom_to_value_dictionaries(
+ core_sbom: dict[str, Any],
+ full_sbom: dict[str, Any],
+ include_open_psf_scorecard: bool,
+ include_github_stats: bool,
+ limit_output: int | None,
+ github_token: str | None = None,
+ project_name: str | None = None,
+) -> list[dict[str, Any]]:
+ core_dependencies = set()
+ dev_deps = set(normalize_package_name(name) for name in
DEVEL_DEPS_PATH.read_text().splitlines())
+ num_deps = 0
+ all_dependency_value_dicts = []
+ for dependency in core_sbom["components"]:
+ normalized_name = normalize_package_name(dependency["name"])
+ if project_name and normalized_name != project_name:
+ continue
+ core_dependencies.add(normalized_name)
+ is_devel = normalized_name in dev_deps
+ value_dict = convert_sbom_entry_to_dict(
+ dependency,
+ is_core=True,
+ is_devel=is_devel,
+ include_open_psf_scorecard=include_open_psf_scorecard,
+ include_github_stats=include_github_stats,
+ github_token=github_token,
+ )
+ if value_dict:
+ all_dependency_value_dicts.append(value_dict)
+ num_deps += 1
+ if limit_output and num_deps >= limit_output:
+ return all_dependency_value_dicts
+ for dependency in full_sbom["components"]:
+ normalized_name = normalize_package_name(dependency["name"])
+ if project_name and normalized_name != project_name:
+ continue
+ if normalized_name not in core_dependencies:
is_devel = normalized_name in dev_deps
- convert_sbom_to_csv(
- writer,
+ value_dict = convert_sbom_entry_to_dict(
dependency,
- is_core=True,
+ is_core=False,
is_devel=is_devel,
include_open_psf_scorecard=include_open_psf_scorecard,
+ include_github_stats=include_github_stats,
+ github_token=github_token,
)
+ if value_dict:
+ all_dependency_value_dicts.append(value_dict)
num_deps += 1
- for dependency in full_sbom["components"]:
- name = dependency["name"]
- normalized_name = normalize_package_name(name)
- if normalized_name not in core_dependencies:
- is_devel = normalized_name in dev_deps
- convert_sbom_to_csv(
- writer,
- dependency,
- is_core=False,
- is_devel=is_devel,
- include_open_psf_scorecard=include_open_psf_scorecard,
- )
- num_deps += 1
-
- get_console().print(f"[info]Exported {num_deps} dependencies to
{csv_file}")
+ if limit_output and num_deps >= limit_output:
+ return all_dependency_value_dicts
+ get_console().print(f"[info]Processed {num_deps} dependencies")
+ return all_dependency_value_dicts
diff --git a/dev/breeze/src/airflow_breeze/commands/sbom_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/sbom_commands_config.py
index d5b26a6a29..bb0936b99b 100644
--- a/dev/breeze/src/airflow_breeze/commands/sbom_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/sbom_commands_config.py
@@ -22,6 +22,7 @@ SBOM_COMMANDS: dict[str, str | list[str]] = {
"update-sbom-information",
"build-all-airflow-images",
"generate-providers-requirements",
+ "export-dependency-information",
],
}
@@ -95,11 +96,37 @@ SBOM_PARAMETERS: dict[str, list[dict[str, str |
list[str]]]] = {
{
"name": "Export dependency information flags",
"options": [
- "--csv-file",
"--airflow-version",
"--python",
"--include-open-psf-scorecard",
+ "--include-github-stats",
+ ],
+ },
+ {
+ "name": "CSV flags",
+ "options": [
+ "--csv-file",
+ ],
+ },
+ {
+ "name": "Github auth flags",
+ "options": [
+ "--github-token",
],
- }
+ },
+ {
+ "name": "Google spreadsheet flags",
+ "options": [
+ "--json-credentials-file",
+ "--google-spreadsheet-id",
+ ],
+ },
+ {
+ "name": "Debugging flags",
+ "options": [
+ "--limit-output",
+ "--project-name",
+ ],
+ },
],
}
diff --git a/dev/breeze/src/airflow_breeze/utils/cdxgen.py
b/dev/breeze/src/airflow_breeze/utils/cdxgen.py
index 3d129b4674..00256f8f10 100644
--- a/dev/breeze/src/airflow_breeze/utils/cdxgen.py
+++ b/dev/breeze/src/airflow_breeze/utils/cdxgen.py
@@ -24,7 +24,6 @@ import signal
import sys
import time
from abc import abstractmethod
-from csv import DictWriter
from dataclasses import dataclass
from multiprocessing.pool import Pool
from pathlib import Path
@@ -538,7 +537,7 @@ def get_vcs(dependency: dict[str, Any]) -> str:
if "externalReferences" in dependency:
for reference in dependency["externalReferences"]:
if reference["type"] == "vcs":
- return reference["url"]
+ return reference["url"].replace("http://";, "https://";)
return ""
@@ -570,8 +569,112 @@ OPEN_PSF_CHECKS = [
"SAST",
]
+CHECK_DOCS: dict[str, str] = {}
-def get_open_psf_scorecard(vcs):
+
+KNOWN_REPUTABLE_FOUNDATIONS = [
+ "apache",
+ "python",
+ "zopefoundation",
+ "uqfoundation",
+ "numpy",
+ "django",
+]
+
+KNOWN_STRONG_COMMUNITIES = ["pallets-eco", "celery", "fsspec", "aio-libs",
"pyasn1", "pytest-dev", "aio-libs"]
+
+KNOWN_COMPANIES = ["getsentry", "prometheus", "Textualize", "google",
"googleapis", "boto"]
+
+KNOWN_STABLE_PROJECTS = [
+ "Deprecated",
+ "Flask-Bcrypt",
+ "SQLAlchemy-Utils",
+ "aiohttp",
+ "aiosignal",
+ "async-timeout",
+ "backoff",
+ "botocore",
+ "cgroupspy",
+ "charset-normalizer",
+ "colorlog",
+ "decorator",
+ "google-re2",
+ "h11",
+ "inflection",
+ "isodate",
+ "jmespath",
+ "lazy-object-proxy",
+ "ldap3",
+ "multidict",
+ "prison",
+ "prometheus_client",
+ "pure-sasl",
+ "rfc3339-validator",
+ "six",
+ "setproctitle",
+ "text-unidecode",
+ "thrift-sasl",
+ "tzdata",
+ "vine",
+]
+
+KNOWN_LOW_IMPORTANCE_PROJECTS = [
+ "flask-appbuilder",
+]
+
+KNOWN_MEDIUM_IMPORTANCE_PROJECTS: list[str] = []
+
+KNOWN_HIGH_IMPORTANCE_PROJECTS: list[str] = []
+
+# Project we have a relationship with the community
+RELATIONSHIP_PROJECT = [
+ "flask-appbuilder",
+ "thrift-sasl",
+ "python-slugify",
+ "plyvel",
+ "pure-sasl",
+ "python-nvd3",
+ "flask-caching",
+ "universal-pathlib",
+]
+
+
+def get_github_stats(vcs: str, project_name: str, github_token: str | None) ->
dict[str, Any]:
+ import requests
+
+ result = {}
+ if vcs and vcs.startswith("https://github.com/";):
+ importance = "Low"
+ api_url = vcs.replace("https://github.com/";,
"https://api.github.com/repos/";)
+ if api_url.endswith("/"):
+ api_url = api_url[:-1]
+ headers = {"Authorization": f"token {github_token}"} if github_token
else {}
+ get_console().print(f"[info]Retrieving Github Stats from {api_url}")
+ response = requests.get(api_url, headers=headers)
+ if response.status_code == 404:
+ get_console().print(f"[warning]Github API returned 404 for
{api_url}")
+ return {}
+ response.raise_for_status()
+ github_data = response.json()
+ stargazer_count = github_data.get("stargazers_count")
+ forks_count = github_data.get("forks_count")
+ if project_name in KNOWN_LOW_IMPORTANCE_PROJECTS:
+ importance = "Low"
+ elif project_name in KNOWN_MEDIUM_IMPORTANCE_PROJECTS:
+ importance = "Medium"
+ elif project_name in KNOWN_HIGH_IMPORTANCE_PROJECTS:
+ importance = "High"
+ elif forks_count > 1000 or stargazer_count > 1000:
+ importance = "High"
+ elif stargazer_count > 100 or forks_count > 100:
+ importance = "Medium"
+ result["Industry importance"] = importance
+ else:
+ get_console().print(f"[warning]Not retrieving Github Stats for {vcs}")
+ return result
+
+
+def get_open_psf_scorecard(vcs: str, project_name: str) -> dict[str, Any]:
import requests
repo_url = vcs.split("://")[1]
@@ -586,34 +689,64 @@ def get_open_psf_scorecard(vcs):
if "checks" in open_psf_scorecard:
for check in open_psf_scorecard["checks"]:
check_name = check["name"]
+ score = check["score"]
results["OPSF-" + check_name] = check["score"]
reason = check.get("reason") or ""
if check.get("details"):
reason += "\n".join(check["details"])
results["OPSF-Details-" + check_name] = reason
+ CHECK_DOCS[check_name] = check["documentation"]["short"] + "\n" +
check["documentation"]["url"]
+ if check_name == "Maintained":
+ if project_name in KNOWN_STABLE_PROJECTS:
+ lifecycle_status = "Stable"
+ else:
+ if score == 0:
+ lifecycle_status = "Abandoned"
+ elif score < 6:
+ lifecycle_status = "Somewhat maintained"
+ else:
+ lifecycle_status = "Actively maintained"
+ results["Lifecycle status"] = lifecycle_status
+ if check_name == "Vulnerabilities":
+ results["Unpatched Vulns"] = "Yes" if score != 10 else ""
return results
-def convert_sbom_to_csv(
- writer: DictWriter,
+def get_governance(vcs: str | None):
+ if not vcs or not vcs.startswith("https://github.com/";):
+ return ""
+ organization = vcs.split("/")[3]
+ if organization.lower() in KNOWN_REPUTABLE_FOUNDATIONS:
+ return "Reputable Foundation"
+ if organization.lower() in KNOWN_STRONG_COMMUNITIES:
+ return "Strong Community"
+ if organization.lower() in KNOWN_COMPANIES:
+ return "Company"
+ return "Loose community/ Single Person"
+
+
+def convert_sbom_entry_to_dict(
dependency: dict[str, Any],
is_core: bool,
is_devel: bool,
- include_open_psf_scorecard: bool = False,
-) -> None:
+ include_open_psf_scorecard: bool,
+ include_github_stats: bool,
+ github_token: str | None,
+) -> dict[str, Any] | None:
"""
- Convert SBOM to CSV
- :param writer: CSV writer
+ Convert SBOM to Row for CSV or spreadsheet output
:param dependency: Dependency to convert
:param is_core: Whether the dependency is core or not
+ :param is_devel: Whether the dependency is devel or not
+ :param include_open_psf_scorecard: Whether to include Open PSF Scorecard
"""
get_console().print(f"[info]Converting {dependency['name']} to CSV")
vcs = get_vcs(dependency)
name = dependency.get("name", "")
if name.startswith("apache-airflow"):
- return
+ return None
row = {
- "Name": dependency.get("name", ""),
+ "Name": normalize_package_name(dependency.get("name", "")),
"Author": dependency.get("author", ""),
"Version": dependency.get("version", ""),
"Description": dependency.get("description"),
@@ -623,20 +756,32 @@ def convert_sbom_to_csv(
"Purl": dependency.get("purl"),
"Pypi": get_pypi_link(dependency),
"Vcs": vcs,
+ "Governance": get_governance(vcs),
}
if vcs and include_open_psf_scorecard:
- open_psf_scorecard = get_open_psf_scorecard(vcs)
+ open_psf_scorecard = get_open_psf_scorecard(vcs, name)
row.update(open_psf_scorecard)
- writer.writerow(row)
+ if vcs and include_github_stats:
+ github_stats = get_github_stats(vcs=vcs, project_name=name,
github_token=github_token)
+ row.update(github_stats)
+ if name in RELATIONSHIP_PROJECT:
+ row["Relationship"] = "Yes"
+ return row
-def get_field_names(include_open_psf_scorecard: bool) -> list[str]:
+def get_field_names(include_open_psf_scorecard: bool, include_github_stats:
bool) -> list[str]:
names = ["Name", "Author", "Version", "Description", "Core", "Devel",
"Licenses", "Purl", "Pypi", "Vcs"]
if include_open_psf_scorecard:
names.append("OPSF-Score")
for check in OPEN_PSF_CHECKS:
names.append("OPSF-" + check)
names.append("OPSF-Details-" + check)
+ names.append("Governance")
+ if include_open_psf_scorecard:
+ names.extend(["Lifecycle status", "Unpatched Vulns"])
+ if include_github_stats:
+ names.append("Industry importance")
+ names.append("Relationship")
return names
