Hello, Thank you for posting this. After having a look at the current NVD situation, I think that the removal of this file is premature, because of the adding of the 2.0 JSON feeds https://nvd.nist.gov/vuln/data-feeds#divJson20Feeds
So for people who want to use the NVD data directly from the source still, I think it makes sense to implement the download of the JSON 2.0 feeds (should be closer to the nvd-native than nvd2-native, and *faster*) and then decommission both nvd1 and the current nvd2 (API based). Kind regards, Marta On Tue, Nov 4, 2025 at 9:38 AM Niko Mauno <[email protected]> wrote: > > > On 11/3/25 17:26, Marko, Peter wrote: > > This should be commented on by Marta, I'm not sure if she brought this > back even though it was not working anymore. > > In case it's accepted, contribution to yocto-docs should follow. > > > > Peter > > > > Thanks for pointing this out Peter, I'll make a note to submit also > relevant changes to yocto-docs as needed. > -Niko > > >> -----Original Message----- > >> From: Niko Mauno <[email protected]> > >> Sent: Monday, November 3, 2025 15:32 > >> To: [email protected] > >> Cc: [email protected]; [email protected]; Marko, Peter (FT D EU SK > >> BFS1) <[email protected]>; Niko Mauno <[email protected]> > >> Subject: [PATCH 3/5] cve-update: Drop obsolete NVD1 support > >> > >> Since enabling NVD1 as NVD_DB_VERSION nowadays leads to BitBake failure > >> > >> WARNING: cve-update-db-native-1.0-r0 do_fetch: Failed to fetch CVE > data (HTTP > >> Error 403: Forbidden) > >> WARNING: cve-update-db-native-1.0-r0 do_fetch: Host IPs are > 172.65.90.26, > >> 172.65.90.25, 172.65.90.24, 172.65.90.27, 2606:4700:78::90:0:180, > >> 2606:4700:78::90:0:183, 2606:4700:78::90:0:181, 2606:4700:78::90:0:182 > >> WARNING: cve-update-db-native-1.0-r0 do_fetch: CVE database update > failed > >> ERROR: cve-update-db-native-1.0-r0 do_unpack: Error executing a > python > >> function in exec_func_python() autogenerated: > >> > >> Remove the support for obsolete NVD1. > >> > >> Signed-off-by: Niko Mauno <[email protected]> > >> --- > >> meta/classes/cve-check.bbclass | 8 +- > >> .../recipes-core/meta/cve-update-db-native.bb | 87 > +++---------------- > >> 2 files changed, 15 insertions(+), 80 deletions(-) > >> > >> diff --git a/meta/classes/cve-check.bbclass > b/meta/classes/cve-check.bbclass > >> index c63ebd56e1..259c699af2 100644 > >> --- a/meta/classes/cve-check.bbclass > >> +++ b/meta/classes/cve-check.bbclass > >> @@ -31,11 +31,11 @@ > >> CVE_PRODUCT ??= "${BPN}" > >> CVE_VERSION ??= "${PV}" > >> > >> -# Possible database sources: NVD1, NVD2, FKIE > >> +# Possible database sources: NVD2, FKIE > >> NVD_DB_VERSION ?= "FKIE" > >> > >> # Use different file names for each database source, as they > synchronize at > >> different moments, so may be slightly different > >> -CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if > >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdcve_1-3.db' if > >> d.getVar('NVD_DB_VERSION') == 'NVD1' else 'nvdfkie_1-1.db'}" > >> +CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if > >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-1.db'}" > >> CVE_CHECK_DB_FETCHER ?= "${@'cve-update-nvd2-native' if > >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'cve-update-db-native'}" > >> CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" > >> CVE_CHECK_DB_FILE ?= > >> "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" > >> @@ -108,8 +108,8 @@ python () { > >> extend_cve_status(d) > >> > >> nvd_database_type = d.getVar("NVD_DB_VERSION") > >> - if nvd_database_type not in ("NVD1", "NVD2", "FKIE"): > >> - bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD1, > >> NVD2, FKIE. Defaulting to NVD2") > >> + if nvd_database_type not in ("NVD2", "FKIE"): > >> + bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD2, > >> FKIE. Defaulting to NVD2") > >> d.setVar("NVD_DB_VERSION", "NVD2") > >> } > >> > >> diff --git a/meta/recipes-core/meta/cve-update-db-native.bb > b/meta/recipes- > >> core/meta/cve-update-db-native.bb > >> index 3a6dc95580..4423216be5 100644 > >> --- a/meta/recipes-core/meta/cve-update-db-native.bb > >> +++ b/meta/recipes-core/meta/cve-update-db-native.bb > >> @@ -11,7 +11,6 @@ deltask do_compile > >> deltask do_install > >> deltask do_populate_sysroot > >> > >> -NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-"; > >> FKIE_URL ?= "https://github.com/fkie-cad/nvd-json-data- > >> feeds/releases/latest/download/CVE-" > >> > >> # CVE database update interval, in seconds. By default: once a day > (23*60*60). > >> @@ -108,30 +107,12 @@ def cleanup_db_download(db_tmp_file): > >> if os.path.exists(db_tmp_file): > >> os.remove(db_tmp_file) > >> > >> -def db_file_names(d, year, is_nvd): > >> - if is_nvd: > >> - year_url = d.getVar('NVDCVE_URL') + str(year) > >> - meta_url = year_url + ".meta" > >> - json_url = year_url + ".json.gz" > >> - return json_url, meta_url > >> +def db_file_names(d, year): > >> year_url = d.getVar('FKIE_URL') + str(year) > >> meta_url = year_url + ".meta" > >> json_url = year_url + ".json.xz" > >> return json_url, meta_url > >> > >> -def host_db_name(d, is_nvd): > >> - if is_nvd: > >> - return "nvd.nist.gov" > >> - return "github.com" > >> - > >> -def db_decompress(d, data, is_nvd): > >> - import gzip, lzma > >> - > >> - if is_nvd: > >> - return gzip.decompress(data).decode('utf-8') > >> - # otherwise > >> - return lzma.decompress(data) > >> - > >> def update_db_file(db_tmp_file, d): > >> """ > >> Update the given database file > >> @@ -139,12 +120,12 @@ def update_db_file(db_tmp_file, d): > >> import bb.progress > >> import bb.utils > >> from datetime import date > >> + import lzma > >> import sqlite3 > >> import urllib > >> > >> YEAR_START = 2002 > >> cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) > >> - is_nvd = d.getVar("NVD_DB_VERSION") == "NVD1" > >> > >> # Connect to database > >> conn = sqlite3.connect(db_tmp_file) > >> @@ -155,7 +136,7 @@ def update_db_file(db_tmp_file, d): > >> for i, year in enumerate(range(YEAR_START, date.today().year > + 1)): > >> bb.note("Updating %d" % year) > >> ph.update((float(i + 1) / total_years) * 100) > >> - json_url, meta_url = db_file_names(d, year, is_nvd) > >> + json_url, meta_url = db_file_names(d, year) > >> > >> # Retrieve meta last modified date > >> try: > >> @@ -164,7 +145,7 @@ def update_db_file(db_tmp_file, d): > >> cve_f.write('Warning: CVE db update error, Unable to > fetch CVE > >> data.\n\n') > >> bb.warn("Failed to fetch CVE data (%s)" % e) > >> import socket > >> - result = socket.getaddrinfo(host_db_name(d, is_nvd), > 443, > >> proto=socket.IPPROTO_TCP) > >> + result = socket.getaddrinfo("github.com", 443, > >> proto=socket.IPPROTO_TCP) > >> bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t > in result))) > >> return False > >> > >> @@ -192,7 +173,7 @@ def update_db_file(db_tmp_file, d): > >> try: > >> response = urllib.request.urlopen(json_url, > >> timeout=cve_socket_timeout) > >> if response: > >> - update_db(d, conn, db_decompress(d, > response.read(), is_nvd)) > >> + update_db(conn, > lzma.decompress(response.read())) > >> conn.execute("insert or replace into META values > (?, ?)", [year, > >> last_modified]).close() > >> except urllib.error.URLError as e: > >> cve_f.write('Warning: CVE db update error, CVE > data is > >> outdated.\n\n') > >> @@ -224,17 +205,14 @@ def initialize_db(conn): > >> > >> c.close() > >> > >> -def parse_node_and_insert(conn, node, cveId, is_nvd): > >> +def parse_node_and_insert(conn, node, cveId): > >> # Parse children node if needed > >> for child in node.get('children', ()): > >> - parse_node_and_insert(conn, child, cveId, is_nvd) > >> + parse_node_and_insert(conn, child, cveId) > >> > >> - def cpe_generator(is_nvd): > >> + def cpe_generator(): > >> match_string = "cpeMatch" > >> cpe_string = 'criteria' > >> - if is_nvd: > >> - match_string = "cpe_match" > >> - cpe_string = 'cpe23Uri' > >> > >> for cpe in node.get(match_string, ()): > >> if not cpe['vulnerable']: > >> @@ -290,44 +268,7 @@ def parse_node_and_insert(conn, node, cveId, > is_nvd): > >> # Save processing by representing as -. > >> yield [cveId, vendor, product, '-', '', '', ''] > >> > >> - conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, > ?)", > >> cpe_generator(is_nvd)).close() > >> - > >> -def update_db_nvdjson(conn, jsondata): > >> - import json > >> - root = json.loads(jsondata) > >> - > >> - for elt in root['CVE_Items']: > >> - if not elt['impact']: > >> - continue > >> - > >> - accessVector = None > >> - vectorString = None > >> - cvssv2 = 0.0 > >> - cvssv3 = 0.0 > >> - cvssv4 = 0.0 > >> - cveId = elt['cve']['CVE_data_meta']['ID'] > >> - cveDesc = > elt['cve']['description']['description_data'][0]['value'] > >> - date = elt['lastModifiedDate'] > >> - try: > >> - accessVector = > elt['impact']['baseMetricV2']['cvssV2']['accessVector'] > >> - vectorString = > elt['impact']['baseMetricV2']['cvssV2']['vectorString'] > >> - cvssv2 = > elt['impact']['baseMetricV2']['cvssV2']['baseScore'] > >> - except KeyError: > >> - cvssv2 = 0.0 > >> - try: > >> - accessVector = accessVector or > >> elt['impact']['baseMetricV3']['cvssV3']['attackVector'] > >> - vectorString = vectorString or > >> elt['impact']['baseMetricV3']['cvssV3']['vectorString'] > >> - cvssv3 = > elt['impact']['baseMetricV3']['cvssV3']['baseScore'] > >> - except KeyError: > >> - accessVector = accessVector or "UNKNOWN" > >> - cvssv3 = 0.0 > >> - > >> - conn.execute("insert or replace into NVD values (?, ?, ?, ?, > ?, ?, ?, ?)", > >> - [cveId, cveDesc, cvssv2, cvssv3, cvssv4, date, > accessVector, > >> vectorString]).close() > >> - > >> - configurations = elt['configurations']['nodes'] > >> - for config in configurations: > >> - parse_node_and_insert(conn, config, cveId, True) > >> + conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, > ?)", > >> cpe_generator()).close() > >> > >> def get_metric_entry(metric): > >> primaries = [c for c in metric if c['type'] == "Primary"] > >> @@ -338,7 +279,7 @@ def get_metric_entry(metric): > >> return secondaries[0] > >> return None > >> > >> -def update_db_fkie(conn, jsondata): > >> +def update_db(conn, jsondata): > >> import json > >> root = json.loads(jsondata) > >> > >> @@ -403,13 +344,7 @@ def update_db_fkie(conn, jsondata): > >> for config in elt['configurations']: > >> # This is suboptimal as it doesn't handle AND/OR and > negate, but is > >> better than nothing > >> for node in config.get("nodes") or []: > >> - parse_node_and_insert(conn, node, cveId, False) > >> - > >> -def update_db(d, conn, jsondata): > >> - if (d.getVar("NVD_DB_VERSION") == "FKIE"): > >> - return update_db_fkie(conn, jsondata) > >> - else: > >> - return update_db_nvdjson(conn, jsondata) > >> + parse_node_and_insert(conn, node, cveId) > >> > >> do_fetch[nostamp] = "1" > >> > >> -- > >> 2.47.3 > > > >
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#226122): https://lists.openembedded.org/g/openembedded-core/message/226122 Mute This Topic: https://lists.openembedded.org/mt/116098345/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
