Le mar. 26 nov. 2024 à 14:02, Vincent Prince via lists.openembedded.org < [email protected]> a écrit :
> Hello, > > I'm wondering if I'm the only one who couldn't complete do_fetch of > cve-update-nvd2-native with maximum retries on CVE API? > I tried with or without API Key, from several different machines, I > also tried to increase retries number etc, without success. > I did a successful full download with a crazy retry number (100). It took a long time but it finished eventually. You can monitor the progress with bitbake cve-update-nvd2-native -c fetch -v > I see on the NVD status page the following warning : > "Due to changes described below, a large number of CVE records have > recently been updated, resulting in a large increase in API requests. > We are aware of the issue and are working to mitigate it. Thank you > for your patience." > Thanks for the update, I did not think of checking the website -_-' > I see the Buildroot switched from NVD to alternative FKIE github > database due to that API 2.0 inconsistency :/ > https://github.com/fkie-cad/nvd-json-data-feeds If the situation at NVD does not change we might have to also switch I'm not sure what are my options here, quite odd that cybersecurity > organisation get impacted by DDoS :D > > Best regards, > Vincent > > Le jeu. 7 nov. 2024 à 04:38, Steve Sakoman via lists.openembedded.org > <[email protected]> a écrit : > > > > From: Richard Purdie <[email protected]> > > > > Rtaher than trying to use a sqlite database over NFS from DL_DIR, work > from > > a local copy in STAGING DIR after fetching. > > > > Signed-off-by: Richard Purdie <[email protected]> > > (cherry picked from commit 03596904392d257572a905a182b92c780d636744) > > Signed-off-by: Steve Sakoman <[email protected]> > > --- > > meta/classes/cve-check.bbclass | 7 ++++--- > > .../meta/cve-update-nvd2-native.bb | 18 +++++++++++++----- > > 2 files changed, 17 insertions(+), 8 deletions(-) > > > > diff --git a/meta/classes/cve-check.bbclass > b/meta/classes/cve-check.bbclass > > index d287cf1457..ed219bf472 100644 > > --- a/meta/classes/cve-check.bbclass > > +++ b/meta/classes/cve-check.bbclass > > @@ -31,8 +31,9 @@ > > CVE_PRODUCT ??= "${BPN}" > > CVE_VERSION ??= "${PV}" > > > > -CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" > > -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2-2.db" > > +CVE_CHECK_DB_FILENAME ?= "nvdcve_2-2.db" > > +CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" > > +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" > > CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" > > > > CVE_CHECK_LOG ?= "${T}/cve.log" > > @@ -198,7 +199,7 @@ python do_cve_check () { > > } > > > > addtask cve_check before do_build > > -do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" > > +do_cve_check[depends] = "cve-update-nvd2-native:do_unpack" > > do_cve_check[nostamp] = "1" > > > > python cve_check_cleanup () { > > diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb > b/meta/recipes-core/meta/cve-update-nvd2-native.bb > > index 92177712dc..5fbe9095cc 100644 > > --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb > > +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb > > @@ -8,7 +8,6 @@ INHIBIT_DEFAULT_DEPS = "1" > > > > inherit native > > > > -deltask do_unpack > > deltask do_patch > > deltask do_configure > > deltask do_compile > > @@ -35,7 +34,9 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" > > # Number of attempts for each http query to nvd server before giving up > > CVE_DB_UPDATE_ATTEMPTS ?= "5" > > > > -CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" > > +CVE_CHECK_DB_DLDIR_FILE ?= > "${DL_DIR}/CVE_CHECK/${CVE_CHECK_DB_FILENAME}" > > +CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock" > > +CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp" > > > > python () { > > if not bb.data.inherits_class("cve-check", d): > > @@ -52,9 +53,9 @@ python do_fetch() { > > > > bb.utils.export_proxies(d) > > > > - db_file = d.getVar("CVE_CHECK_DB_FILE") > > + db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE") > > db_dir = os.path.dirname(db_file) > > - db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") > > + db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE") > > > > cleanup_db_download(db_file, db_tmp_file) > > # By default let's update the whole database (since time 0) > > @@ -77,6 +78,7 @@ python do_fetch() { > > pass > > > > bb.utils.mkdirhier(db_dir) > > + bb.utils.mkdirhier(os.path.dirname(db_tmp_file)) > > if os.path.exists(db_file): > > shutil.copy2(db_file, db_tmp_file) > > > > @@ -89,10 +91,16 @@ python do_fetch() { > > os.remove(db_tmp_file) > > } > > > > -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" > > +do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}" > > do_fetch[file-checksums] = "" > > do_fetch[vardeps] = "" > > > > +python do_unpack() { > > + import shutil > > + shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), > d.getVar("CVE_CHECK_DB_FILE")) > > +} > > +do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} > ${CVE_CHECK_DB_FILE_LOCK}" > > + > > def cleanup_db_download(db_file, db_tmp_file): > > """ > > Cleanup the download space from possible failed downloads > > -- > > 2.34.1 > > > > > > > > > > > > -- Yoann Congal Smile ECS - Tech expert
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#207852): https://lists.openembedded.org/g/openembedded-core/message/207852 Mute This Topic: https://lists.openembedded.org/mt/109438548/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
