It is seen that during bulk updates on the NVD side the server struggles to keep up with the default/max of 2000 entries per page and we see a lot of incomplete read errors resulting in proper db sync failures most of the times. Lowering the per page value noticably increases the reliability of the process and hence should ideally be configurable.
Signed-off-by: Awais B <[email protected]> --- meta/recipes-core/meta/cve-update-nvd2-native.bb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb b/meta/recipes-core/meta/cve-update-nvd2-native.bb index 945bd1d927..731cbb5d88 100644 --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb @@ -34,6 +34,10 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" # Number of attempts for each http query to nvd server before giving up CVE_DB_UPDATE_ATTEMPTS ?= "5" +# Maximum number of CVE records per API response. +# Lowering this value can help avoid incomplete read errors during bulk NVD updates. +CVE_DB_RESULTS_PER_PAGE ?= "" + CVE_CHECK_DB_DLDIR_FILE ?= "${DL_DIR}/CVE_CHECK/${CVE_CHECK_DB_FILENAME}" CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock" CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp" @@ -217,6 +221,15 @@ def update_db_file(db_tmp_file, d, database_time): api_key = d.getVar("NVDCVE_API_KEY") or None attempts = int(d.getVar("CVE_DB_UPDATE_ATTEMPTS")) + results_per_page = d.getVar("CVE_DB_RESULTS_PER_PAGE") + RESULTS_PER_PAGE_MAX = 2000 # imposed by NVD + if results_per_page: + results_per_page = int(results_per_page) + if results_per_page > RESULTS_PER_PAGE_MAX: + bb.warn("CVE_DB_RESULTS_PER_PAGE exceeds maximum of %d, capping" % RESULTS_PER_PAGE_MAX) + results_per_page = RESULTS_PER_PAGE_MAX + req_args['resultsPerPage'] = results_per_page + # Recommended by NVD wait_time = 6 if api_key: -- 2.34.1
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#239562): https://lists.openembedded.org/g/openembedded-core/message/239562 Mute This Topic: https://lists.openembedded.org/mt/119973225/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
