Re: [OE-core] [PATCH 2/2] cve-update-db-native: Remove hash column from database.
Hello Ross,
> Can you rebase this on top of the patches I sent yesterday to change
> the path construction to use os.path.join() please.
I can't find the patches your are referring to. My patches are rebased
on the last master, and I don't see a patch from you in master-next.
Pierre
Le jeu. 18 juil. 2019 à 15:10, Burton, Ross a écrit :
>
>
> Ross
>
> On Thu, 18 Jul 2019 at 13:41, Pierre Le Magourou wrote:
> >
> > From: Pierre Le Magourou
> >
> > djb2 hash algorithm was found to do collisions, so the database was
> > sometime missing data. Remove this hash mechanism, clear and populate
> > elements from scratch in PRODUCTS table if the current year needs an
> > update.
> >
> > Signed-off-by: Pierre Le Magourou
> > ---
> > meta/classes/cve-check.bbclass | 12 ++--
> > meta/recipes-core/meta/cve-update-db-native.bb | 21 +++--
> > 2 files changed, 13 insertions(+), 20 deletions(-)
> >
> > diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
> > index 512d4c7302..c00d2910be 100644
> > --- a/meta/classes/cve-check.bbclass
> > +++ b/meta/classes/cve-check.bbclass
> > @@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}"
> > CVE_VERSION ??= "${PV}"
> >
> > CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
> > -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve.db"
> > +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.0.db"
> >
> > CVE_CHECK_LOG ?= "${T}/cve.log"
> > CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
> > @@ -200,11 +200,11 @@ def check_cves(d, patched_cves):
> > c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ?",
> > (product,))
> >
> > for row in c:
> > -cve = row[1]
> > -version_start = row[4]
> > -operator_start = row[5]
> > -version_end = row[6]
> > -operator_end = row[7]
> > +cve = row[0]
> > +version_start = row[3]
> > +operator_start = row[4]
> > +version_end = row[5]
> > +operator_end = row[6]
> >
> > if cve in cve_whitelist:
> > bb.note("%s-%s has been whitelisted for %s" % (product,
> > pv, cve))
> > diff --git a/meta/recipes-core/meta/cve-update-db-native.bb
> > b/meta/recipes-core/meta/cve-update-db-native.bb
> > index 72d1f48835..3519beae5f 100644
> > --- a/meta/recipes-core/meta/cve-update-db-native.bb
> > +++ b/meta/recipes-core/meta/cve-update-db-native.bb
> > @@ -30,7 +30,7 @@ python do_populate_cve_db() {
> > YEAR_START = 2002
> >
> > db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
> > -db_file = db_dir + '/nvdcve.db'
> > +db_file = db_dir + '/nvdcve_1.0.db'
> > json_tmpfile = db_dir + '/nvd.json.gz'
> > proxy = d.getVar("https_proxy")
> > cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
> > @@ -65,6 +65,10 @@ python do_populate_cve_db() {
> > c.execute("select DATE from META where YEAR = ?", (year,))
> > meta = c.fetchone()
> > if not meta or meta[0] != last_modified:
> > +# Clear products table entries corresponding to current year
> > +cve_year = 'CVE-' + str(year) + '%'
> > +c.execute("delete from PRODUCTS where ID like ?", (cve_year,))
> > +
> > # Update db with current year json file
> > req = urllib.request.Request(json_url)
> > if proxy:
> > @@ -91,27 +95,16 @@ python do_populate_cve_db() {
> > conn.close()
> > }
> >
> > -# DJB2 hash algorithm
> > -def hash_djb2(s):
> > -hash = 5381
> > -for x in s:
> > -hash = (( hash << 5) + hash) + ord(x)
> > -
> > -return hash & 0x
> > -
> > def initialize_db(c):
> > c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE
> > TEXT)")
> > c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY
> > TEXT, \
> > SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
> > -c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE,
> > ID TEXT, \
> > +c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
> > VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START
> > TEXT, \
> > VERSION_END T
[OE-core] [PATCH 1/2] cve-check: Replace CVE_CHECK_CVE_WHITELIST by CVE_CHECK_WHITELIST
From: Pierre Le Magourou
CVE_CHECK_WHITELIST does not contain version anymore, as it was not
used. This variable should be set per recipe.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 22 +++---
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index e8668b2566..512d4c7302 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -39,15 +39,12 @@ CVE_CHECK_CREATE_MANIFEST ??= "1"
# Whitelist for packages (PN)
CVE_CHECK_PN_WHITELIST ?= ""
-# Whitelist for CVE and version of package. If a CVE is found then the PV is
-# compared with the version list, and if found the CVE is considered
-# patched.
-#
-# The value should be valid Python in this format:
-# {
-# 'CVE-2014-2524': ('6.3','5.2')
-# }
-CVE_CHECK_CVE_WHITELIST ?= "{}"
+# Whitelist for CVE. If a CVE is found, then it is considered patched.
+# The value is a string containing space separated CVE values:
+#
+# CVE_CHECK_WHITELIST = 'CVE-2014-2524 CVE-2018-1234'
+#
+CVE_CHECK_WHITELIST ?= ""
python do_cve_check () {
"""
@@ -185,7 +182,10 @@ def check_cves(d, patched_cves):
bb.note("Recipe has been whitelisted, skipping check")
return ([], [])
-cve_whitelist = ast.literal_eval(d.getVar("CVE_CHECK_CVE_WHITELIST"))
+old_cve_whitelist = d.getVar("CVE_CHECK_CVE_WHITELIST")
+if old_cve_whitelist:
+bb.warn("CVE_CHECK_CVE_WHITELIST is deprecated, please use
CVE_CHECK_WHITELIST.")
+cve_whitelist = d.getVar("CVE_CHECK_WHITELIST").split()
import sqlite3
db_file = d.getVar("CVE_CHECK_DB_FILE")
@@ -206,7 +206,7 @@ def check_cves(d, patched_cves):
version_end = row[6]
operator_end = row[7]
-if pv in cve_whitelist.get(cve, []):
+if cve in cve_whitelist:
bb.note("%s-%s has been whitelisted for %s" % (product, pv,
cve))
elif cve in patched_cves:
bb.note("%s has been patched" % (cve))
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/2] cve-update-db-native: Remove hash column from database.
From: Pierre Le Magourou
djb2 hash algorithm was found to do collisions, so the database was
sometime missing data. Remove this hash mechanism, clear and populate
elements from scratch in PRODUCTS table if the current year needs an
update.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 12 ++--
meta/recipes-core/meta/cve-update-db-native.bb | 21 +++--
2 files changed, 13 insertions(+), 20 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 512d4c7302..c00d2910be 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}"
CVE_VERSION ??= "${PV}"
CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
-CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve.db"
+CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.0.db"
CVE_CHECK_LOG ?= "${T}/cve.log"
CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
@@ -200,11 +200,11 @@ def check_cves(d, patched_cves):
c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ?", (product,))
for row in c:
-cve = row[1]
-version_start = row[4]
-operator_start = row[5]
-version_end = row[6]
-operator_end = row[7]
+cve = row[0]
+version_start = row[3]
+operator_start = row[4]
+version_end = row[5]
+operator_end = row[6]
if cve in cve_whitelist:
bb.note("%s-%s has been whitelisted for %s" % (product, pv,
cve))
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb
b/meta/recipes-core/meta/cve-update-db-native.bb
index 72d1f48835..3519beae5f 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -30,7 +30,7 @@ python do_populate_cve_db() {
YEAR_START = 2002
db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
-db_file = db_dir + '/nvdcve.db'
+db_file = db_dir + '/nvdcve_1.0.db'
json_tmpfile = db_dir + '/nvd.json.gz'
proxy = d.getVar("https_proxy")
cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
@@ -65,6 +65,10 @@ python do_populate_cve_db() {
c.execute("select DATE from META where YEAR = ?", (year,))
meta = c.fetchone()
if not meta or meta[0] != last_modified:
+# Clear products table entries corresponding to current year
+cve_year = 'CVE-' + str(year) + '%'
+c.execute("delete from PRODUCTS where ID like ?", (cve_year,))
+
# Update db with current year json file
req = urllib.request.Request(json_url)
if proxy:
@@ -91,27 +95,16 @@ python do_populate_cve_db() {
conn.close()
}
-# DJB2 hash algorithm
-def hash_djb2(s):
-hash = 5381
-for x in s:
-hash = (( hash << 5) + hash) + ord(x)
-
-return hash & 0x
-
def initialize_db(c):
c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE
TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
-c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID
TEXT, \
+c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
VERSION_END TEXT, OPERATOR_END TEXT)")
def insert_elt(c, db_values):
-product_str = db_values[0] + db_values[1] + db_values[2] + db_values[3]
-hashstr = hash_djb2(product_str)
-db_values.insert(0, hashstr)
-query = "insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?, ?, ?)"
+query = "insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)"
c.execute(query, db_values)
def parse_node_and_insert(c, node, cveId):
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/4] cve-update-db: New recipe to update CVE database
Hi Kevin, > I found that the hash function is causing collisions in the generated > database such that some CVEs are being overwritten because of the UNIQUE > constraint on the HASH column. For example, CVE-2018-1000873 has the same > hash of 623198722 as CVE-2018-18338. This results in one of the two CVEs not > appearing in the database. This is problematic. I kept using djb2 hash function, because it was the one used in the previous cve-check-tool and it was fast. But it might not be the right hash function to use. Do you have a better hash function in mind ? I can also drop hash function, remove everything from the database and recreate all entries at each update but it will increase database update time. I don't have the same hash as you for CVE-2018-1000873 and CVE-2018-18338, do you use my latest patches from master ? I did several changes recently. Pierre Le Magourou -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] cve-update-db: Skip recipe when cve-check class is not loaded.
> > Great, the "bitbake universe --runall=fetch" works now. > I just forgot to rebase this patch on the previous ones that are in master-next, I sent a rebased v2. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v2] cve-update-db-native: Skip recipe when cve-check class is not loaded.
From: Pierre Le Magourou
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db-native.bb | 5 +
1 file changed, 5 insertions(+)
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb
b/meta/recipes-core/meta/cve-update-db-native.bb
index d658c7bfea..e16c41a72f 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -13,6 +13,11 @@ deltask do_compile
deltask do_install
deltask do_populate_sysroot
+python () {
+if not d.getVar("CVE_CHECK_DB_FILE"):
+raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not
loaded.")
+}
+
python do_populate_cve_db() {
"""
Update NVD database with json data feed
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [meta-oe][PATCH v5] cve-update-db: do_populate_cve_db depends on do_fetch
Hello, > Did you try > > $ bitbake universe --runall=fetch > > Please? It still doesn't work on latest master branch. > I sent a patch to skip cve-update-db recipe if the cve-check class is not loaded. This should fix the problem. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] cve-update-db: Skip recipe when cve-check class is not loaded.
From: Pierre Le Magourou
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 5 +
1 file changed, 5 insertions(+)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index ae8f1a958b..cfeee91c2e 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -13,6 +13,11 @@ deltask do_compile
deltask do_install
deltask do_populate_sysroot
+python () {
+if not d.getVar("CVE_CHECK_DB_FILE"):
+raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not
loaded.")
+}
+
python do_populate_cve_db() {
"""
Update NVD database with json data feed
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v2 2/3] cve-update-db: Use NVD CPE data to populate PRODUCTS table
From: Pierre Le Magourou
Instead of using expanded list of affected versions that is not
reliable, use the 'cpe_match' node in the 'configurations' json node.
For cve-check to correctly match affected CVE, the sqlite database need to
contain operator_start, operator_end and the corresponding versions fields.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db-native.bb | 88 ++
1 file changed, 74 insertions(+), 14 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb
b/meta/recipes-core/meta/cve-update-db-native.bb
index ae8f1a958b..d658c7bfea 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -25,7 +25,7 @@ python do_populate_cve_db() {
YEAR_START = 2002
db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
-db_file = db_dir + '/nvd-json.db'
+db_file = db_dir + '/nvdcve.db'
json_tmpfile = db_dir + '/nvd.json.gz'
proxy = d.getVar("https_proxy")
cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
@@ -99,9 +99,76 @@ def initialize_db(c):
c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID
TEXT, \
-VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
-c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
-(PRODUCT, VERSION)")
+VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
+VERSION_END TEXT, OPERATOR_END TEXT)")
+
+def insert_elt(c, db_values):
+product_str = db_values[0] + db_values[1] + db_values[2] + db_values[3]
+hashstr = hash_djb2(product_str)
+db_values.insert(0, hashstr)
+query = "insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?, ?, ?)"
+c.execute(query, db_values)
+
+def parse_node_and_insert(c, node, cveId):
+# Parse children node if needed
+try:
+for child in node['children']:
+parse_node_and_insert(c, child, cveId)
+except:
+pass
+
+# Exit if the cpe_match node does not exists
+try:
+cpe_match = node['cpe_match']
+except:
+return
+
+for cpe in cpe_match:
+if not cpe['vulnerable']:
+return
+cpe23 = cpe['cpe23Uri'].split(':')
+vendor = cpe23[3]
+product = cpe23[4]
+version = cpe23[5]
+
+if version != '*':
+# Version is defined, this is a '=' match
+db_values = [cveId, vendor, product, version, '=', '', '']
+insert_elt(c, db_values)
+else:
+# Parse start version, end version and operators
+op_start = ''
+op_end = ''
+v_start = ''
+v_end = ''
+
+try:
+if cpe['versionStartIncluding']:
+op_start = '>='
+v_start = cpe['versionStartIncluding']
+except:
+pass
+try:
+if cpe['versionStartExcluding']:
+op_start = '>'
+v_start = cpe['versionStartExcluding']
+except:
+pass
+try:
+if cpe['versionEndIncluding']:
+op_end = '<='
+v_end = cpe['versionEndIncluding']
+except:
+pass
+try:
+if cpe['versionEndExcluding']:
+op_end = '<'
+v_end = cpe['versionEndExcluding']
+except:
+pass
+
+db_values = [cveId, vendor, product, v_start, op_start, v_end,
op_end]
+insert_elt(c, db_values)
def update_db(c, json_filename):
import json
@@ -125,16 +192,9 @@ def update_db(c, json_filename):
c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
[cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
-for vendor in elt['cve']['affects']['vendor']['vendor_data']:
-for product in vendor['product']['product_data']:
-for version in product['version']['version_data']:
-product_str =
cveId+vendor['vendor_name']+product['product_name']+version['version_value']
-hashstr = hash_djb2(product_str)
-c.execute("insert or replace into PRODUCTS values (?, ?,
?, ?, ?, ?)",
-[ hashstr, cveId, vendor['vendor_name'],
-product['product_name'],
version['version_value'],
-version['version_affected']])
-
+configurations = elt['configurations']['nodes']
+for config in configurations:
+
[OE-core] [PATCH v2 1/3] cve-check: Depends on cve-update-db-native
From: Pierre Le Magourou
do_populate_cve_db is a native task.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 2 +-
meta/recipes-core/meta/{cve-update-db.bb => cve-update-db-native.bb} | 0
2 files changed, 1 insertion(+), 1 deletion(-)
rename meta/recipes-core/meta/{cve-update-db.bb => cve-update-db-native.bb}
(100%)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 81071e3f19..6ffa0c4688 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -63,7 +63,7 @@ python do_cve_check () {
}
addtask cve_check after do_unpack before do_build
-do_cve_check[depends] = "cve-update-db:do_populate_cve_db"
+do_cve_check[depends] = "cve-update-db-native:do_populate_cve_db"
do_cve_check[nostamp] = "1"
python cve_check_cleanup () {
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db-native.bb
similarity index 100%
rename from meta/recipes-core/meta/cve-update-db.bb
rename to meta/recipes-core/meta/cve-update-db-native.bb
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v2 3/3] cve-check: Update unpatched CVE matching
From: Pierre Le Magourou
Now that cve-update-db added CPE information to NVD database. We can
check for unpatched versions with operators '<', '<=', '>', and '>='.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 54 +++---
1 file changed, 40 insertions(+), 14 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 6ffa0c4688..ffd624333f 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}"
CVE_VERSION ??= "${PV}"
CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
-CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvd-json.db"
+CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve.db"
CVE_CHECK_LOG ?= "${T}/cve.log"
CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
@@ -189,27 +189,53 @@ def check_cves(d, patched_cves):
conn = sqlite3.connect(db_file)
c = conn.cursor()
-query = """SELECT * FROM PRODUCTS WHERE
- (PRODUCT IS '{0}' AND VERSION = '{1}' AND OPERATOR IS '=') OR
- (PRODUCT IS '{0}' AND OPERATOR IS '<=');"""
+query = "SELECT * FROM PRODUCTS WHERE PRODUCT IS '{0}';"
+
for product in products:
for row in c.execute(query.format(product, pv)):
cve = row[1]
-version = row[4]
-
-try:
-discardVersion = LooseVersion(version) < LooseVersion(pv)
-except:
-discardVersion = True
+version_start = row[4]
+operator_start = row[5]
+version_end = row[6]
+operator_end = row[7]
if pv in cve_whitelist.get(cve, []):
bb.note("%s-%s has been whitelisted for %s" % (product, pv,
cve))
elif cve in patched_cves:
bb.note("%s has been patched" % (cve))
-elif discardVersion:
-bb.debug(2, "Do not consider version %s " % (version))
else:
-cves_unpatched.append(cve)
+if (operator_start == '=' and pv == version_start):
+cves_unpatched.append(cve)
+else:
+if operator_start:
+try:
+to_append_start = (operator_start == '>=' and
LooseVersion(pv) >= LooseVersion(version_start))
+to_append_start |= (operator_start == '>' and
LooseVersion(pv) > LooseVersion(version_start))
+except:
+bb.note("%s: Failed to compare %s %s %s for %s" %
+(product, pv, operator_start,
version_start, cve))
+to_append_start = False
+else:
+to_append_start = False
+
+if operator_end:
+try:
+to_append_end = (operator_end == '<=' and
LooseVersion(pv) <= LooseVersion(version_end))
+to_append_end |= (operator_end == '<' and
LooseVersion(pv) < LooseVersion(version_end))
+except:
+bb.note("%s: Failed to compare %s %s %s for %s" %
+(product, pv, operator_end, version_end,
cve))
+to_append_end = False
+else:
+to_append_end = False
+
+if operator_start and operator_end:
+to_append = to_append_start and to_append_end
+else:
+to_append = to_append_start or to_append_end
+
+if to_append:
+cves_unpatched.append(cve)
bb.debug(2, "%s-%s is not patched for %s" % (product, pv, cve))
conn.close()
@@ -217,7 +243,7 @@ def check_cves(d, patched_cves):
def get_cve_info(d, cves):
"""
-Get CVE information from the database used by cve-check-tool.
+Get CVE information from the database.
Unfortunately the only way to get CVE info is set the output to
html (hard to parse) or query directly the database.
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [meta-oe][PATCH 1/3] cve-check: Depends on cve-update-db-native
Hello, > > From: Pierre Le Magourou > > > > do_populate_cve_db is a native task. > > > > Signed-off-by: Pierre Le Magourou < > > pierre.lemagou...@softbankrobotics.com> > > --- > > meta/classes/cve-check.bbclass | 2 +- > > meta/recipes-core/meta/cve-update-db.bb | 1 + > > 2 files changed, 2 insertions(+), 1 deletion(-) > > > https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/1015 > > :( The autobuilder job launches cve-update-db and cve-update-db-native in parallel, this cannot work because they both try to update the same sqlite db. As cve-update-db is only used for the build machine (native only), we only need a cve-update-db-native.bb recipe. I'll send a v2 patch to fix that. ps: Sorry I just realized I was sending the patches to the wrong destination (meta-oe instead of oe-core) Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [meta-oe][PATCH 1/3] cve-check: Depends on cve-update-db-native
From: Pierre Le Magourou
do_populate_cve_db is a native task.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 2 +-
meta/recipes-core/meta/cve-update-db.bb | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 81071e3f19..6ffa0c4688 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -63,7 +63,7 @@ python do_cve_check () {
}
addtask cve_check after do_unpack before do_build
-do_cve_check[depends] = "cve-update-db:do_populate_cve_db"
+do_cve_check[depends] = "cve-update-db-native:do_populate_cve_db"
do_cve_check[nostamp] = "1"
python cve_check_cleanup () {
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index ae8f1a958b..8e553b4f9b 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -141,3 +141,4 @@ addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"
+BBCLASSEXTEND =+ "native"
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [meta-oe][PATCH 2/3] cve-update-db: Use NVD CPE data to populate PRODUCTS table
From: Pierre Le Magourou
Instead of using expanded list of affected versions that is not
reliable, use the 'cpe_match' node in the 'configurations' json node.
For cve-check to correctly match affected CVE, the sqlite database need to
contain operator_start, operator_end and the corresponding versions fields.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 88 +++--
1 file changed, 74 insertions(+), 14 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 8e553b4f9b..3ba80a0d28 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -25,7 +25,7 @@ python do_populate_cve_db() {
YEAR_START = 2002
db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
-db_file = db_dir + '/nvd-json.db'
+db_file = db_dir + '/nvdcve.db'
json_tmpfile = db_dir + '/nvd.json.gz'
proxy = d.getVar("https_proxy")
cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
@@ -99,9 +99,76 @@ def initialize_db(c):
c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID
TEXT, \
-VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
-c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
-(PRODUCT, VERSION)")
+VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
+VERSION_END TEXT, OPERATOR_END TEXT)")
+
+def insert_elt(c, db_values):
+product_str = db_values[0] + db_values[1] + db_values[2] + db_values[3]
+hashstr = hash_djb2(product_str)
+db_values.insert(0, hashstr)
+query = "insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?, ?, ?)"
+c.execute(query, db_values)
+
+def parse_node_and_insert(c, node, cveId):
+# Parse children node if needed
+try:
+for child in node['children']:
+parse_node_and_insert(c, child, cveId)
+except:
+pass
+
+# Exit if the cpe_match node does not exists
+try:
+cpe_match = node['cpe_match']
+except:
+return
+
+for cpe in cpe_match:
+if not cpe['vulnerable']:
+return
+cpe23 = cpe['cpe23Uri'].split(':')
+vendor = cpe23[3]
+product = cpe23[4]
+version = cpe23[5]
+
+if version != '*':
+# Version is defined, this is a '=' match
+db_values = [cveId, vendor, product, version, '=', '', '']
+insert_elt(c, db_values)
+else:
+# Parse start version, end version and operators
+op_start = ''
+op_end = ''
+v_start = ''
+v_end = ''
+
+try:
+if cpe['versionStartIncluding']:
+op_start = '>='
+v_start = cpe['versionStartIncluding']
+except:
+pass
+try:
+if cpe['versionStartExcluding']:
+op_start = '>'
+v_start = cpe['versionStartExcluding']
+except:
+pass
+try:
+if cpe['versionEndIncluding']:
+op_end = '<='
+v_end = cpe['versionEndIncluding']
+except:
+pass
+try:
+if cpe['versionEndExcluding']:
+op_end = '<'
+v_end = cpe['versionEndExcluding']
+except:
+pass
+
+db_values = [cveId, vendor, product, v_start, op_start, v_end,
op_end]
+insert_elt(c, db_values)
def update_db(c, json_filename):
import json
@@ -125,16 +192,9 @@ def update_db(c, json_filename):
c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
[cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
-for vendor in elt['cve']['affects']['vendor']['vendor_data']:
-for product in vendor['product']['product_data']:
-for version in product['version']['version_data']:
-product_str =
cveId+vendor['vendor_name']+product['product_name']+version['version_value']
-hashstr = hash_djb2(product_str)
-c.execute("insert or replace into PRODUCTS values (?, ?,
?, ?, ?, ?)",
-[ hashstr, cveId, vendor['vendor_name'],
-product['product_name'],
version['version_value'],
-version['version_affected']])
-
+configurations = elt['configurations']['nodes']
+for config in configurations:
+parse_node_and_insert(c, config, cveId)
addtask do_popula
[OE-core] [meta-oe][PATCH 3/3] cve-check: Update unpatched CVE matching
From: Pierre Le Magourou
Now that cve-update-db added CPE information to NVD database. We can
check for unpatched versions with operators '<', '<=', '>', and '>='.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 54 +++---
1 file changed, 40 insertions(+), 14 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 6ffa0c4688..ffd624333f 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}"
CVE_VERSION ??= "${PV}"
CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
-CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvd-json.db"
+CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve.db"
CVE_CHECK_LOG ?= "${T}/cve.log"
CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
@@ -189,27 +189,53 @@ def check_cves(d, patched_cves):
conn = sqlite3.connect(db_file)
c = conn.cursor()
-query = """SELECT * FROM PRODUCTS WHERE
- (PRODUCT IS '{0}' AND VERSION = '{1}' AND OPERATOR IS '=') OR
- (PRODUCT IS '{0}' AND OPERATOR IS '<=');"""
+query = "SELECT * FROM PRODUCTS WHERE PRODUCT IS '{0}';"
+
for product in products:
for row in c.execute(query.format(product, pv)):
cve = row[1]
-version = row[4]
-
-try:
-discardVersion = LooseVersion(version) < LooseVersion(pv)
-except:
-discardVersion = True
+version_start = row[4]
+operator_start = row[5]
+version_end = row[6]
+operator_end = row[7]
if pv in cve_whitelist.get(cve, []):
bb.note("%s-%s has been whitelisted for %s" % (product, pv,
cve))
elif cve in patched_cves:
bb.note("%s has been patched" % (cve))
-elif discardVersion:
-bb.debug(2, "Do not consider version %s " % (version))
else:
-cves_unpatched.append(cve)
+if (operator_start == '=' and pv == version_start):
+cves_unpatched.append(cve)
+else:
+if operator_start:
+try:
+to_append_start = (operator_start == '>=' and
LooseVersion(pv) >= LooseVersion(version_start))
+to_append_start |= (operator_start == '>' and
LooseVersion(pv) > LooseVersion(version_start))
+except:
+bb.note("%s: Failed to compare %s %s %s for %s" %
+(product, pv, operator_start,
version_start, cve))
+to_append_start = False
+else:
+to_append_start = False
+
+if operator_end:
+try:
+to_append_end = (operator_end == '<=' and
LooseVersion(pv) <= LooseVersion(version_end))
+to_append_end |= (operator_end == '<' and
LooseVersion(pv) < LooseVersion(version_end))
+except:
+bb.note("%s: Failed to compare %s %s %s for %s" %
+(product, pv, operator_end, version_end,
cve))
+to_append_end = False
+else:
+to_append_end = False
+
+if operator_start and operator_end:
+to_append = to_append_start and to_append_end
+else:
+to_append = to_append_start or to_append_end
+
+if to_append:
+cves_unpatched.append(cve)
bb.debug(2, "%s-%s is not patched for %s" % (product, pv, cve))
conn.close()
@@ -217,7 +243,7 @@ def check_cves(d, patched_cves):
def get_cve_info(d, cves):
"""
-Get CVE information from the database used by cve-check-tool.
+Get CVE information from the database.
Unfortunately the only way to get CVE info is set the output to
html (hard to parse) or query directly the database.
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [meta-oe][PATCH v5] cve-update-db: do_populate_cve_db depends on do_fetch
Hello, > This patch broke "bitbake universe --runall=fetch" totally: > > The line which caused the problem is: > addtask do_populate_cve_db before do_fetch > > Would you please fix it? Or I can help if needed. > This seems related to the NVD website down problem. I sent a patch to manage inaccessible URLs. cve-update-db will not fail anymore if the NVD data feeds cannot be downloaded. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [meta-oe][PATCH] cve-update-db: Catch request.urlopen errors.
From: Pierre Le Magourou
If the NVD url is not accessible, print a warning on top of the CVE
report, and continue. The database will not be fully updated, but
cve_check can still run on the previous database.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 5 +++--
meta/recipes-core/meta/cve-update-db.bb | 30 +-
2 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 1e7e8dd441..81071e3f19 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -51,14 +51,15 @@ python do_cve_check () {
Check recipe for patched and unpatched CVEs
"""
-if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")):
+if os.path.exists(d.getVar("CVE_CHECK_DB_FILE")):
patched_cves = get_patches_cves(d)
patched, unpatched = check_cves(d, patched_cves)
if patched or unpatched:
cve_data = get_cve_info(d, patched + unpatched)
cve_write_data(d, patched, unpatched, cve_data)
else:
-bb.note("Failed to update CVE database, skipping CVE check")
+bb.note("No CVE database found, skipping CVE check")
+
}
addtask cve_check after do_unpack before do_build
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 3e5bae8b1d..ae8f1a958b 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -28,6 +28,7 @@ python do_populate_cve_db() {
db_file = db_dir + '/nvd-json.db'
json_tmpfile = db_dir + '/nvd.json.gz'
proxy = d.getVar("https_proxy")
+cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
if not os.path.isdir(db_dir):
os.mkdir(db_dir)
@@ -47,9 +48,13 @@ python do_populate_cve_db() {
req = urllib.request.Request(meta_url)
if proxy:
req.set_proxy(proxy, 'https')
-with urllib.request.urlopen(req) as r:
-date_line = str(r.read().splitlines()[0])
-last_modified = re.search('lastModifiedDate:(.*)',
date_line).group(1)
+try:
+with urllib.request.urlopen(req, timeout=1) as r:
+date_line = str(r.read().splitlines()[0])
+last_modified = re.search('lastModifiedDate:(.*)',
date_line).group(1)
+except:
+cve_f.write('Warning: CVE db update error, CVE data is
outdated.\n\n')
+break
# Compare with current db last modified date
c.execute("select DATE from META where YEAR = '%d'" % year)
@@ -59,19 +64,26 @@ python do_populate_cve_db() {
req = urllib.request.Request(json_url)
if proxy:
req.set_proxy(proxy, 'https')
-with urllib.request.urlopen(req) as r, open(json_tmpfile, 'wb') as
tmpfile:
-shutil.copyfileobj(r, tmpfile)
+try:
+with urllib.request.urlopen(req, timeout=1) as r, \
+ open(json_tmpfile, 'wb') as tmpfile:
+shutil.copyfileobj(r, tmpfile)
+except:
+cve_f.write('Warning: CVE db update error, CVE data is
outdated.\n\n')
+break
+
with gzip.open(json_tmpfile, 'rt') as jsonfile:
update_db(c, jsonfile)
c.execute("insert or replace into META values (?, ?)",
[year, last_modified])
+# Update success, set the date to cve_check file.
+if year == date.today().year:
+cve_f.write('CVE database update : %s\n\n' % date.today())
+
+cve_f.close()
conn.commit()
conn.close()
-
-cve_check_tmp_file = d.getVar("TMPDIR") + '/cve_check'
-with open(cve_check_tmp_file, 'a'):
-os.utime(cve_check_tmp_file, None)
}
# DJB2 hash algorithm
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] cve-update-db: Use std library instead of urllib3
Hi, > > On 07/02/2019 03:39 PM, ChenQi wrote: > > > A recent do_populate_cve_db failure: > > > https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/1005/steps/7/logs/errors > > > > > > > > > Could you please help look at what's going on? > On Tue, 2019-07-02 at 16:08 +0800, ChenQi wrote: > > I just tired to open the web but failed. The web seems down. > > > > We were making some changes to the autobuilder to speed it up (changing > database backends), its back now. > The NVD website is down, cve-update-db should not fail in that case. I need to update it, so it prints a warning instead. I am also wondering if we should mirror NVD json data feeds, to avoid too many requests from Yocto users. (but I don't know how to do that) Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [meta-oe][PATCH v5] cve-update-db: do_populate_cve_db depends on do_fetch
From: Pierre Le Magourou
To be able to populate NVD database on a fetchall
(bitbake --run-all=fetch), set the do_populate_cve_db task to be
executed before do_fetch.
Do not get CVE_CHECK_DB_DIR, CVE_CHECK_DB_FILE and CVE_CHECK_TMP_FILE
variable because do_populate_cve_db can be called in a context where
cve-check class is not loaded.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 21 +
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 4c896dc880..3e5bae8b1d 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -6,7 +6,6 @@ PACKAGES = ""
inherit nopackages
-deltask do_fetch
deltask do_unpack
deltask do_patch
deltask do_configure
@@ -24,11 +23,16 @@ python do_populate_cve_db() {
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-;
YEAR_START = 2002
-JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz'
+
+db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
+db_file = db_dir + '/nvd-json.db'
+json_tmpfile = db_dir + '/nvd.json.gz'
proxy = d.getVar("https_proxy")
+if not os.path.isdir(db_dir):
+os.mkdir(db_dir)
+
# Connect to database
-db_file = d.getVar("CVE_CHECK_DB_FILE")
conn = sqlite3.connect(db_file)
c = conn.cursor()
@@ -55,9 +59,9 @@ python do_populate_cve_db() {
req = urllib.request.Request(json_url)
if proxy:
req.set_proxy(proxy, 'https')
-with urllib.request.urlopen(req) as r, open(JSON_TMPFILE, 'wb') as
tmpfile:
+with urllib.request.urlopen(req) as r, open(json_tmpfile, 'wb') as
tmpfile:
shutil.copyfileobj(r, tmpfile)
-with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
+with gzip.open(json_tmpfile, 'rt') as jsonfile:
update_db(c, jsonfile)
c.execute("insert or replace into META values (?, ?)",
[year, last_modified])
@@ -65,8 +69,9 @@ python do_populate_cve_db() {
conn.commit()
conn.close()
-with open(d.getVar("CVE_CHECK_TMP_FILE"), 'a'):
-os.utime(d.getVar("CVE_CHECK_TMP_FILE"), None)
+cve_check_tmp_file = d.getVar("TMPDIR") + '/cve_check'
+with open(cve_check_tmp_file, 'a'):
+os.utime(cve_check_tmp_file, None)
}
# DJB2 hash algorithm
@@ -120,7 +125,7 @@ def update_db(c, json_filename):
-addtask do_populate_cve_db before do_cve_check
+addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [meta-oe][PATCH v4] cve-update-db: do_populate_cve_db depends on do_fetch
From: Pierre Le Magourou
To be able to populate NVD database on a fetchall
(bitbake --run-all=fetch), set the do_populate_cve_db task to be
executed before do_fetch.
Do not get CVE_CHECK_DB_DIR, CVE_CHECK_DB_FILE and CVE_CHECK_TMP_FILE
variable because do_populate_cve_db can be called in a context where
cve-check class is not loaded.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 22 ++
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 522fd23807..0875c00835 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -6,7 +6,6 @@ PACKAGES = ""
inherit nopackages
-deltask do_fetch
deltask do_unpack
deltask do_patch
deltask do_configure
@@ -24,10 +23,16 @@ python do_populate_cve_db() {
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-;
YEAR_START = 2002
-JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz'
+
+db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
+db_file = db_dir + '/nvd-json.db'
+json_tmpfile = db_dir + '/nvd.json.gz'
+proxy = d.getVar("https_proxy")
+
+if not os.path.isdir(db_dir):
+os.mkdir(db_dir)
# Connect to database
-db_file = d.getVar("CVE_CHECK_DB_FILE")
conn = sqlite3.connect(db_file)
c = conn.cursor()
@@ -50,9 +55,9 @@ python do_populate_cve_db() {
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Update db with current year json file
-with http.request('GET', json_url, preload_content=False) as r,
open(JSON_TMPFILE, 'wb') as tmpfile:
+with http.request('GET', json_url, preload_content=False) as r,
open(json_tmpfile, 'wb') as tmpfile:
shutil.copyfileobj(r, tmpfile)
-with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
+with gzip.open(json_tmpfile, 'rt') as jsonfile:
update_db(c, jsonfile)
c.execute("insert or replace into META values (?, ?)",
[year, last_modified])
@@ -60,8 +65,9 @@ python do_populate_cve_db() {
conn.commit()
conn.close()
-with open(d.getVar("CVE_CHECK_TMP_FILE"), 'a'):
-os.utime(d.getVar("CVE_CHECK_TMP_FILE"), None)
+cve_check_tmp_file = d.getVar("TMPDIR") + '/cve_check'
+with open(cve_check_tmp_file, 'a'):
+os.utime(cve_check_tmp_file, None)
}
# DJB2 hash algorithm
@@ -115,7 +121,7 @@ def update_db(c, json_filename):
-addtask do_populate_cve_db before do_cve_check
+addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [meta-oe][PATCH v3] cve-update-db: do_populate_cve_db depends on do_fetch
From: Pierre Le Magourou
To be able to populate NVD database on a fetchall
(bitbake --run-all=fetch), set the do_populate_cve_db task to be
executed before do_fetch.
Do not get CVE_CHECK_DB_DIR, CVE_CHECK_DB_FILE and CVE_CHECK_TMP_FILE
variable because do_populate_cve_db can be called in a context where
cve-check class is not loaded.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 22 ++
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 522fd23807..5c14cf60af 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -6,7 +6,6 @@ PACKAGES = ""
inherit nopackages
-deltask do_fetch
deltask do_unpack
deltask do_patch
deltask do_configure
@@ -24,10 +23,16 @@ python do_populate_cve_db() {
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-;
YEAR_START = 2002
-JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz'
+
+db_dir = d.getVar("DL_DIR") + 'CVE_CHECK'
+db_file = db_dir + '/nvd-json.db'
+json_tmpfile = db_dir + '/nvd.json.gz'
+proxy = d.getVar("https_proxy")
+
+if not os.path.isdir(db_dir):
+os.mkdir(db_dir)
# Connect to database
-db_file = d.getVar("CVE_CHECK_DB_FILE")
conn = sqlite3.connect(db_file)
c = conn.cursor()
@@ -50,9 +55,9 @@ python do_populate_cve_db() {
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Update db with current year json file
-with http.request('GET', json_url, preload_content=False) as r,
open(JSON_TMPFILE, 'wb') as tmpfile:
+with http.request('GET', json_url, preload_content=False) as r,
open(json_tmpfile, 'wb') as tmpfile:
shutil.copyfileobj(r, tmpfile)
-with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
+with gzip.open(json_tmpfile, 'rt') as jsonfile:
update_db(c, jsonfile)
c.execute("insert or replace into META values (?, ?)",
[year, last_modified])
@@ -60,8 +65,9 @@ python do_populate_cve_db() {
conn.commit()
conn.close()
-with open(d.getVar("CVE_CHECK_TMP_FILE"), 'a'):
-os.utime(d.getVar("CVE_CHECK_TMP_FILE"), None)
+cve_check_tmp_file = d.getVar("TMPDIR") + '/cve_check'
+with open(cve_check_tmp_file, 'a'):
+os.utime(cve_check_tmp_file, None)
}
# DJB2 hash algorithm
@@ -115,7 +121,7 @@ def update_db(c, json_filename):
-addtask do_populate_cve_db before do_cve_check
+addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [meta-oe][PATCH v2] cve-update-db: do_populate_cve_db depends on do_fetch
> Thanks for the fast turnaround on the patch. This seems to move the
> problem to:
>
> https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/990/steps/7/logs/step1b
>
> :(
>
> Perhaps we need to skip the recipe if the class isn't enabled? Is that
> the reason its failing like this?
It is failing with 'sqlite3.OperationalError' exception when opening
the sqlite3 db file.
I reproduced the bug by removing the ${DL_DIR}/CVE_CHECK directory
I forgot to check that the ${DL_DIR}/CVE_CHECK directory exists before
creating sqlite db file in it.
I'll send a v3 patch to fix this.
Pierre
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [meta-oe][PATCH v2] cve-update-db: do_populate_cve_db depends on do_fetch
From: Pierre Le Magourou
To be able to populate NVD database on a fetchall
(bitbake --run-all=fetch), set the do_populate_cve_db task to be
executed before do_fetch.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 522fd23807..e096db7717 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -4,9 +4,12 @@ LICENSE = "MIT"
INHIBIT_DEFAULT_DEPS = "1"
PACKAGES = ""
+CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
+CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvd-json.db"
+CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
+
inherit nopackages
-deltask do_fetch
deltask do_unpack
deltask do_patch
deltask do_configure
@@ -115,7 +118,7 @@ def update_db(c, json_filename):
-addtask do_populate_cve_db before do_cve_check
+addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/4] cve-update-db: New recipe to update CVE database
Hi, > It looks like CVE_CHECK_DB_DIR has no default value which resulted in: > > https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/988/steps/7/logs/step1b > > We only started seeing that error after your later patch to add back > the do_fetch task. build-appliance is trying to collect up all the > sources it may need. > I see the problem, it happens when cve-update-db do_fetch task is executed and cve-check class is not inherited. I sent a v2 patch that sets default value to CVE_CHECK_DB_DIR. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] cve-update-db: Manage proxy if needed.
From: Pierre Le Magourou
If https_proxy environment variable is defined, manage proxy to be able
to download meta and json data feeds from https://nvd.nist.gov
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 5778e00e79..bcc8d4ab1a 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -24,6 +24,7 @@ python do_populate_cve_db() {
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-;
YEAR_START = 2002
JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz'
+proxy = d.getVar("https_proxy")
# Connect to database
db_file = d.getVar("CVE_CHECK_DB_FILE")
@@ -38,7 +39,10 @@ python do_populate_cve_db() {
json_url = year_url + ".json.gz"
# Retrieve meta last modified date
-with urllib.request.urlopen(meta_url) as r:
+req = urllib.request.Request(meta_url)
+if proxy:
+req.set_proxy(proxy, 'https')
+with urllib.request.urlopen(req) as r:
date_line = str(r.read().splitlines()[0])
last_modified = re.search('lastModifiedDate:(.*)',
date_line).group(1)
@@ -47,7 +51,10 @@ python do_populate_cve_db() {
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Update db with current year json file
-with urllib.request.urlopen(json_url) as r, open(JSON_TMPFILE,
'wb') as tmpfile:
+req = urllib.request.Request(json_url)
+if proxy:
+req.set_proxy(proxy, 'https')
+with urllib.request.urlopen(req) as r, open(JSON_TMPFILE, 'wb') as
tmpfile:
shutil.copyfileobj(r, tmpfile)
with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
update_db(c, jsonfile)
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] cve-update-db: do_populate_cve_db depends on do_fetch
From: Pierre Le Magourou To be able to populate NVD database on a fetchall (bitbake --run-all=fetch), set the do_populate_cve_db task to be executed before do_fetch. Signed-off-by: Pierre Le Magourou --- meta/recipes-core/meta/cve-update-db.bb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meta/recipes-core/meta/cve-update-db.bb b/meta/recipes-core/meta/cve-update-db.bb index 1f48820cc6..5778e00e79 100644 --- a/meta/recipes-core/meta/cve-update-db.bb +++ b/meta/recipes-core/meta/cve-update-db.bb @@ -6,7 +6,6 @@ PACKAGES = "" inherit nopackages -deltask do_fetch deltask do_unpack deltask do_patch deltask do_configure @@ -113,7 +112,7 @@ def update_db(c, json_filename): -addtask do_populate_cve_db before do_cve_check +addtask do_populate_cve_db before do_fetch do_populate_cve_db[nostamp] = "1" EXCLUDE_FROM_WORLD = "1" -- 2.11.0 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/4] cve-update-db: New recipe to update CVE database
Hi, > Also, the CVE db is updated using this custom task without link to > do_fetch, which means a fetchall task would not update the database for > off line NO_NETWORK builds. > > Could the task be added as dependency to do_fetch() or are there some other > side effects? > Yes I can do that, I don't think this will cause side effects. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] cve-update-db: Use std library instead of urllib3
From: Pierre Le Magourou
urllib3 was used in this recipe but it was not set as a
dependency. As it is not specifically needed, rewrite the recipe with
urllib from the standard library.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 10 --
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
index 522fd23807..1f48820cc6 100644
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -19,7 +19,7 @@ python do_populate_cve_db() {
Update NVD database with json data feed
"""
-import sqlite3, urllib3, shutil, gzip, re
+import sqlite3, urllib, shutil, gzip, re
from datetime import date
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-;
@@ -33,16 +33,14 @@ python do_populate_cve_db() {
initialize_db(c)
-http = urllib3.PoolManager()
-
for year in range(YEAR_START, date.today().year + 1):
year_url = BASE_URL + str(year)
meta_url = year_url + ".meta"
json_url = year_url + ".json.gz"
# Retrieve meta last modified date
-with http.request('GET', meta_url, preload_content=False) as r:
-date_line = str(r.data.splitlines()[0])
+with urllib.request.urlopen(meta_url) as r:
+date_line = str(r.read().splitlines()[0])
last_modified = re.search('lastModifiedDate:(.*)',
date_line).group(1)
# Compare with current db last modified date
@@ -50,7 +48,7 @@ python do_populate_cve_db() {
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Update db with current year json file
-with http.request('GET', json_url, preload_content=False) as r,
open(JSON_TMPFILE, 'wb') as tmpfile:
+with urllib.request.urlopen(json_url) as r, open(JSON_TMPFILE,
'wb') as tmpfile:
shutil.copyfileobj(r, tmpfile)
with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
update_db(c, jsonfile)
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/4] cve-update-db: New recipe to update CVE database
Hi, > > This adds python3 urllib3 (python3-urllib3 in Debian) to build environment > > dependencies. It's the first user of urllib3 in poky, AFAIK. Maybe > > documentation could be updated too, e.g. > > https://www.yoctoproject.org/docs/latest/mega-manual/mega-manual.html#brief-build-system-packages > > Somehow I didn't notice it, thanks. > > Pierre, can you rewrite this to use standard library instead of urllib3? Yes of course, I only need urrlib to fetch the NVD json feeds, so I don't need urllib3, I can use standard library. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/4] cve-update-db: New recipe to update CVE database
> Not sure which of the changes is responsible, but this is new: > WARNING: flex-native-2.6.0-r0 do_cve_check: Found unpatched CVE > (CVE-2015-1773) > > https://nvd.nist.gov/vuln/detail/CVE-2015-1773 > > Note that the flex tool is completely unrelated to Apache Flex. > > I see, the 4/4 patch is responsible for that (Consider CVE that affects versions with less than operator). It takes into account the comparison operator in the json NVD file (new 'version_affected' field that was not in the XML data feed). So this CVE matches because 2.6.0 <= 4.14.0. But it should not match because it concerns another product (flex_project/flex vs Apache/flex). There is indeed a problem I didn't manage. The CVE_PRODUCT variable we use in cve-check only takes the product name (here 'flex') into account, we should also consider the vendor name (here 'flex_project'). Without this patch (4/4), the behaviour should be the same as before. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 2/4] cve-check: Remove dependency to cve-check-tool-native
> Does this mean we can delete the cve-check-tool recipe itself too? > Yes, with this patch cve-check class does not need cve-check-tool anymore. Pierre -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 4/4] cve-check: Consider CVE that affects versions with less than operator
From: Pierre Le Magourou
In the NVD json CVE feed, affected versions can be strictly matched to a
version, but they can also be matched with the operator '<='.
Add a new condition in the sqlite query to match affected versions that
are defined with the operator '<='. Then use LooseVersion to discard all
versions that are not relevant.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 16 ++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index e7540b8c1f..379f7121cc 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -166,6 +166,7 @@ def check_cves(d, patched_cves):
Connect to the NVD database and find unpatched cves.
"""
import ast, csv, tempfile, subprocess, io
+from distutils.version import LooseVersion
cves_unpatched = []
# CVE_PRODUCT can contain more than one product (eg. curl/libcurl)
@@ -186,14 +187,25 @@ def check_cves(d, patched_cves):
conn = sqlite3.connect(db_file)
c = conn.cursor()
-query = "SELECT * FROM PRODUCTS WHERE PRODUCT IS '%s' AND VERSION IS '%s';"
+query = """SELECT * FROM PRODUCTS WHERE
+ (PRODUCT IS '{0}' AND VERSION = '{1}' AND OPERATOR IS '=') OR
+ (PRODUCT IS '{0}' AND OPERATOR IS '<=');"""
for idx in range(len(bpn)):
-for row in c.execute(query % (bpn[idx],pv)):
+for row in c.execute(query.format(bpn[idx],pv)):
cve = row[1]
+version = row[4]
+
+try:
+discardVersion = LooseVersion(version) < LooseVersion(pv)
+except:
+discardVersion = True
+
if pv in cve_whitelist.get(cve,[]):
bb.note("%s-%s has been whitelisted for %s" % (bpn[idx], pv,
cve))
elif cve in patched_cves:
bb.note("%s has been patched" % (cve))
+elif discardVersion:
+bb.debug(2, "Do not consider version %s " % (version))
else:
cves_unpatched.append(cve)
bb.debug(2, "%s-%s is not patched for %s" % (bpn[idx], pv,
cve))
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 3/4] cve-check: Manage CVE_PRODUCT with more than one name
From: Pierre Le Magourou
In some rare cases (eg. curl recipe) the CVE_PRODUCT contains more than
one name.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 25 ++---
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 28619c7bd4..e7540b8c1f 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -168,9 +168,10 @@ def check_cves(d, patched_cves):
import ast, csv, tempfile, subprocess, io
cves_unpatched = []
-bpn = d.getVar("CVE_PRODUCT")
+# CVE_PRODUCT can contain more than one product (eg. curl/libcurl)
+bpn = d.getVar("CVE_PRODUCT").split()
# If this has been unset then we're not scanning for CVEs here (for
example, image recipes)
-if not bpn:
+if len(bpn) == 0:
return ([], [])
pv = d.getVar("CVE_VERSION").split("+git")[0]
cve_whitelist = ast.literal_eval(d.getVar("CVE_CHECK_CVE_WHITELIST"))
@@ -184,16 +185,18 @@ def check_cves(d, patched_cves):
db_file = d.getVar("CVE_CHECK_DB_FILE")
conn = sqlite3.connect(db_file)
c = conn.cursor()
+
query = "SELECT * FROM PRODUCTS WHERE PRODUCT IS '%s' AND VERSION IS '%s';"
-for row in c.execute(query % (bpn,pv)):
-cve = row[1]
-if pv in cve_whitelist.get(cve,[]):
-bb.note("%s-%s has been whitelisted for %s" % (bpn, pv, cve))
-elif cve in patched_cves:
-bb.note("%s has been patched" % (cve))
-else:
-cves_unpatched.append(cve)
-bb.debug(2, "%s-%s is not patched for %s" % (bpn, pv, cve))
+for idx in range(len(bpn)):
+for row in c.execute(query % (bpn[idx],pv)):
+cve = row[1]
+if pv in cve_whitelist.get(cve,[]):
+bb.note("%s-%s has been whitelisted for %s" % (bpn[idx], pv,
cve))
+elif cve in patched_cves:
+bb.note("%s has been patched" % (cve))
+else:
+cves_unpatched.append(cve)
+bb.debug(2, "%s-%s is not patched for %s" % (bpn[idx], pv,
cve))
conn.close()
return (list(patched_cves), cves_unpatched)
--
2.11.0
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/4] cve-update-db: New recipe to update CVE database
From: Pierre Le Magourou
cve-check-tool-native do_populate_cve_db task was using deprecated NVD
xml data feeds, cve-update-db uses NVD json data feeds.
Sqlite database schema was updated to take into account CVSSv3 CVE
scores and operator in affected product versions.
A new META table was added to store the last modification date of the
NVD json data feeds.
Signed-off-by: Pierre Le Magourou
---
meta/recipes-core/meta/cve-update-db.bb | 121
1 file changed, 121 insertions(+)
create mode 100644 meta/recipes-core/meta/cve-update-db.bb
diff --git a/meta/recipes-core/meta/cve-update-db.bb
b/meta/recipes-core/meta/cve-update-db.bb
new file mode 100644
index 00..522fd23807
--- /dev/null
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -0,0 +1,121 @@
+SUMMARY = "Updates the NVD CVE database"
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+PACKAGES = ""
+
+inherit nopackages
+
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
+deltask do_populate_sysroot
+
+python do_populate_cve_db() {
+"""
+Update NVD database with json data feed
+"""
+
+import sqlite3, urllib3, shutil, gzip, re
+from datetime import date
+
+BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-;
+YEAR_START = 2002
+JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz'
+
+# Connect to database
+db_file = d.getVar("CVE_CHECK_DB_FILE")
+conn = sqlite3.connect(db_file)
+c = conn.cursor()
+
+initialize_db(c)
+
+http = urllib3.PoolManager()
+
+for year in range(YEAR_START, date.today().year + 1):
+year_url = BASE_URL + str(year)
+meta_url = year_url + ".meta"
+json_url = year_url + ".json.gz"
+
+# Retrieve meta last modified date
+with http.request('GET', meta_url, preload_content=False) as r:
+date_line = str(r.data.splitlines()[0])
+last_modified = re.search('lastModifiedDate:(.*)',
date_line).group(1)
+
+# Compare with current db last modified date
+c.execute("select DATE from META where YEAR = '%d'" % year)
+meta = c.fetchone()
+if not meta or meta[0] != last_modified:
+# Update db with current year json file
+with http.request('GET', json_url, preload_content=False) as r,
open(JSON_TMPFILE, 'wb') as tmpfile:
+shutil.copyfileobj(r, tmpfile)
+with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
+update_db(c, jsonfile)
+c.execute("insert or replace into META values (?, ?)",
+[year, last_modified])
+
+conn.commit()
+conn.close()
+
+with open(d.getVar("CVE_CHECK_TMP_FILE"), 'a'):
+os.utime(d.getVar("CVE_CHECK_TMP_FILE"), None)
+}
+
+# DJB2 hash algorithm
+def hash_djb2(s):
+hash = 5381
+for x in s:
+hash = (( hash << 5) + hash) + ord(x)
+
+return hash & 0x
+
+def initialize_db(c):
+c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE
TEXT)")
+c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
+SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
+c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID
TEXT, \
+VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
+c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
+(PRODUCT, VERSION)")
+
+def update_db(c, json_filename):
+import json
+root = json.load(json_filename)
+
+for elt in root['CVE_Items']:
+if not elt['impact']:
+continue
+
+cveId = elt['cve']['CVE_data_meta']['ID']
+cveDesc = elt['cve']['description']['description_data'][0]['value']
+date = elt['lastModifiedDate']
+accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
+cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
+
+try:
+cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
+except:
+cvssv3 = 0.0
+
+c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
+[cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
+
+for vendor in elt['cve']['affects']['vendor']['vendor_data']:
+for product in vendor['product']['product_data']:
+for version in product['version']['version_data']:
+product_str =
cveId+vendor['vendor_name']+product['product_name']+version['version_value']
+hashstr = hash_djb2(product_str)
+c.execute("insert or replace into PRODUCTS values (?, ?,
?, ?, ?, ?)",
+
[OE-core] [PATCH 2/4] cve-check: Remove dependency to cve-check-tool-native
From: Pierre Le Magourou
Use the new update-cve-db recipe to update database.
Signed-off-by: Pierre Le Magourou
---
meta/classes/cve-check.bbclass | 71 --
1 file changed, 26 insertions(+), 45 deletions(-)
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 743bc08a4f..28619c7bd4 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}"
CVE_VERSION ??= "${PV}"
CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
-CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvd.db"
+CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvd-json.db"
CVE_CHECK_LOG ?= "${T}/cve.log"
CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
@@ -62,7 +62,7 @@ python do_cve_check () {
}
addtask cve_check after do_unpack before do_build
-do_cve_check[depends] = "cve-check-tool-native:do_populate_sysroot
cve-check-tool-native:do_populate_cve_db"
+do_cve_check[depends] = "cve-update-db:do_populate_cve_db"
do_cve_check[nostamp] = "1"
python cve_check_cleanup () {
@@ -163,61 +163,40 @@ def get_patches_cves(d):
def check_cves(d, patched_cves):
"""
-Run cve-check-tool looking for patched and unpatched CVEs.
+Connect to the NVD database and find unpatched cves.
"""
-
import ast, csv, tempfile, subprocess, io
-cves_patched = []
cves_unpatched = []
bpn = d.getVar("CVE_PRODUCT")
# If this has been unset then we're not scanning for CVEs here (for
example, image recipes)
if not bpn:
return ([], [])
pv = d.getVar("CVE_VERSION").split("+git")[0]
-cves = " ".join(patched_cves)
-cve_db_dir = d.getVar("CVE_CHECK_DB_DIR")
cve_whitelist = ast.literal_eval(d.getVar("CVE_CHECK_CVE_WHITELIST"))
-cve_cmd = "cve-check-tool"
-cmd = [cve_cmd, "--no-html", "--skip-update", "--csv", "--not-affected",
"-t", "faux", "-d", cve_db_dir]
# If the recipe has been whitlisted we return empty lists
if d.getVar("PN") in d.getVar("CVE_CHECK_PN_WHITELIST").split():
bb.note("Recipe has been whitelisted, skipping check")
return ([], [])
-try:
-# Write the faux CSV file to be used with cve-check-tool
-fd, faux = tempfile.mkstemp(prefix="cve-faux-")
-with os.fdopen(fd, "w") as f:
-for pn in bpn.split():
-f.write("%s,%s,%s,\n" % (pn, pv, cves))
-cmd.append(faux)
-
-output = subprocess.check_output(cmd).decode("utf-8")
-bb.debug(2, "Output of command %s:\n%s" % ("\n".join(cmd), output))
-except subprocess.CalledProcessError as e:
-bb.warn("Couldn't check for CVEs: %s (output %s)" % (e, e.output))
-finally:
-os.remove(faux)
-
-for row in csv.reader(io.StringIO(output)):
-# Third row has the unpatched CVEs
-if row[2]:
-for cve in row[2].split():
-# Skip if the CVE has been whitlisted for the current version
-if pv in cve_whitelist.get(cve,[]):
-bb.note("%s-%s has been whitelisted for %s" % (bpn, pv,
cve))
-else:
-cves_unpatched.append(cve)
-bb.debug(2, "%s-%s is not patched for %s" % (bpn, pv, cve))
-# Fourth row has patched CVEs
-if row[3]:
-for cve in row[3].split():
-cves_patched.append(cve)
-bb.debug(2, "%s-%s is patched for %s" % (bpn, pv, cve))
-
-return (cves_patched, cves_unpatched)
+import sqlite3
+db_file = d.getVar("CVE_CHECK_DB_FILE")
+conn = sqlite3.connect(db_file)
+c = conn.cursor()
+query = "SELECT * FROM PRODUCTS WHERE PRODUCT IS '%s' AND VERSION IS '%s';"
+for row in c.execute(query % (bpn,pv)):
+cve = row[1]
+if pv in cve_whitelist.get(cve,[]):
+bb.note("%s-%s has been whitelisted for %s" % (bpn, pv, cve))
+elif cve in patched_cves:
+bb.note("%s has been patched" % (cve))
+else:
+cves_unpatched.append(cve)
+bb.debug(2, "%s-%s is not patched for %s" % (bpn, pv, cve))
+conn.close()
+
+return (list(patched_cves), cves_unpatched)
def get_cve_info(d, cves):
"""
@@ -241,9 +220,10 @@ def get_cve_info(d, cves):
for row in cur.execute(query, tuple(cves)):
cve_data[row[0]] = {}
cve_data[row[0]]["summary"] = row[1]
-cve_data[row[0]]["score"] = row[2]
-cve_data[row[0]]["modified"] = row[3
