Hi Nicholas,
sorry for the long delay in getting back to you on this topic. I finally
set aside the time to go through your work, and it's quite
impressive. I'll need to do a bit more testing, but we should be able to
integrate your contribution into the security repository, and use that
to provide OVAL definitions again.
Thanks again for your work. Cheers,
--Seb
On Nov/13, Luedtke, Nicholas S wrote:
> Attached is a working solution to this bug. Right now parsing the JSON
> Security Tracker Information results in a one definition per CVE. I
> hope to reduce this to one definition per package. (Shouldn't be too
> much work) Also included in the tar is are more updates to the legacy
> versions of parsing the dsa/wml files. If you need a debian oval
> interpreter I previously attached one to this forum.
>
> I encourage you to use -h option for the JSON parsing script as
> downloading the security tracker each time is cumbersome.
>
> shasums:
> 5d13f72cccc6ad1774e2437c98c26bebfc2100ac 68910 diff.txt
> 57fa67e77ed5212d65deca0483b7ac90b5bb7203 21423 oval.tar.gz
>
> --
> Nicholas Luedtke
> Linux for HPE Helion OpenStack, Hewlett Packard Enterprise
> Binary files oval_old/oval/definition/differ.pyc and
> oval/oval/definition/differ.pyc differ
> diff -urN oval_old/oval/definition/generator.py
> oval/oval/definition/generator.py
> --- oval_old/oval/definition/generator.py 2011-10-12 15:14:34.000000000
> -0600
> +++ oval/oval/definition/generator.py 2015-11-13 11:20:30.043834000 -0700
> @@ -1,5 +1,5 @@
> # -*- coding: utf-8 -*-
> -# oval.definitio.generator - generate well-formed xml file with
> +# oval.definition.generator - generate well-formed xml file with
> # OVAL definitions of Debian Security Advisories.
> # Use various optimizations to minimize result XML
> #
> @@ -19,44 +19,44 @@
> RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])'
> + u'|' + u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' %
> (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
> unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
> unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff))
> regex = re.compile(RE_XML_ILLEGAL)
>
> -
> +
> class OvalGeneratorException (Exception):
> pass
>
> class DSAFormatException (OvalGeneratorException):
> - code = 1
> -
> + code = 1
> +
> def __createXMLElement (name, descr = None, attrs = {}):
> - """
> - Create XML element with text descr and attributes attrs
> -
> - Keyword arguments:
> - name -- Name of XML element
> - descr -- content of textNode (default None)
> - attrs -- attributes of element (default {})
> -
> - Return created XML element
> - """
> -
> - doc = xml.dom.minidom.Document ()
> - element = doc.createElement (name)
> -
> - for (attr, value) in attrs.items():
> - for match in regex.finditer(attr):
> - attr = attr[:match.start()] + "?" + attr[match.end():]
> - for match in regex.finditer(value):
> - value = value[:match.start()] + "?" + value[match.end():]
> - attribute = doc.createAttribute (attr.encode("utf8"))
> - attribute.value = value.encode("utf8")
> - element.attributes.setNamedItem (attribute)
> -
> - if descr != None:
> - for match in regex.finditer(descr):
> - descr = descr[:match.start()] + "?" + descr[match.end():]
> - description = doc.createTextNode (descr.encode("utf8"))
> - element.appendChild (description)
> -
> - return (element)
> + """
> + Create XML element with text descr and attributes attrs
> +
> + Keyword arguments:
> + name -- Name of XML element
> + descr -- content of textNode (default None)
> + attrs -- attributes of element (default {})
> +
> + Return created XML element
> + """
> +
> + doc = xml.dom.minidom.Document ()
> + element = doc.createElement (name)
> +
> + for (attr, value) in attrs.items():
> + for match in regex.finditer(attr):
> + attr = attr[:match.start()] + "?" + attr[match.end():]
> + for match in regex.finditer(value):
> + value = value[:match.start()] + "?" + value[match.end():]
> + attribute = doc.createAttribute (attr.encode("utf8"))
> + attribute.value = value.encode("utf8")
> + element.attributes.setNamedItem (attribute)
> +
> + if descr != None:
> + for match in regex.finditer(descr):
> + descr = descr[:match.start()] + "?" + descr[match.end():]
> + description = doc.createTextNode (descr.encode("utf8"))
> + element.appendChild (description)
> +
> + return (element)
>
> namespace = "oval:org.debian.oval"
> tests = __createXMLElement ("tests")
> @@ -66,464 +66,464 @@
> testsCurId = 1
> objectsCurId = 1
> statesCurId = 1
> -
> releaseArchHash = {"2.0" : 2, "2.1" : 4, "2.2": 6, "3.0" : 11, "3.1" : 12,
> "4.0" : 11, "5.0": 12, "6.0": 11}
> testsHash = {"arch" : {}, "release": {}, "obj": {}, "fileSte": {},
> "unameSte" : {}, "dpkgSte": {}}
> #We need more info about alpha, arm, hppa, bmips, lmips
> unameArchTable = {'i386' : 'i686', 'amd64' : 'x86-64', 'ia64' : 'ia64',
> 'powerpc' : 'ppc', 's390' : 's390x', 'm86k' : 'm86k'}
>
> def __trimzero (val):
> - value = val[:]
> - while value[0] == "0":
> - value = value[1:]
> - return value
> + value = val[:]
> + while value[0] == "0":
> + value = value[1:]
> + return value
>
> def __getNewId (type):
> - """Generate new unique id for tests, objects or states
> -
> - Argument keqywords:
> - type -- type of generated id test | object | state
> -
> - return Generate id like <namespace>:tst|obj|ste:<id>
> - """
> - global testsCurId, objectsCurId, statesCurId
> -
> - if type == "test":
> - result = "%s:tst:%d" % (namespace, testsCurId)
> - testsCurId += 1
> -
> - if type == "object":
> - result = "%s:obj:%d" % (namespace, objectsCurId)
> - objectsCurId += 1
> -
> - if type == "state":
> - result = "%s:ste:%d" % (namespace, statesCurId)
> - statesCurId += 1
> -
> - return (result)
> + """Generate new unique id for tests, objects or states
> +
> + Argument keqywords:
> + type -- type of generated id test | object | state
> +
> + return Generate id like <namespace>:tst|obj|ste:<id>
> + """
> + global testsCurId, objectsCurId, statesCurId
> +
> + if type == "test":
> + result = "%s:tst:%d" % (namespace, testsCurId)
> + testsCurId += 1
> +
> + if type == "object":
> + result = "%s:obj:%d" % (namespace, objectsCurId)
> + objectsCurId += 1
> +
> + if type == "state":
> + result = "%s:ste:%d" % (namespace, statesCurId)
> + statesCurId += 1
> +
> + return result
>
> def __createOVALDpkginfoObject (name):
> - """ Generate OVAL dpkginfo_object definition """
> -
> - if not testsHash["obj"].has_key(name):
> - objectId = __getNewId ("object");
> - object = __createXMLElement("dpkginfo_object",
> - attrs={"id":objectId,
> - "version":"1",
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
> - object.appendChild ( __createXMLElement ("name", name))
> - objects.appendChild (object)
> -
> - testsHash["obj"][name] = objectId
> -
> - return (testsHash["obj"][name])
> + """ Generate OVAL dpkginfo_object definition """
> +
> + if not testsHash["obj"].has_key(name):
> + objectId = __getNewId ("object");
> + object = __createXMLElement("dpkginfo_object",
> + attrs={"id":objectId,
> + "version":"1",
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
> + object.appendChild ( __createXMLElement ("name", name))
> + objects.appendChild (object)
> +
> + testsHash["obj"][name] = objectId
> +
> + return (testsHash["obj"][name])
>
> def __createOVALTextfilecontentObject (pattern, path = "/etc", filename =
> "debian_version"):
> - """ Generate OVAL textfilecontent_object definition """
> - name = path + filename + pattern
> -
> - if not testsHash["obj"].has_key(name):
> - objectId = __getNewId ("object");
> - object = __createXMLElement("textfilecontent_object",
> - attrs={"id":objectId,
> - "version":"1",
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
> - object.appendChild ( __createXMLElement ("path", path))
> - object.appendChild ( __createXMLElement ("filename", filename))
> - object.appendChild ( __createXMLElement ("line", pattern,
> attrs={"operation" : "pattern match"}))
> - objects.appendChild (object)
> -
> - testsHash["obj"][name] = objectId
> -
> - return (testsHash["obj"][name])
> + """ Generate OVAL textfilecontent_object definition """
> + name = path + filename + pattern
> +
> + if not testsHash["obj"].has_key(name):
> + objectId = __getNewId ("object");
> + object = __createXMLElement("textfilecontent_object",
> + attrs={"id":objectId,
> + "version":"1",
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
> + object.appendChild ( __createXMLElement ("path", path))
> + object.appendChild ( __createXMLElement ("filename", filename))
> + object.appendChild ( __createXMLElement ("line", pattern,
> attrs={"operation" : "pattern match"}))
> + objects.appendChild (object)
> +
> + testsHash["obj"][name] = objectId
> +
> + return (testsHash["obj"][name])
>
> def __createOVALUnameObject ():
> - """ Generate OVAL textfilecontent_object definition """
> - name = "uname_object"
> -
> - if not testsHash["obj"].has_key(name):
> - objectId = __getNewId ("object");
> - object = __createXMLElement("uname_object",
> - attrs={"id":objectId,
> - "version":"1",
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
> - objects.appendChild (object)
> -
> - testsHash["obj"][name] = objectId
> -
> - return (testsHash["obj"][name])
> + """ Generate OVAL textfilecontent_object definition """
> + name = "uname_object"
> +
> + if not testsHash["obj"].has_key(name):
> + objectId = __getNewId ("object");
> + object = __createXMLElement("uname_object",
> + attrs={"id":objectId,
> + "version":"1",
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
> + objects.appendChild (object)
> +
> + testsHash["obj"][name] = objectId
> +
> + return (testsHash["obj"][name])
>
> def __createOVALState (value, operation = "less than"):
> - """ Generate OVAL state definition
> -
> - Use state hash for optimization of resulted XML
> - """
> - #TODO: Add arch state generation
> - if not testsHash["dpkgSte"].has_key(operation) or not
> testsHash["dpkgSte"][operation].has_key(value):
> - stateId = __getNewId ("state")
> -
> - state = __createXMLElement("dpkginfo_state",
> - attrs={"id":stateId,
> - "version":"1",
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
> - state.appendChild ( __createXMLElement ("evr", "0:"+value,
> -
> {"datatype":"evr_string",
> -
> "operation":operation}))
> - states.appendChild (state)
> -
> - testsHash["dpkgSte"][operation] = {value : stateId}
> -
> - return (testsHash["dpkgSte"][operation][value])
> + """ Generate OVAL state definition
> +
> + Use state hash for optimization of resulted XML
> + """
> + #TODO: Add arch state generation
> + if not testsHash["dpkgSte"].has_key(operation) or not
> testsHash["dpkgSte"][operation].has_key(value):
> + stateId = __getNewId ("state")
> +
> + state = __createXMLElement("dpkginfo_state",
> + attrs={"id":stateId,
> + "version":"1",
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
> + state.appendChild ( __createXMLElement ("evr", "0:"+value,
> + {"datatype":"evr_string",
> + "operation":operation}))
> + states.appendChild (state)
> +
> + testsHash["dpkgSte"][operation] = {value : stateId}
> +
> + return (testsHash["dpkgSte"][operation][value])
>
> def __createOVALUnameState (field, value, operation = "equals"):
> - """ Generate OVAL uname state definition
> -
> - Use unameArchTable to convert dsa arch to uname arch value
> - """
> -
> - try:
> - value = unameArchTable[value]
> - except KeyError:
> - pass
> -
> - #TODO: Add arch state generation
> - if not testsHash["unameSte"].has_key(operation) or not
> testsHash["unameSte"][operation].has_key(value):
> - stateId = __getNewId ("state")
> -
> - state = __createXMLElement("uname_state",
> - attrs={"id":stateId,
> - "version":"1",
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
> - state.appendChild ( __createXMLElement (field, value,
> -
> {"operation":operation}))
> - states.appendChild (state)
> -
> - testsHash["unameSte"][operation] = {value : stateId}
> -
> - return (testsHash["unameSte"][operation][value])
> + """ Generate OVAL uname state definition
> +
> + Use unameArchTable to convert dsa arch to uname arch value
> + """
> +
> + try:
> + value = unameArchTable[value]
> + except KeyError:
> + pass
> +
> + #TODO: Add arch state generation
> + if not testsHash["unameSte"].has_key(operation) or not
> testsHash["unameSte"][operation].has_key(value):
> + stateId = __getNewId ("state")
> +
> + state = __createXMLElement("uname_state",
> + attrs={"id":stateId,
> + "version":"1",
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
> + state.appendChild ( __createXMLElement (field, value,
> + {"operation":operation}))
> + states.appendChild (state)
> +
> + testsHash["unameSte"][operation] = {value : stateId}
> +
> + return (testsHash["unameSte"][operation][value])
>
> def __createOVALTextfilecontentState (value, operation = "equals"):
> - """ Generate OVAL state definition
> -
> - Use state hash for optimization of resulted XML
> - """
> - #TODO: Add arch state generation
> - if not testsHash["fileSte"].has_key(operation) or not
> testsHash["fileSte"][operation].has_key(value):
> - stateId = __getNewId ("state")
> -
> - state = __createXMLElement("textfilecontent_state",
> - attrs={"id":stateId,
> - "version":"1",
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
> - state.appendChild ( __createXMLElement ("line", value,
> -
> {"operation":operation}))
> - states.appendChild (state)
> -
> - testsHash["fileSte"][operation] = {value : stateId}
> -
> - return (testsHash["fileSte"][operation][value])
> -
> + """ Generate OVAL state definition
> +
> + Use state hash for optimization of resulted XML
> + """
> + #TODO: Add arch state generation
> + if not testsHash["fileSte"].has_key(operation) or not
> testsHash["fileSte"][operation].has_key(value):
> + stateId = __getNewId ("state")
> +
> + state = __createXMLElement("textfilecontent_state",
> + attrs={"id":stateId,
> + "version":"1",
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
> + state.appendChild ( __createXMLElement ("line", value,
> + {"operation":operation}))
> + states.appendChild (state)
> +
> + testsHash["fileSte"][operation] = {value : stateId}
> +
> + return (testsHash["fileSte"][operation][value])
> +
> def __createDPKGTest(name, version):
> - """ Generate OVAL DPKG test """
> -
> - ref = __getNewId ("test")
> - test = __createXMLElement("dpkginfo_test",
> - attrs={"id":ref,
> - "version":"1",
> - "check":"all",
> - "check_existence":"at_least_one_exists",
> - "comment":"%s is earlier than %s" % (name,
> version),
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";
> - })
> - test.appendChild ( __createXMLElement("object", attrs={"object_ref" :
> __createOVALDpkginfoObject (name)}))
> - test.appendChild ( __createXMLElement("state", attrs={"state_ref" :
> __createOVALState (version)}))
> - tests.appendChild(test)
> + """ Generate OVAL DPKG test """
> +
> + ref = __getNewId ("test")
> + test = __createXMLElement("dpkginfo_test",
> + attrs={"id":ref,
> + "version":"1",
> + "check":"all",
> + "check_existence":"at_least_one_exists",
> + "comment":"%s is earlier than %s" % (name, version),
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";
> + })
> + test.appendChild ( __createXMLElement("object", attrs={"object_ref" :
> __createOVALDpkginfoObject (name)}))
> + test.appendChild ( __createXMLElement("state", attrs={"state_ref" :
> __createOVALState (version)}))
> + tests.appendChild(test)
> +
> + return (ref)
>
> - return (ref)
> -
> def __createTest(testType, value):
> - """ Generate OVAL test for release or architecture cases"""
> -
> - if not testsHash[testType].has_key(value):
> - comment = None
> -
> - ref = __getNewId("test")
> -
> - if testType == "release":
> - objectId = __createOVALTextfilecontentObject ("\d\.\d")
> - comment = "Debian GNU/Linux %s is installed" % value
> -
> - test = __createXMLElement("textfilecontent_test",
> - attrs={"id":ref,
> - "version":"1",
> - "check":"all",
> - "check_existence":"at_least_one_exists",
> - "comment":comment,
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";
> - })
> - test.appendChild ( __createXMLElement("object",
> attrs={"object_ref" : objectId}))
> - test.appendChild ( __createXMLElement("state",
> attrs={"state_ref" : __createOVALTextfilecontentState (value, "equals")}))
> -
> - else:
> - objectId = __createOVALUnameObject ()
> - comment = "Installed architecture is %s" % value
> -
> - test = __createXMLElement("uname_test",
> - attrs={"id":ref,
> - "version":"1",
> - "check":"all",
> - "check_existence":"at_least_one_exists",
> - "comment":comment,
> -
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";
> - })
> - test.appendChild ( __createXMLElement("object",
> attrs={"object_ref" : objectId}))
> - if value != "all":
> - test.appendChild ( __createXMLElement("state",
> attrs={"state_ref" : __createOVALUnameState ("processor_type", value,
> "equals")}))
> -
> - tests.appendChild(test)
> -
> - testsHash[testType][value] = ref
> -
> - return (testsHash[testType][value])
> + """ Generate OVAL test for release or architecture cases"""
> +
> + if not testsHash[testType].has_key(value):
> + comment = None
> +
> + ref = __getNewId("test")
> +
> + if testType == "release":
> + objectId = __createOVALTextfilecontentObject ("\d\.\d")
> + comment = "Debian GNU/Linux %s is installed" % value
> +
> + test = __createXMLElement("textfilecontent_test",
> + attrs={"id":ref,
> + "version":"1",
> + "check":"all",
> + "check_existence":"at_least_one_exists",
> + "comment":comment,
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";
> + })
> + test.appendChild ( __createXMLElement("object",
> attrs={"object_ref" : objectId}))
> + test.appendChild ( __createXMLElement("state",
> attrs={"state_ref" : __createOVALTextfilecontentState (value, "equals")}))
> +
> + else:
> + objectId = __createOVALUnameObject ()
> + comment = "Installed architecture is %s" % value
> +
> + test = __createXMLElement("uname_test",
> + attrs={"id":ref,
> + "version":"1",
> + "check":"all",
> + "check_existence":"at_least_one_exists",
> + "comment":comment,
> +
> "xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";
> + })
> + test.appendChild ( __createXMLElement("object",
> attrs={"object_ref" : objectId}))
> + if value != "all":
> + test.appendChild ( __createXMLElement("state",
> attrs={"state_ref" : __createOVALUnameState ("processor_type", value,
> "equals")}))
> +
> + tests.appendChild(test)
> +
> + testsHash[testType][value] = ref
> +
> + return (testsHash[testType][value])
>
> def __createGeneratorHeader ():
> - """
> - Create OVAL definitions XML generator element.
> -
> - return xml.dom.minidom.Document with header information
> - """
> -
> - doc = xml.dom.minidom.Document ()
> - generator = doc.createElement ("generator")
> -
> - generator.appendChild ( __createXMLElement ("oval:product_name",
> "Debian") )
> - generator.appendChild ( __createXMLElement ("oval:schema_version",
> "5.3") )
> - generator.appendChild ( __createXMLElement ("oval:timestamp",
> datetime.datetime.now().strftime ("%Y-%m-%dT%H:%M:%S.188-04:00")) )
> + """
> + Create OVAL definitions XML generator element.
> +
> + return xml.dom.minidom.Document with header information
> + """
> +
> + doc = xml.dom.minidom.Document ()
> + generator = doc.createElement ("generator")
> +
> + generator.appendChild ( __createXMLElement ("oval:product_name",
> "Debian") )
> + generator.appendChild ( __createXMLElement ("oval:schema_version",
> "5.3") )
> + generator.appendChild ( __createXMLElement ("oval:timestamp",
> datetime.datetime.now().strftime ("%Y-%m-%dT%H:%M:%S.188-04:00")) )
>
> - return (generator)
> + return (generator)
>
> def createPlatformDefinition (release, data, dsa):
> - """ Generate OVAL definitions for current release
> -
> - Generate full criteria tree for specified release. Tests,
> states and objects
> - stored in global dictionaries.
> - Use differ module for otimize generated tree.
> -
> - Argument keywords:
> - release -- Debian release
> - data -- dict with information about packages
> - dsa - DSA id
> -
> - return Generated XML fragment
> - """
> - #Raise exception if we receive too small data
> - if len(data) == 0:
> - logging.log(logging.WARNING, "DSA %s: Information of affected
> platforms is not available." % dsa)
> -
> - softwareCriteria = __createXMLElement ("criteria", attrs = {"comment" :
> "Release section", "operator" : "AND"})
> - softwareCriteria.appendChild ( __createXMLElement ("criterion",
> attrs={"test_ref" : __createTest("release", release), "comment" : "Debian %s
> is installed" % release}))
> -
> - archCriteria = __createXMLElement ("criteria", attrs = {"comment" :
> "Architecture section", "operator" : "OR"})
> -
> - # Handle architecture independed section
> - if data.has_key ("all"):
> - archIndepCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Architecture independet section", "operator" : "AND"})
> -
> - archIndepCriteria.appendChild ( __createXMLElement
> ("criterion", attrs = {"test_ref" : __createTest("arch", "all"), "comment" :
> "all architecture"}))
> - #Build packages section only if we have more then one package
> - if len (data["all"]) > 1:
> - packageCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Packages section", "operator" : "OR"})
> - archIndepCriteria.appendChild (packageCriteria)
> - else:
> - packageCriteria = archIndepCriteria
> -
> - for pkg in data["all"].keys():
> - packageCriteria.appendChild ( __createXMLElement
> ("criterion", attrs = {"test_ref" : __createDPKGTest(pkg, data["all"][pkg]),
> "comment" : "%s DPKG is earlier than %s" % (pkg, data["all"][pkg])}))
> -
> - archCriteria.appendChild (archIndepCriteria)
> -
> - # Optimize packages tree in 2 stages
> - diff = differ ()
> - for i in range(2):
> -
> - if i == 0:
> - dsaData = data
> - else:
> - dsaData = diff.getDiffer()
> -
> - diff.Clean()
> - for (key, value) in dsaData.iteritems():
> - if key != "all":
> - diff.compareElement(key, value)
> -
> - eq = diff.getEqual()
> - di = diff.getDiffer()
> -
> - # Generate XML for optimized packages
> - if (len(eq)):
> - if len(diff.getArchs()) != releaseArchHash[release]:
> - archDependCriteria = __createXMLElement
> ("criteria", attrs={"comment" : "Architecture depended section", "operator" :
> "AND"})
> -
> - supportedArchCriteria = __createXMLElement
> ("criteria", attrs={"comment" : "Supported architectures section", "operator"
> : "OR"})
> - for arch in diff.getArchs():
> - supportedArchCriteria.appendChild (
> __createXMLElement ("criterion", attrs = {"test_ref" : __createTest("arch",
> arch), "comment" : "%s architecture" % arch}))
> - archDependCriteria.appendChild
> (supportedArchCriteria)
> -
> - packageCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Packages section", "operator" : "OR"})
> - for bpkg in eq.keys():
> - packageCriteria.appendChild (
> __createXMLElement ("criterion", attrs = {"test_ref" : __createDPKGTest(bpkg,
> eq[bpkg]), "comment" : "%s DPKG is earlier than %s" % (bpkg, eq[bpkg])}))
> -
> - if len(diff.getArchs()) != releaseArchHash[release]:
>
> - archDependCriteria.appendChild (packageCriteria)
> - archCriteria.appendChild (archDependCriteria)
> - else:
> - archCriteria.appendChild (packageCriteria)
> -
> - # Generate XML for all other packages
> - if len(di):
> - archDependCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Architecture depended section", "operator" : "AND"})
> -
> - for (key, value) in di.iteritems():
> - supportedPlatformCriteria = __createXMLElement
> ("criteria", attrs={"comment" : "Supported platform section", "operator" :
> "AND"})
> - supportedPlatformCriteria.appendChild (
> __createXMLElement ("criterion", attrs = {"test_ref" : __createTest("arch",
> key), "comment" : "%s architecture" % key}))
> -
> - packageCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Packages section", "operator" : "OR"})
> -
> - for bpkg in di[key].keys():
> - packageCriteria.appendChild (
> __createXMLElement ("criterion", attrs = {"test_ref" : __createDPKGTest(bpkg,
> di[key][bpkg]), "comment" : "%s DPKG is earlier than %s" % (bpkg,
> di[key][bpkg])}))
> - supportedPlatformCriteria.appendChild
> (packageCriteria)
> -
> - archDependCriteria.appendChild (supportedPlatformCriteria)
> - archCriteria.appendChild (archDependCriteria)
> -
> - softwareCriteria.appendChild (archCriteria)
> -
> - return (softwareCriteria)
> + """ Generate OVAL definitions for current release
> +
> + Generate full criteria tree for specified release. Tests, states and
> objects
> + stored in global dictionaries.
> + Use differ module for optimize generated tree.
> +
> + Argument keywords:
> + release -- Debian release
> + data -- dict with information about packages
> + dsa - DSA id
> +
> + return Generated XML fragment
> + """
> + #Raise exception if we receive too small data
> + if len(data) == 0:
> + logging.log(logging.WARNING, "DSA %s: Information of affected
> platforms is not available." % dsa)
> +
> + softwareCriteria = __createXMLElement ("criteria", attrs = {"comment" :
> "Release section", "operator" : "AND"})
> + softwareCriteria.appendChild ( __createXMLElement ("criterion",
> attrs={"test_ref" : __createTest("release", release), "comment" : "Debian %s
> is installed" % release}))
> +
> + archCriteria = __createXMLElement ("criteria", attrs = {"comment" :
> "Architecture section", "operator" : "OR"})
> +
> + # Handle architecture independent section
> + if data.has_key ("all"):
> + archIndepCriteria = __createXMLElement ("criteria", attrs={"comment"
> : "Architecture independent section", "operator" : "AND"})
> +
> + archIndepCriteria.appendChild ( __createXMLElement ("criterion",
> attrs = {"test_ref" : __createTest("arch", "all"), "comment" : "all
> architecture"}))
> + #Build packages section only if we have more then one package
> + if len (data["all"]) > 1:
> + packageCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Packages section", "operator" : "OR"})
> + archIndepCriteria.appendChild (packageCriteria)
> + else:
> + packageCriteria = archIndepCriteria
> +
> + for pkg in data["all"].keys():
> + packageCriteria.appendChild ( __createXMLElement ("criterion",
> attrs = {"test_ref" : __createDPKGTest(pkg, data["all"][pkg]), "comment" :
> "%s DPKG is earlier than %s" % (pkg, data["all"][pkg])}))
> +
> + archCriteria.appendChild (archIndepCriteria)
> +
> + # Optimize packages tree in 2 stages
> + diff = differ()
> + for i in range(2):
> +
> + if i == 0:
> + dsaData = data
> + else:
> + dsaData = diff.getDiffer()
> +
> + diff.Clean()
> + for (key, value) in dsaData.iteritems():
> + if key != "all":
> + diff.compareElement(key, value)
> +
> + eq = diff.getEqual()
> + di = diff.getDiffer()
> +
> + # Generate XML for optimized packages
> + if (len(eq)):
> + if len(diff.getArchs()) != releaseArchHash[release]:
> + archDependCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Architecture depended section", "operator" : "AND"})
> +
> + supportedArchCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Supported architectures section", "operator" : "OR"})
> + for arch in diff.getArchs():
> + supportedArchCriteria.appendChild ( __createXMLElement
> ("criterion", attrs = {"test_ref" : __createTest("arch", arch), "comment" :
> "%s architecture" % arch}))
> + archDependCriteria.appendChild (supportedArchCriteria)
> +
> + packageCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Packages section", "operator" : "OR"})
> + for bpkg in eq.keys():
> + packageCriteria.appendChild ( __createXMLElement
> ("criterion", attrs = {"test_ref" : __createDPKGTest(bpkg, eq[bpkg]),
> "comment" : "%s DPKG is earlier than %s" % (bpkg, eq[bpkg])}))
> +
> + if len(diff.getArchs()) != releaseArchHash[release]:
> + archDependCriteria.appendChild (packageCriteria)
> + archCriteria.appendChild (archDependCriteria)
> + else:
> + archCriteria.appendChild (packageCriteria)
> +
> + # Generate XML for all other packages
> + if len(di):
> + archDependCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Architecture depended section", "operator" : "AND"})
> +
> + for (key, value) in di.iteritems():
> + supportedPlatformCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Supported platform section", "operator" : "AND"})
> + supportedPlatformCriteria.appendChild ( __createXMLElement
> ("criterion", attrs = {"test_ref" : __createTest("arch", key), "comment" :
> "%s architecture" % key}))
> +
> + packageCriteria = __createXMLElement ("criteria",
> attrs={"comment" : "Packages section", "operator" : "OR"})
> +
> + for bpkg in di[key].keys():
> + packageCriteria.appendChild ( __createXMLElement
> ("criterion", attrs = {"test_ref" : __createDPKGTest(bpkg, di[key][bpkg]),
> "comment" : "%s DPKG is earlier than %s" % (bpkg, di[key][bpkg])}))
> + supportedPlatformCriteria.appendChild (packageCriteria)
> +
> + archDependCriteria.appendChild (supportedPlatformCriteria)
> + archCriteria.appendChild (archDependCriteria)
> +
> + softwareCriteria.appendChild (archCriteria)
> +
> + return (softwareCriteria)
>
> def createDefinition (dsa, dsaref):
> - """ Generate OVAL header of Definition tag
> -
> - Print general informaton about OVAL definition. Use
> createPlatformDefinition for generate criteria
> - sections for each affected release.
> -
> - Argument keywords:
> - dsa -- DSA dentificator
> - dsaref -- DSA parsed data
> - """
> - if not dsaref.has_key("release"):
> - logging.log(logging.WARNING, "DSA %s: Release definition not
> well formatted. Ignoring this DSA." % dsa)
> - raise DSAFormatException
> -
> - if not dsaref.has_key("packages"):
> - logging.log(logging.WARNING, "DSA %s: Package information
> missed. Ignoring this DSA." % dsa)
> - dsaref["packages"] = ""
> -
> - if not dsaref.has_key("description"):
> - logging.log(logging.WARNING, "DSA %s: Description information
> missed." % dsa)
> - dsaref["description"] = ""
> -
> - if not dsaref.has_key("moreinfo"):
> - logging.log(logging.WARNING, "DSA %s: Moreinfo information
> missed." % dsa)
> - dsaref["moreinfo"] = ""
> -
> - if not dsaref.has_key("secrefs"):
> - logging.log(logging.WARNING, "DSA %s: Secrefs information
> missed." % dsa)
> - dsaref["secrefs"] = ""
> -
> - doc = xml.dom.minidom.Document ()
> -
> - ### Definition block: Metadata, Notes, Criteria
> - ### TODO: Replace DSA id with unique id
> - definition = __createXMLElement ("definition", attrs = {"id" :
> "oval:org.debian:def:%s" % __trimzero(dsa), "version" : "1", "class" :
> "vulnerability"})
> -
> - ### Definition : Metadata : title, affected, reference, description ###
> - metadata = __createXMLElement ("metadata")
> - metadata.appendChild (__createXMLElement ("title",
> dsaref["description"]))
> -
> - ### Definition : Metadata : Affected : platform, product ###
> - affected = __createXMLElement ("affected", attrs = {"family" : "unix"})
> - for platform in dsaref["release"]:
> - affected.appendChild ( __createXMLElement ("platform", "Debian
> GNU/Linux %s" % platform))
> - affected.appendChild ( __createXMLElement ("product",
> dsaref.get("packages")))
> -
> - metadata.appendChild (affected)
> - ### Definition : Metadata : Affected : END ###
> -
> - refpatern = re.compile (r'((CVE|CAN)-[\d-]+)')
> - for ref in dsaref.get("secrefs").split(" "):
> - result = refpatern.search(ref)
> - if result:
> - (ref_id, source) = result.groups()
> - metadata.appendChild ( __createXMLElement ("reference",
> attrs = {"source" : source, "ref_id" : ref_id, "ref_url" :
> "http://cve.mitre.org/cgi-bin/cvename.cgi?name=%s"; % ref_id}) )
> -
> - #TODO: move this info to other place
> - metadata.appendChild ( __createXMLElement ("description", "What
> information can i put there?"))
> - debianMetadata = __createXMLElement ("debian")
> - if dsaref.has_key("date"):
> - debianMetadata.appendChild ( __createXMLElement ("date",
> dsaref["date"]) )
> - debianMetadata.appendChild ( __createXMLElement ("moreinfo",
> dsaref["moreinfo"]) )
> - metadata.appendChild (debianMetadata)
> - definition.appendChild ( metadata )
> -
> - ### Definition : Criteria ###
> - if len(dsaref["release"]) > 1:
> - #f we have more than one release - generate additional criteria
> section
> - platformCriteria = __createXMLElement ("criteria", attrs =
> {"comment" : "Platform section", "operator" : "OR"})
> - definition.appendChild (platformCriteria)
> - else:
> - platformCriteria = definition
> -
> - for platform in dsaref["release"]:
> - data = dsaref["release"][platform]
> - platformCriteria.appendChild
> (createPlatformDefinition(platform, data, dsa))
> -
> - ### Definition : Criteria END ###
> -
> - return (definition)
> -
> -def createOVALDefinitions (dsaref):
> - """ Generate XML OVAL definition tree for range of DSA
> -
> - Generate namespace section and use other functions to generate
> definitions,
> - tests, objects and states subsections.
> -
> - return -- Generated OVAL XML definition
> - """
> - doc = xml.dom.minidom.Document ()
> -
> - root = __createXMLElement ("oval_definitions",
> - attrs= {
> - "xsi:schemaLocation" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#independent
> independent-definitions-schema.xsd
> http://oval.mitre.org/XMLSchema/oval-definitions-5#linux
> linux-definitions-schema.xsd
> http://oval.mitre.org/XMLSchema/oval-definitions-5#unix
> unix-definitions-schema.xsd
> http://oval.mitre.org/XMLSchema/oval-definitions-5
> oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5
> oval-common-schema.xsd",
> - "xmlns:xsi" :
> "http://www.w3.org/2001/XMLSchema-instance";,
> - "xmlns:ind-def " :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";,
> - "xmlns:linux-def" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";,
> - "xmlns:oval-def" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5";,
> - "xmlns:unix-def" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";,
> - "xmlns" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5";,
> - "xmlns:oval" :
> "http://oval.mitre.org/XMLSchema/oval-common-5";
> - }
> - )
> - doc.appendChild (root)
> - root.appendChild ( __createGeneratorHeader () )
> -
> - definitions = doc.createElement ("definitions")
> -
> - keyids = dsaref.keys()
> - keyids.sort()
> - for dsa in keyids:
> - try:
> - definitions.appendChild (createDefinition(dsa,
> dsaref[dsa]))
> - except DSAFormatException:
> - logging.log (logging.WARNING, "DSA %s: Bad data file.
> Ignoring this DSA." % dsa)
> -
> - root.appendChild (definitions)
> -
> - root.appendChild(tests)
> - root.appendChild(objects)
> - root.appendChild(states)
> + """ Generate OVAL header of Definition tag
> +
> + Print general information about OVAL definition. Use
> createPlatformDefinition for generate criteria
> + sections for each affected release.
> +
> + Argument keywords:
> + dsa -- DSA identifier
> + dsaref -- DSA parsed data
> + """
> +
> + if not dsaref.has_key("release"):
> + logging.log(logging.WARNING, "DSA %s: Release definition not well
> formatted. Ignoring this DSA." % dsa)
> + raise DSAFormatException
> +
> + if not dsaref.has_key("packages"):
> + logging.log(logging.WARNING, "DSA %s: Package information missed.
> Ignoring this DSA." % dsa)
> + dsaref["packages"] = ""
> +
> + if not dsaref.has_key("description"):
> + logging.log(logging.WARNING, "DSA %s: Description information
> missed." % dsa)
> + dsaref["description"] = ""
> +
> + if not dsaref.has_key("moreinfo"):
> + logging.log(logging.WARNING, "DSA %s: Moreinfo information missed."
> % dsa)
> + dsaref["moreinfo"] = ""
> +
> + if not dsaref.has_key("secrefs"):
> + logging.log(logging.WARNING, "DSA %s: Secrefs information missed." %
> dsa)
> + dsaref["secrefs"] = ""
> +
> + doc = xml.dom.minidom.Document ()
> +
> + ### Definition block: Metadata, Notes, Criteria
> + ### TODO: Replace DSA id with unique id
> + definition = __createXMLElement ("definition", attrs = {"id" :
> "oval:org.debian:def:%s" % __trimzero(dsa), "version" : "1", "class" :
> "vulnerability"})
> +
> + ### Definition : Metadata : title, affected, reference, description ###
> + metadata = __createXMLElement ("metadata")
> + metadata.appendChild (__createXMLElement ("title",
> dsaref["description"]))
> +
> + ### Definition : Metadata : Affected : platform, product ###
> + affected = __createXMLElement ("affected", attrs = {"family" : "unix"})
> + for platform in dsaref["release"]:
> + affected.appendChild ( __createXMLElement ("platform", "Debian
> GNU/Linux %s" % platform))
> + affected.appendChild ( __createXMLElement ("product",
> dsaref.get("packages")))
> +
> + metadata.appendChild (affected)
> + ### Definition : Metadata : Affected : END ###
> +
> + refpatern = re.compile (r'((CVE|CAN)-[\d-]+)')
> + for ref in dsaref.get("secrefs").split(" "):
> + result = refpatern.search(ref)
> + if result:
> + (ref_id, source) = result.groups()
> + metadata.appendChild ( __createXMLElement ("reference", attrs =
> {"source" : source, "ref_id" : ref_id, "ref_url" :
> "http://cve.mitre.org/cgi-bin/cvename.cgi?name=%s"; % ref_id}) )
> +
> + #TODO: move this info to other place
> + metadata.appendChild ( __createXMLElement ("description", "What
> information can i put there?"))
> + debianMetadata = __createXMLElement ("debian")
> + if dsaref.has_key("date"):
> + debianMetadata.appendChild ( __createXMLElement ("date",
> dsaref["date"]) )
> + debianMetadata.appendChild ( __createXMLElement ("moreinfo",
> dsaref["moreinfo"]) )
> + metadata.appendChild (debianMetadata)
> + definition.appendChild ( metadata )
> +
> + ### Definition : Criteria ###
> + if len(dsaref["release"]) > 1:
> + #f we have more than one release - generate additional criteria
> section
> + platformCriteria = __createXMLElement ("criteria", attrs =
> {"comment" : "Platform section", "operator" : "OR"})
> + definition.appendChild (platformCriteria)
> + else:
> + platformCriteria = definition
> +
> + for platform in dsaref["release"]:
> + data = dsaref["release"][platform]
> + platformCriteria.appendChild (createPlatformDefinition(platform,
> data, dsa))
> +
> + ### Definition : Criteria END ###
> +
> + return (definition)
> +
> +def createOVALDefinitions(dsaref):
> + """ Generate XML OVAL definition tree for range of DSA
> +
> + Generate namespace section and use other functions to generate
> definitions,
> + tests, objects and states subsections.
> +
> + return -- Generated OVAL XML definition
> + """
> + doc = xml.dom.minidom.Document ()
> +
> + root = __createXMLElement ("oval_definitions",
> + attrs= {
> + "xsi:schemaLocation" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#independent
> independent-definitions-schema.xsd
> http://oval.mitre.org/XMLSchema/oval-definitions-5#linux
> linux-definitions-schema.xsd
> http://oval.mitre.org/XMLSchema/oval-definitions-5#unix
> unix-definitions-schema.xsd
> http://oval.mitre.org/XMLSchema/oval-definitions-5
> oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5
> oval-common-schema.xsd",
> + "xmlns:xsi" :
> "http://www.w3.org/2001/XMLSchema-instance";,
> + "xmlns:ind-def " :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";,
> + "xmlns:linux-def" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";,
> + "xmlns:oval-def" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5";,
> + "xmlns:unix-def" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";,
> + "xmlns" :
> "http://oval.mitre.org/XMLSchema/oval-definitions-5";,
> + "xmlns:oval" :
> "http://oval.mitre.org/XMLSchema/oval-common-5";
> + }
> + )
> + doc.appendChild (root)
> + root.appendChild ( __createGeneratorHeader () )
> +
> + definitions = doc.createElement ("definitions")
> +
> + keyids = dsaref.keys()
> + keyids.sort()
> + for dsa in keyids:
> + try:
> + definitions.appendChild (createDefinition(dsa, dsaref[dsa]))
> + except DSAFormatException:
> + logging.log (logging.WARNING, "DSA %s: Bad data file. Ignoring
> this DSA." % dsa)
> +
> + root.appendChild (definitions)
> +
> + root.appendChild(tests)
> + root.appendChild(objects)
> + root.appendChild(states)
>
> - return doc
> + return doc
>
> def printOVALDefinitions (doc):
> - if doc.getElementsByTagName("definitions")[0].hasChildNodes():
> - print doc.toprettyxml()
> + if doc.getElementsByTagName("definitions")[0].hasChildNodes():
> + print doc.toprettyxml()
> Binary files oval_old/oval/definition/generator.pyc and
> oval/oval/definition/generator.pyc differ
> Binary files oval_old/oval/definition/__init__.pyc and
> oval/oval/definition/__init__.pyc differ
> Binary files oval_old/oval/__init__.pyc and oval/oval/__init__.pyc differ
> diff -urN oval_old/oval/parser/dsa.py oval/oval/parser/dsa.py
> --- oval_old/oval/parser/dsa.py 2011-10-11 17:41:19.000000000 -0600
> +++ oval/oval/parser/dsa.py 2015-11-13 09:34:04.843882000 -0700
> @@ -9,6 +9,7 @@
> # <isvulnerable>
> # <fixed>
> #
> +# (c) 2015 Nicholas Luedtke
> # (c) 2007 Pavel Vinogradov
> # (c) 2004 Javier Fernandez-Sanguino
>
> # Licensed under the GNU General Public License version 2.
> @@ -17,105 +18,113 @@
> import os
> import logging
>
> +
> # Format of data files is:
> -#<define-tag pagetitle>DSA-###-# PACKAGE</define-tag>
>
> -#<define-tag report_date>yyyy-mm-dd</define-tag>
>
> -#<define-tag secrefs>CAN|CVE-XXXX-XXXX</define-tag>
>
> -#<define-tag packages>PACKAGE</define-tag>
>
> -#<define-tag isvulnerable>yes|no</define-tag>
>
> -#<define-tag fixed>yes|no</define-tag>
> -def parseFile (path):
> - """ Parse data file with information of Debian Security Advisories
> +# <define-tag pagetitle>DSA-###-# PACKAGE</define-tag>
> +# <define-tag report_date>yyyy-mm-dd</define-tag>
> +# <define-tag secrefs>CAN|CVE-XXXX-XXXX</define-tag>
> +# <define-tag packages>PACKAGE</define-tag>
> +# <define-tag isvulnerable>yes|no</define-tag>
> +# <define-tag fixed>yes|no</define-tag>
> +def parseFile(path):
> + """ Parse data file with information of Debian Security Advisories
>
> Keyword arguments:
> path -- full path to data file
>
> return list (dsa id, tags and packages data)"""
> -
> -
> - data = {}
> - deb_ver = None
> - fdeb_ver = None
> -
> - filename = os.path.basename (path)
>
> - patern = re.compile(r'dsa-(\d+)')
> - result = patern.search(filename)
> - if result:
> - dsa = result.groups()[0]
> - else:
> - logging.log(logging.WARNING, "File %s does not look like a
> proper DSA, not checking" % filename)
> - return (None)
> + data = {}
> + deb_ver = None
> + fdeb_ver = None
> +
> + filename = os.path.basename(path)
> +
> + patern = re.compile(r'dsa-(\d+)')
> + result = patern.search(filename)
> +
> + if result:
> + dsa = result.groups()[0]
> +
> + else:
> + logging.log(logging.WARNING,
> + "File %s does not look like a proper DSA, not checking"
> % filename)
> + return (None)
> +
> + logging.log(logging.DEBUG, "Parsing DSA %s from file %s" % (dsa,
> filename))
> +
> + dsaFile = open(path)
> +
> + for line in dsaFile:
> + line = line.decode("ISO-8859-2")
> + datepatern = re.compile(r'report_date>([\d-]+)</define-tag>')
> + result = datepatern.search(line)
> + if result:
> + date = result.groups()[0]
> + normDate = lambda (date): "-".join(
> + [(len(p) > 1 and p or "0" + p) for p in date.split("-")])
> + data["date"] = normDate(date)
> +
> + refspatern = re.compile(r'secrefs>(.*?)</define-tag>')
> + result = refspatern.search(line)
> + if result:
> + data["secrefs"] = result.groups()[0]
> + logging.log(logging.DEBUG,
> + "Extracted security references: " + data["secrefs"])
> +
> + pakpatern = re.compile(r'packages>(.*?)</define-tag>')
> + result = pakpatern.search(line)
> + if result:
> + data["packages"] = result.groups()[0]
> +
> + vulpatern = re.compile(r'isvulnerable>(.*?)</define-tag>')
> + result = vulpatern.search(line)
> + if result:
> + data["vulnarable"] = result.groups()[0]
> +
> + fixpatern = re.compile(r'fixed>(.*?)</define-tag>')
> + result = fixpatern.search(line)
> + if result:
> + data["fixed"] = result.groups()[0]
> +
> + versionpatern = re.compile(
> + r'<h3>Debian GNU/Linux (\d.\d) \((.*?)\)</h3>')
> + result = versionpatern.search(line)
> + if result:
> + fdeb_ver = result.groups()[0]
> +
> + # Alternative format for data files
> + versionpatern = re.compile(r'affected_release>([\d\.]+)<')
> + result = versionpatern.search(line)
> + if result:
> + fdeb_ver = result.groups()[0]
> +
> + if fdeb_ver:
> + deb_ver = fdeb_ver
> + fdeb_ver = None
> + if data.has_key("release"):
> + if data["release"].has_key(deb_ver):
> + logging.log(logging.WARNING,
> + "DSA %s: Found second files section for
> release %s" % (
> + dsa, deb_ver))
> + else:
> + data["release"][deb_ver] = {}
> + else:
> + data["release"] = {deb_ver: {}}
> +
> + # Binary packages are pushed into array
> + # Those are prepended by fileurls
> + # TODO: Packages do _NOT_ include epochs
> + # (that should be fixed)
> + if data.has_key("release") and deb_ver:
> + urlpatern = re.compile(r'fileurl
> [\w:/.\-+]+/([\w\-.+~]+)\.deb[^i]')
> + result = urlpatern.search(line)
> + if result:
> + (package, version, architecture) =
> result.groups()[0].split("_")
> +
> + if data["release"][deb_ver].has_key(architecture):
> + data["release"][deb_ver][architecture][package] = version
> + else:
> + data["release"][deb_ver][architecture] = {package:
> version}
>
> - logging.log (logging.DEBUG, "Parsing DSA %s from file %s" % (dsa,
> filename))
> -
> - dsaFile = open(path)
> -
> - for line in dsaFile:
> - line= line.decode ("ISO-8859-2")
> - datepatern = re.compile (r'report_date>([\d-]+)</define-tag>')
> - result = datepatern.search (line)
> - if result:
> - date = result.groups()[0]
> - normDate = lambda (date): "-".join([(len(p) > 1 and p
> or "0"+p) for p in date.split("-")])
> - data["date"] = normDate(date)
> -
> - refspatern = re.compile (r'secrefs>(.*?)</define-tag>')
> - result = refspatern.search (line)
> - if result:
> - data["secrefs"] = result.groups()[0]
> - logging.log(logging.DEBUG, "Extracted security
> references: " + data["secrefs"])
> -
> - pakpatern = re.compile (r'packages>(.*?)</define-tag>')
> - result = pakpatern.search (line)
> - if result:
> - data["packages"] = result.groups()[0]
> -
> - vulpatern = re.compile (r'isvulnerable>(.*?)</define-tag>')
> - result = vulpatern.search (line)
> - if result:
> - data["vulnarable"] = result.groups()[0]
> -
> - fixpatern = re.compile (r'fixed>(.*?)</define-tag>')
> - result = fixpatern.search (line)
> - if result:
> - data["fixed"] = result.groups()[0]
> -
> - versionpatern = re.compile (r'<h3>Debian GNU/Linux (\d.\d)
> \((.*?)\)</h3>')
> - result = versionpatern.search (line)
> - if result:
> - fdeb_ver = result.groups()[0]
> -
> - # Alternative format for data files
> - versionpatern = re.compile (r'affected_release>([\d\.]+)<')
> - result = versionpatern.search (line)
> - if result:
> - fdeb_ver = result.groups()[0]
> -
> - if fdeb_ver:
> - deb_ver = fdeb_ver
> - fdeb_ver = None
> - if data.has_key("release"):
> - if data["release"].has_key(deb_ver):
> - logging.log(logging.WARNING, "DSA %s:
> Found second files section for release %s" % (dsa, deb_ver))
> - else:
> - data["release"][deb_ver] = {}
> - else:
> - data["release"] = {deb_ver: {}}
> -
> - # Binary packages are pushed into array
> - # Those are prepended by fileurls
> - # TODO: Packages do _NOT_ include epochs
> - # (that should be fixed)
> - if data.has_key("release") and deb_ver:
> - urlpatern = re.compile (r'fileurl
> [\w:/.\-+]+/([\w\-.+~]+)\.deb[^i]')
> - result = urlpatern.search (line)
> - if result:
> - (package, version, architecture) =
> result.groups()[0].split("_")
> -
> - if
> data["release"][deb_ver].has_key(architecture):
> -
> data["release"][deb_ver][architecture][package] = version
> - else:
> - data["release"][deb_ver][architecture]
> = {package : version}
> -
> - return (dsa, data)
> + return (dsa, data)
> Binary files oval_old/oval/parser/dsa.pyc and oval/oval/parser/dsa.pyc differ
> Binary files oval_old/oval/parser/__init__.pyc and
> oval/oval/parser/__init__.pyc differ
> diff -urN oval_old/oval/parser/wml.py oval/oval/parser/wml.py
> --- oval_old/oval/parser/wml.py 2013-01-21 00:48:41.000000000 -0700
> +++ oval/oval/parser/wml.py 2015-11-06 08:50:30.632363000 -0700
> @@ -1,13 +1,14 @@
> # -*- coding: utf-8 -*-
> -# oval.parser.wml - module to parse descriptions of
> +# oval.parser.wml - module to parse descriptions of
> # Debian Security Advisories stored in wml format.
> # Extrected tags:
> # <description>
> # <moreinfo>- Paragraphs before descriptions of
> # each release status
> #
> -# (c) 2007 Pavel Vinogradov
> -# (c) 2004 Javier Fernandez-Sanguino
>
> +# (c) 2015 Nicholas Luedtke
> +# (c) 2007 Pavel Vinogradov
> +# (c) 2004 Javier Fernandez-Sanguino
> # Licensed under the GNU General Public License version 2.
>
> import re
> @@ -15,76 +16,111 @@
> import sys
> import logging
>
> +DEBIAN_VERSION = {"wheezy" : "7.0", "jessie" : "8.2", "stretch" : "9.0",
> + "sid" : "9.0", "etch" : "4.0", "squeeze":"6.0",
> "lenny":"5.0"}
> +
> # Format of wml files is:
> #<define-tag description>DESCRIPTION</define-tag>
> #<define-tag moreinfo>Multiline information</define-tag>
> def parseFile (path):
> - """ Parse wml file with description of Debian Security Advisories
> -
> - Keyword arguments:
> - path -- full path to wml file
> -
> - return list (dsa id, tags data)"""
> -
> - data = {}
> - moreinfo = False
> -
> - filename = os.path.basename (path)
> -
> - patern = re.compile(r'dsa-(\d+)')
> - result = patern.search(filename)
> - if result:
> - dsa = result.groups()[0]
> - else:
> - logging.log(logging.WARNING, "File %s does not look like a
> proper DSA wml description, not checking" % filename)
> - return (None)
> -
> - logging.log (logging.DEBUG, "Parsing information for DSA %s from wml
> file %s" % (dsa, filename))
> -
> - try:
> - wmlFile = open(path)
> -
> - for line in wmlFile:
> - line= line.decode ("ISO-8859-2")
> -
> - descrpatern = re.compile
> (r'description>(.*?)</define-tag>')
> - result = descrpatern.search (line)
> - if result:
> - data["description"] = result.groups()[0]
> - continue
> -
> - sinfopatern = re.compile (r'<define-tag moreinfo>(.*?)')
> - result = sinfopatern.search (line)
> - if result:
> - moreinfo = True
> - data["moreinfo"] = result.groups()[0]
> - continue
> -
> - einfopatern = re.compile (r'</define-tag>')
> - if moreinfo and einfopatern.search (line):
> - data["moreinfo"] =
> __parseMoreinfo(data["moreinfo"])
> - moreinfo = False
> - continue
> -
> - if moreinfo:
> - data["moreinfo"] += line
> - continue
> -
> - except IOError:
> - logging.log (logging.ERROR, "Can't work with file %s" % path)
> -
> - return (dsa, data)
> + """ Parse wml file with description of Debian Security Advisories
> +
> + Keyword arguments:
> + path -- full path to wml file
> +
> + return list (dsa id, tags data)"""
> +
> + data = {}
> + moreinfo = False
> + pack_ver = ""
> + deb_version = ""
> + releases = {}
> +
> + filename = os.path.basename (path)
> +
> + patern = re.compile(r'dsa-(\d+)')
> + result = patern.search(filename)
> + if result:
> + dsa = result.groups()[0]
> + else:
> + logging.log(logging.WARNING, "File %s does not look like a proper
> DSA wml description, not checking" % filename)
> + return (None)
> +
> + logging.log (logging.DEBUG, "Parsing information for DSA %s from wml
> file %s" % (dsa, filename))
> +
> + try:
> + wmlFile = open(path)
> +
> + for line in wmlFile:
> + line= line.decode ("ISO-8859-2")
> +
> + descrpatern = re.compile (r'description>(.*?)</define-tag>')
> + result = descrpatern.search (line)
> + if result:
> + data["description"] = result.groups()[0]
> + continue
> +
> + sinfopatern = re.compile (r'<define-tag moreinfo>(.*?)')
> + result = sinfopatern.search (line)
> + if result:
> + moreinfo = True
> + data["moreinfo"] = result.groups()[0]
> + continue
> +
> + einfopatern = re.compile (r'</define-tag>')
> + if moreinfo and einfopatern.search (line):
> + data["moreinfo"] = __parseMoreinfo(data["moreinfo"])
> + moreinfo = False
> + continue
> +
> + if moreinfo:
> + data["moreinfo"] += line
> + #continue
> +
> + dversion_pattern = re.compile(r'distribution \((.*?)\)')
> + result = dversion_pattern.search(line)
> + if result:
> + deb_version = result.groups()[0]
> +
> + new_version_pattern = re.compile(r'version (.*?).</p>')
> + result = new_version_pattern.search(line)
> + if result and deb_version != "":
> + pack_ver = result.groups()[0]
> + releases.update({DEBIAN_VERSION[deb_version]: {u"all":
> {grabPackName(path) : pack_ver}}})
> +
> +
> +
> + except IOError:
> + logging.log (logging.ERROR, "Can't work with file %s" % path)
> +
> + return dsa, data, releases
>
> def __parseMoreinfo (info):
> - """ Remove unnecessary information form moreinfo tag"""
> + """ Remove unnecessary information form moreinfo tag"""
> +
> + p = re.compile ("<p>(.*?)</p>", re.DOTALL)
> + paragraphs = [m.groups()[0] for m in re.finditer(p, info)]
> + result = ""
> +
> + for par in paragraphs:
> + if re.match(re.compile("For the .* distribution"), par):
> + break
> + result += "\n" + par
> +
> + return result
> +
> +def grabPackName(path):
> + """
> + :param path: full path to wml file
> + :return: string: Package Name
> + """
>
> - p = re.compile ("<p>(.*?)</p>", re.DOTALL)
> - paragraphs = [m.groups()[0] for m in re.finditer(p, info)]
> - result = ""
> -
> - for par in paragraphs:
> - if re.match(re.compile("For the .* distribution"), par):
> - break
> - result += "\n" + par
> -
> - return result
> + try:
> + wmlFile = open(path)
> + package_name = re.compile (r'We recommend that you upgrade your
> (.*?) packages')
> + for line in wmlFile:
> + result = package_name.search(line)
> + if result:
> + return result.groups()[0]
> + except IOError:
> + logging.log (logging.ERROR, "Can't work with file %s" % path)
> Binary files oval_old/oval/parser/wml.pyc and oval/oval/parser/wml.pyc differ
> diff -urN oval_old/parseDsa2Oval.py oval/parseDsa2Oval.py
> --- oval_old/parseDsa2Oval.py 2011-10-12 15:14:55.000000000 -0600
> +++ oval/parseDsa2Oval.py 2015-11-13 11:13:33.923838000 -0700
> @@ -2,11 +2,11 @@
> # -*- coding: utf-8 -*-
> # Extracts the data DSA files and creates OVAL queries to
> # be used with the OVAL query interpreter (see http://oval.mitre.org)
> -
> +# (c) 2015 Nicholas Luedtke
> # (c) 2007 Pavel Vinogradov
> # (c) 2004 Javier Fernandez-Sanguino
>
> # Licensed under the GNU General Public License version 2.
>
> -
>
> +
> import os
> import sys
> import getopt
> @@ -18,89 +18,89 @@
>
> dsaref = {}
>
> +
> def usage (prog = "parse-wml-oval.py"):
> - """Print information about script flags and options"""
> + """Print information about script flags and options"""
>
> - print """
> -usage: %s [vh] [-d <directory>]
> -\t-d\twhich directory use for dsa definition search
> -\t-v\tverbose mode
> -\t-h\tthis help
> - """ % prog
> -
> -def printdsas (dsaref):
> + print """usage: %s [vh] [-d <directory>]\t-d\twhich directory use for
> + dsa definition search\t-v\tverbose mode\t-h\tthis help""" % prog
> +
> +def printdsas(dsaref):
> """ Generate and print OVAL Definitions for collected DSA information """
> -
> +
> ovalDefinitions = oval.definition.generator.createOVALDefinitions
> (dsaref)
> oval.definition.generator.printOVALDefinitions (ovalDefinitions)
>
> def parsedirs (directory, postfix, depth):
> - """ Recursive search directory for DSA files contain postfix in their
> names.
> -
> - For this files called oval.parser.dsa.parseFile() for
> extracting DSA information.
> - """
> -
> - if depth == 0:
> - logging.log(logging.DEBUG, "Maximum depth reached at directory
> " + directory)
> - return (0)
> + """ Recursive search directory for DSA files contain postfix in their
> names.
> + For this files called oval.parser.dsa.parseFile() for extracting DSA
> + information.
> + """
> +
> + if depth == 0:
> + logging.log(logging.DEBUG, "Maximum depth reached at directory " +
> directory)
> + return (0)
>
> - for file in os.listdir (directory):
> -
> - path = "%s/%s" % (directory, file)
> -
> - logging.log (logging.DEBUG, "Checking %s (for %s at %s)" %
> (file, postfix, depth))
> + for file in os.listdir (directory):
>
> - if os.access(path, os.R_OK) and os.path.isdir (path) and not
> os.path.islink (path) and file[0] != '.':
> - logging.log(logging.DEBUG, "Entering directory " + path)
> - parsedirs (path, postfix, depth-1)
> + path = "%s/%s" % (directory, file)
> + logging.log (logging.DEBUG, "Checking %s (for %s at %s)" % (file,
> postfix, depth))
>
> + if os.access(path, os.R_OK) and os.path.isdir (path) and not
> os.path.islink (path) and file[0] != '.':
> + logging.log(logging.DEBUG, "Entering directory " + path)
> + parsedirs (path, postfix, depth-1)
> +
> #Parse DSA data files
> - if os.access(path, os.R_OK) and file.endswith(postfix) and
> file[0] != '.' and file[0] != '#':
> - result = dsa.parseFile (path)
> - if result:
> - if dsaref.has_key (result[0]):
> - for (k, v) in result[1].iteritems():
> - dsaref[result[0]][k] = v
> - else:
> - dsaref[result[0]] = result[1]
> -
> + if os.access(path, os.R_OK) and file.endswith(postfix) and file[0]
> != '.' and file[0] != '#':
> + result = dsa.parseFile(path)
> + if result:
> + if dsaref.has_key(result[0]):
> + for (k, v) in result[1].iteritems():
> + dsaref[result[0]][k] = v
> + else:
> + dsaref[result[0]] = result[1]
> +
> #Parse DSA wml descriptions
> - if os.access(path, os.R_OK) and file.endswith(".wml") and
> file[0] != '.' and file[0] != '#':
> - result = wml.parseFile(path)
> - if result:
> - if dsaref.has_key (result[0]):
> - for (k, v) in result[1].iteritems():
> - dsaref[result[0]][k] = v
> - else:
> - dsaref[result[0]] = result[1]
> -
> - return 0
> + if os.access(path, os.R_OK) and file.endswith(".wml") and file[0] !=
> '.' and file[0] != '#':
> + result = wml.parseFile(path)
> + if result:
> + if dsaref.has_key(result[0]):
> + for (k, v) in result[1].iteritems():
> + dsaref[result[0]][k] = v
> + if not dsaref[result[0]].has_key("release"):
> + dsaref[result[0]]['release']=result[2]
> + else:
> + dsaref[result[0]] = result[1]
> + dsaref[result[0]]['release']=result[2]
> +
> + return 0
>
> if __name__ == "__main__":
> -
> +
> # Parse cmd options with getopt
> opts = {}
> -
> +
> #By default we search dsa definitions from current directory, but -d
> option override this
> opts['-d'] = "./"
> -
> +
> try:
> opt, args = getopt.getopt (sys.argv[1:], 'vhd:')
> except getopt.GetoptError:
> usage ()
> sys.exit(1)
> -
> +
> for key, value in opt:
> opts[key] = value
> -
> +
> if opts.has_key ('-h'):
> usage()
> sys.exit(0)
> -
> +
> if opts.has_key('-v'):
> logging.basicConfig(level=logging.DEBUG)
> -
> +
> logging.basicConfig(level=logging.WARNING)
> -
> +
> parsedirs (opts['-d'], '.data', 2)
> +
> printdsas(dsaref)
> diff -urN oval_old/parseJSON2Oval.py oval/parseJSON2Oval.py
> --- oval_old/parseJSON2Oval.py 1969-12-31 17:00:00.000000000 -0700
> +++ oval/parseJSON2Oval.py 2015-11-13 13:16:42.523783000 -0700
> @@ -0,0 +1,191 @@
> +#!/usr/bin/python
> +# -*- coding: utf-8 -*-
> +# Extracts the data from the security tracker and creates OVAL queries to
> +# be used with the OVAL query interpreter (see http://oval.mitre.org)
> +
> +# (c) 2015 Nicholas Luedtke
> +# Licensed under the GNU General Public License version 2.
>
> +
> +import os
> +from subprocess import call
> +import sys
> +import logging
> +import argparse
> +import json
> +from datetime import date
> +import oval.definition.generator
> +from oval.parser import dsa
> +from oval.parser import wml
> +
> +
> +dsaref = {}
> +
> +# TODO: these may need changed or reworked.
> +DEBIAN_VERSION = {"wheezy" : "7.0", "jessie" : "8.2", "stretch" : "9.0",
> + "sid" : "9.0", "etch" : "4.0", "squeeze":"6.0",
> "lenny":"5.0"}
> +
> +def usage (prog = "parse-wml-oval.py"):
> + """Print information about script flags and options"""
> +
> + print """usage: %s [vh] [-d <directory>]\t-d\twhich directory use for
> + dsa definition search\t-v\tverbose mode\t-h\tthis help""" % prog
> +
> +
> +def printdsas(dsaref):
> + """ Generate and print OVAL Definitions for collected DSA information """
> +
> + ovalDefinitions = oval.definition.generator.createOVALDefinitions
> (dsaref)
> + oval.definition.generator.printOVALDefinitions (ovalDefinitions)
> +
> +
> +def parsedirs (directory, postfix, depth):
> + """ Recursive search directory for DSA files contain postfix in their
> names.
> + For this files called oval.parser.dsa.parseFile() for extracting DSA
> + information.
> + """
> + for file in os.listdir (directory):
> +
> + path = "%s/%s" % (directory, file)
> + logging.log (logging.DEBUG, "Checking %s (for %s at %s)" % (file,
> postfix, depth))
> +
> + if os.access(path, os.R_OK) and os.path.isdir (path) and not
> os.path.islink (path) and file[0] != '.':
> + logging.log(logging.DEBUG, "Entering directory " + path)
> + parsedirs (path, postfix, depth-1)
> +
> + #Parse DSA data files
> + if os.access(path, os.R_OK) and file.endswith(postfix) and file[0]
> != '.' and file[0] != '#':
> + result = dsa.parseFile(path)
> + if result:
> + if dsaref.has_key(result[0]):
> + for (k, v) in result[1].iteritems():
> + dsaref[result[0]][k] = v
> + else:
> + dsaref[result[0]] = result[1]
> +
> + #Parse DSA wml descriptions
> + if os.access(path, os.R_OK) and file.endswith(".wml") and file[0] !=
> '.' and file[0] != '#':
> + result = wml.parseFile(path)
> + if result:
> + if dsaref.has_key(result[0]):
> + for (k, v) in result[1].iteritems():
> + dsaref[result[0]][k] = v
> + if not dsaref[result[0]].has_key("release"):
> + dsaref[result[0]]['release']=result[2]
> + else:
> + dsaref[result[0]] = result[1]
> + dsaref[result[0]]['release']=result[2]
> + return 0
> +
> +
> +def parseJSON(json_data, id_num):
> + """
> + Parse the JSON data and extract information needed for OVAL definitions
> + :param id_num: int id number to start at for defintions
> + :param json_data: Json_Data
> + :return:
> + """
> + today = date.today()
> + logging.log(logging.DEBUG, "Start of JSON Parse.")
> + d_num = id_num
> + for package in json_data:
> + logging.log(logging.DEBUG, "Parsing package %s" % package)
> + for CVE in json_data[package]:
> + logging.log(logging.DEBUG, "Getting releases for %s" % CVE)
> + release = {}
> + for rel in json_data[package][CVE]['releases']:
> + if json_data[package][CVE]['releases'][rel]['status'] != \
> + 'resolved':
> + fixed_v = '0'
> + f_str = 'no'
> + else:
> + fixed_v =
> json_data[package][CVE]['releases'][rel]['fixed_version']
> + f_str = 'yes'
> + release.update({DEBIAN_VERSION[rel]: {u'all': {
> + package: fixed_v}}})
> +
> + dsaref.update({str(d_num): {"packages": package,
> + 'description': "",
> + 'vulnerable': "yes",
> + 'date': str(today.isoformat()),
> + 'fixed': f_str, 'moreinfo': "",
> + 'release': release, 'secrefs': CVE}})
> + logging.log(logging.DEBUG, "Created entry in dsaref %s" %
> d_num)
> + d_num += 1
> +
> +
> +def get_json_data(json_file):
> + """
> + Retrieves JSON formatted data from a file.
> + :param json_file:
> + :return: JSON data (dependent on the file loaded, usually a dictionary.)
> + """
> + logging.log(logging.DEBUG, "Extracting JSON file %s" % json_file)
> + with open(json_file, "r") as json_d:
> + d = json.load(json_d)
> + return d
> +
> +
> +def main(args):
> + """
> + Main function for parseJSON2Oval.py
> + :param args:
> + :return:
> + """
> +
> + if args['verbose']:
> + logging.basicConfig(level=logging.DEBUG)
> + else:
> + logging.basicConfig(level=logging.WARNING)
> +
> + # unpack args
> +
> + json_file = args['JSONfile']
> + temp_file = args['tmp']
> + id_num = args['id']
> +
> + if json_file:
> + json_data = get_json_data(json_file)
> + else:
> + logging.log(logging.DEBUG, "Preparing to download JSONfile")
> + if os.path.isfile(temp_file):
> + logging.log(logging.WARNING, "Removing file %s" % temp_file)
> + os.remove(temp_file)
> + logging.log(logging.DEBUG, "Issuing wget for JSON file")
> + args = ['wget',
> 'https://security-tracker.debian.org/tracker/data/json',
> + '-O', temp_file]
> + call(args)
> + logging.log(logging.DEBUG, "File %s received" % temp_file)
> + json_data = get_json_data(temp_file)
> + if os.path.isfile(temp_file):
> + logging.log(logging.DEBUG, "Removing file %s" % temp_file)
> + os.remove(temp_file)
> +
> + parseJSON(json_data, id_num)
> + #parsedirs (opts['-d'], '.data', 2)
> +
> + printdsas(dsaref)
> +
> +if __name__ == "__main__":
> + PARSER = argparse.ArgumentParser(description='Generates oval definitions
> '
> + 'from the JSON file used to
> '
> + 'build the Debian Security '
> + 'Tracker.')
> + PARSER.add_argument('-v', '--verbose', help='Verbose Mode',
> + action="store_true")
> + PARSER.add_argument('-j', '--JSONfile', type=str,
> + help='Local JSON file to use. This will use a local '
> + 'copy of the JSON file instead of downloading
> from'
> + ' it from the server. default=none',
> default=None)
> + PARSER.add_argument('-t', '--tmp', type=str,
> + help='Temporary file to download JSON file to.
> Warning:'
> + ' if this file already exists it will be
> removed '
> + 'prior to downloading the JSON file. default= '
> + './DebSecTrackTMP.t',
> default='./DebSecTrackTMP.t')
> + PARSER.add_argument('--id', type=int,
> + help='id number to start defintions at. default=100',
> + default=100)
> + ARGS = vars(PARSER.parse_args())
> + main(ARGS)
> +
> +
> +
