[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/depend/
commit: 70704ab3f8983dd69a4742cd4c3a0810e4eed7e5 Author: Mike Frysinger gentoo org> AuthorDate: Mon Jan 4 20:51:39 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 10 17:53:45 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=70704ab3 repoman: filter out duplicate dependencies in error messages Some packages list the same atom multiple times (e.g. behind diff USE flags). If one of them throws an error, we end up listing it more than once, and the output can get verbose/useless. pym/repoman/modules/scan/depend/profile.py | 13 +++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/depend/profile.py b/pym/repoman/modules/scan/depend/profile.py index 5b32f66..a714a93 100644 --- a/pym/repoman/modules/scan/depend/profile.py +++ b/pym/repoman/modules/scan/depend/profile.py @@ -201,13 +201,22 @@ class ProfileDependsChecks(ScanBase): # we have some unsolvable deps # remove ! deps, which always show up as unsatisfiable - atoms = [ + all_atoms = [ str(atom.unevaluated_atom) for atom in atoms if not atom.blocker] # if we emptied out our list, continue: - if not atoms: + if not all_atoms: continue + + # Filter out duplicates. We do this by hand (rather + # than use a set) so the order is stable and better + # matches the order that's in the ebuild itself. + atoms = [] + for atom in all_atoms: + if atom not in atoms: + atoms.append(atom) + if self.options.output_style in ['column']: self.qatracker.add_error(mykey, "%s: %s: %s(%s) %s"
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 22a9d9df146499ebf740eb61964633ba7d6a10ef Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 18:42:41 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=22a9d9df repoman: Move some metdata_* variables to module it's used pym/repoman/metadata.py | 4 pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index 7a514dc..a980184 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -28,10 +28,6 @@ if sys.hexversion >= 0x300: if sys.hexversion >= 0x300: basestring = str -metadata_xml_encoding = 'UTF-8' -metadata_xml_declaration = '' \ - % (metadata_xml_encoding,) -metadata_doctype_name = 'pkgmetadata' metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' # force refetch if the local copy creation time is older than this diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3b48b8e..a7150a9 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -23,9 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage -from repoman.metadata import ( - metadata_xml_encoding, metadata_doctype_name, - metadata_dtd_uri, metadata_xml_declaration, parse_metadata_use) +from repoman.metadata import metadata_dtd_uri, parse_metadata_use from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint @@ -38,6 +36,11 @@ from portage.dep import Atom from .use_flags import USEFlagChecks +metadata_xml_encoding = 'UTF-8' +metadata_xml_declaration = '' \ + % (metadata_xml_encoding,) +metadata_doctype_name = 'pkgmetadata' + class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks'''
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: c6d6b003bc0545912f15c82a99c2e948a5c63e60
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 20:54:33 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c6d6b003
repoman/modules/.../pkgmetadata.py: Have xml validation log all qatracker
errors
Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++-
1 file changed, 36 insertions(+), 44 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7117e7d..83ca93f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
(xpkg, metadata_doctype_name,
doctype_name))
# load USE flags from metadata.xml
- self.musedict, metadata_bad = self._parse_metadata_use(
- _metadata_xml, xpkg, metadata_bad)
+ self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
for atom in chain(*self.musedict.values()):
if atom is None:
continue
@@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# Only carry out if in package directory or check forced
if not metadata_bad:
validator = etree.XMLSchema(file=self.metadata_xsd)
- try:
- validator.assertValid(_metadata_xml)
- except etree.DocumentInvalid as error:
- self.qatracker.add_error(
- "metadata.bad",
- xpkg + "/metadata.xml: %s"
- % (str(error))
- )
- del metadata_bad
+ if not validator.validate(_metadata_xml):
+ self._add_validate_errors(xpkg,
validator.error_log)
self.muselist = frozenset(self.musedict)
return False
@@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
% (xpkg, myflag))
return False
- def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+ def _parse_metadata_use(self, xml_tree, xpkg):
"""
Records are wrapped in XML as per GLEP 56
returns a dict with keys constisting of USE flag names and
values
@@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
usetags = xml_tree.findall("use")
if not usetags:
- return uselist, metadata_bad
+ return uselist
# It's possible to have multiple 'use' elements.
for usetag in usetags:
@@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
for flag in flags:
pkg_flag = flag.get("name")
- if pkg_flag is None:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: line: %s,
'%s', missing attribute: name"
- % (xpkg, flag.sourceline,
flag.text))
- continue
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from
python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/metadata/
commit: 65fc4d8a8e0e841e793b3f020c4caeb84ecc
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Mon Jan 25 19:40:29 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=65fc4d8a
repoman: Move parse_metadata_use function to the pkgmetadata.py
Add missing basestring compat code
pym/repoman/metadata.py | 50 -
pym/repoman/modules/scan/metadata/pkgmetadata.py | 56 +++-
2 files changed, 55 insertions(+), 51 deletions(-)
diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index a980184..7c64c8e 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -17,7 +17,6 @@ except ImportError:
# import our initialized portage instance
from repoman._portage import portage
-from portage import exception
from portage import os
from portage import shutil
from portage.output import green
@@ -34,55 +33,6 @@ metadata_xsd_uri =
'http://www.gentoo.org/xml-schema/metadata.xsd'
metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
-def parse_metadata_use(xml_tree):
- """
- Records are wrapped in XML as per GLEP 56
- returns a dict with keys constisting of USE flag names and values
- containing their respective descriptions
- """
- uselist = {}
-
- usetags = xml_tree.findall("use")
- if not usetags:
- return uselist
-
- # It's possible to have multiple 'use' elements.
- for usetag in usetags:
- flags = usetag.findall("flag")
- if not flags:
- # DTD allows use elements containing no flag elements.
- continue
-
- for flag in flags:
- pkg_flag = flag.get("name")
- if pkg_flag is None:
- raise exception.ParseError("missing 'name'
attribute for 'flag' tag")
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
- # (flag_restrict can be None)
- uselist[pkg_flag][flag_restrict] = "
".join("".join(inner_text).split())
-
- return uselist
-
-
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
Fetch metadata.xsd if it doesn't exist or the ctime is older than
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index a7150a9..af53f4b 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -23,7 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception):
# import our initialized portage instance
from repoman._portage import portage
-from repoman.metadata import metadata_dtd_uri, parse_metadata_use
+from repoman.metadata import metadata_dtd_uri
from repoman.checks.herds.herdbase import get_herd_base
from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
@@ -32,16 +32,70 @@ from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
from portage import _encodings, _unicode_encode
+from portage import exception
from portage.dep import Atom
from .use_flags import USEFlagChecks
+if sys.hexversion >= 0x300:
+ # pylint: disable=W0622
+ basestring = str
+
metadata_xml_encoding = 'UTF-8'
metadata_xml_declaration = '' \
% (metadata_xml_encoding,)
metadata_doctype_name = 'pkgmetadata'
+def parse_metadata_use(xml_tree):
+ """
+ Records are wrapped in XML as per GLEP 56
+ returns a dict with keys constisting of USE flag names and values
+ containing their respective descriptions
+ """
+ uselist = {}
+
+ usetags = xml_tree.findall("use")
+ if not usetags:
+ return uselist
+
+ # It's possible to have multiple 'use' elements.
+ for usetag in
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 57d1ef7f8f98275b066735c66d0da05ea103a9da
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 18:46:22 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=57d1ef7f
repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the
PkgMetadata class
This allows for a complete Q/A processing of all the use flags defined in the
xml.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++
1 file changed, 68 insertions(+), 70 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 1594b27..7117e7d 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
-from portage import exception
from portage.dep import Atom
from .use_flags import USEFlagChecks
@@ -43,55 +42,6 @@ metadata_xml_declaration = '' \
metadata_doctype_name = 'pkgmetadata'
-def parse_metadata_use(xml_tree):
- """
- Records are wrapped in XML as per GLEP 56
- returns a dict with keys constisting of USE flag names and values
- containing their respective descriptions
- """
- uselist = {}
-
- usetags = xml_tree.findall("use")
- if not usetags:
- return uselist
-
- # It's possible to have multiple 'use' elements.
- for usetag in usetags:
- flags = usetag.findall("flag")
- if not flags:
- # DTD allows use elements containing no flag elements.
- continue
-
- for flag in flags:
- pkg_flag = flag.get("name")
- if pkg_flag is None:
- raise exception.ParseError("missing 'name'
attribute for 'flag' tag")
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
- # (flag_restrict can be None)
- uselist[pkg_flag][flag_restrict] = "
".join("".join(inner_text).split())
-
- return uselist
-
-
class PkgMetadata(ScanBase, USEFlagChecks):
'''Package metadata.xml checks'''
@@ -180,28 +130,23 @@ class PkgMetadata(ScanBase, USEFlagChecks):
(xpkg, metadata_doctype_name,
doctype_name))
# load USE flags from metadata.xml
- try:
- self.musedict = parse_metadata_use(_metadata_xml)
- except portage.exception.ParseError as e:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad", "%s/metadata.xml: %s" % (xpkg,
e))
- else:
- for atom in chain(*self.musedict.values()):
- if atom is None:
- continue
- try:
- atom = Atom(atom)
- except InvalidAtom as e:
+ self.musedict, metadata_bad = self._parse_metadata_use(
+ _metadata_xml, xpkg, metadata_bad)
+ for atom in chain(*self.musedict.values()):
+ if atom is None:
+ continue
+ try:
+ atom = Atom(atom)
+ except InvalidAtom as e:
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: Invalid atom: %s" %
(xpkg, e))
+ else:
+ if atom.cp != xpkg:
self.qatracker.add_error(
"metadata.bad",
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 633cabfb9215633c554fd967b9875310be3718bd
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Tue May 3 07:18:05 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=633cabfb
repoman: Use lxml for parsing of metadata
Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).
Update import error message to lxml pkg.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++-
1 file changed, 13 insertions(+), 33 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index bcddb3e..317ab56 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,14 +7,14 @@ import sys
from itertools import chain
try:
- import xml.etree.ElementTree
- from xml.parsers.expat import ExpatError
+ from lxml import etree
+ from lxml.etree import ParserError
except (SystemExit, KeyboardInterrupt):
raise
except (ImportError, SystemError, RuntimeError, Exception):
# broken or missing xml support
# http://bugs.python.org/issue14988
- msg = ["Please enable python's \"xml\" USE flag in order to use
repoman."]
+ msg = ["Please emerge dev-python/lxml in order to use repoman."]
from portage.output import EOutput
out = EOutput()
for line in msg:
@@ -26,12 +26,11 @@ from repoman._portage import portage
from repoman.metadata import metadata_dtd_uri
from repoman.checks.herds.herdbase import get_herd_base
from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
-from portage import _encodings, _unicode_encode
from portage import exception
from portage.dep import Atom
@@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# metadata.xml parse check
metadata_bad = False
- xml_info = {}
- xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
# read metadata.xml into memory
try:
- _metadata_xml = xml.etree.ElementTree.parse(
- _unicode_encode(
- os.path.join(checkdir, "metadata.xml"),
- encoding=_encodings['fs'],
errors='strict'),
- parser=xml_parser)
- except (ExpatError, SyntaxError, EnvironmentError) as e:
+ _metadata_xml = etree.parse(os.path.join(checkdir,
'metadata.xml'))
+ except (ParserError, SyntaxError, EnvironmentError) as e:
metadata_bad = True
self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
del e
self.muselist = frozenset(self.musedict)
return False
- if "XML_DECLARATION" not in xml_info:
+ xml_encoding = _metadata_xml.docinfo.encoding
+ if xml_encoding.upper() != metadata_xml_encoding:
self.qatracker.add_error(
"metadata.bad", "%s/metadata.xml: "
- "xml declaration is missing on first line, "
- "should be '%s'" % (xpkg,
metadata_xml_declaration))
- else:
- xml_version, xml_encoding, xml_standalone = \
- xml_info["XML_DECLARATION"]
- if xml_encoding is None or \
- xml_encoding.upper() != metadata_xml_encoding:
- if xml_encoding is None:
- encoding_problem = "but it is undefined"
- else:
- encoding_problem = "not '%s'" %
xml_encoding
- self.qatracker.add_error(
- "metadata.bad", "%s/metadata.xml: "
- "xml declaration encoding should be
'%s', %s" %
- (xpkg, metadata_xml_encoding,
encoding_problem))
+ "xml declaration encoding should be '%s', not
'%s'" %
+ (xpkg, metadata_xml_encoding, xml_encoding))
- if "DOCTYPE" not in xml_info:
+ if not _metadata_xml.docinfo:
metadata_bad = True
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 9c3e213142e0e39f69dc002bd15402a7a2ff22d1 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:45:31 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9c3e2131 repoman: mandate use of python 2.7 or 3.2+ for simplicity pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index af53f4b..bcddb3e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -162,54 +162,48 @@ class PkgMetadata(ScanBase, USEFlagChecks): self.muselist = frozenset(self.musedict) return False - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True + if "DOCTYPE" not in xml_info: + metadata_bad = True + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) + else: + doctype_name, doctype_system, doctype_pubid = \ + xml_info["DOCTYPE"] + if doctype_system != metadata_dtd_uri: + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] - if doctype_system != metadata_dtd_uri: - if doctype_system is None: -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 2eec213a7226380983f270276fef67f9c0337cab Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 07:39:09 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2eec213a repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks This change based on original work done by Dirkjan Ochtman ochtman.nl> pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 1 file changed, 12 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3ca7897..4921b6f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage from repoman.metadata import metadata_dtd_uri -from repoman.checks.herds.herdbase import get_herd_base -from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom @@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks): "%s/metadata.xml: Atom contains " "unexpected cat/pn: %s" % (xpkg, atom)) - # Run other metadata.xml checkers - try: - check_metadata(_metadata_xml, get_herd_base( - self.repoman_settings)) - except (UnknownHerdsError, ) as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/
commit: 4175b9b3b39aa0f502921435bdae34a74679e53b
Author: Brian Dolbec gentoo org>
AuthorDate: Sun May 8 20:07:02 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4175b9b3
repoman Comment out the $ID header check for now. bug 579460
Leave the code in place for now, it may be repo config enabled later.
pym/repoman/modules/scan/ebuild/checks.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/modules/scan/ebuild/checks.py
b/pym/repoman/modules/scan/ebuild/checks.py
index 8cdc230..fb3e019 100644
--- a/pym/repoman/modules/scan/ebuild/checks.py
+++ b/pym/repoman/modules/scan/ebuild/checks.py
@@ -108,8 +108,8 @@ class EbuildHeader(LineCheck):
return errors.COPYRIGHT_ERROR
elif num == 1 and line.rstrip('\n') != self.gentoo_license:
return errors.LICENSE_ERROR
- elif num == 2 and line.rstrip('\n') != self.id_header:
- return errors.ID_HEADER_ERROR
+ #elif num == 2 and line.rstrip('\n') != self.id_header:
+ # return errors.ID_HEADER_ERROR
class EbuildWhitespace(LineCheck):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: cfc31c04bf6e1d59b2756e3ac315a2040cdd162d Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:08:50 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfc31c04 repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection The docinfo is always created, so is not valid to use like a boolean to detect a missing doctype. doctype is a null string or equivalent and as such the correct choice for the if statement. Failure discoverd by: Göktürk Yüksek binghamton.edu> Thank you pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 83ca93f..d8344c2 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -106,7 +106,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): "xml declaration encoding should be '%s', not '%s'" % (xpkg, metadata_xml_encoding, xml_encoding)) - if not _metadata_xml.docinfo: + if not _metadata_xml.docinfo.doctype: metadata_bad = True self.qatracker.add_error( "metadata.bad",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: b78d61dd3e4281a43523aea1a24e395a2674c3ab
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Tue May 3 06:33:47 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b78d61dd
repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors
pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 ---
1 file changed, 75 insertions(+), 73 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3c1c2d0..3b48b8e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks):
metadata_bad = True
self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
del e
+ self.muselist = frozenset(self.musedict)
+ return False
+
+ if not hasattr(xml_parser, 'parser') or \
+ sys.hexversion < 0x207 or \
+ (sys.hexversion > 0x300 and sys.hexversion <
0x302):
+ # doctype is not parsed with python 2.6 or 3.1
+ pass
else:
- if not hasattr(xml_parser, 'parser') or \
- sys.hexversion < 0x207 or \
- (sys.hexversion > 0x300 and sys.hexversion
< 0x302):
- # doctype is not parsed with python 2.6 or 3.1
- pass
+ if "XML_DECLARATION" not in xml_info:
+ self.qatracker.add_error(
+ "metadata.bad", "%s/metadata.xml: "
+ "xml declaration is missing on first
line, "
+ "should be '%s'" % (xpkg,
metadata_xml_declaration))
else:
- if "XML_DECLARATION" not in xml_info:
+ xml_version, xml_encoding, xml_standalone = \
+ xml_info["XML_DECLARATION"]
+ if xml_encoding is None or \
+ xml_encoding.upper() !=
metadata_xml_encoding:
+ if xml_encoding is None:
+ encoding_problem = "but it is
undefined"
+ else:
+ encoding_problem = "not '%s'" %
xml_encoding
self.qatracker.add_error(
"metadata.bad",
"%s/metadata.xml: "
- "xml declaration is missing on
first line, "
- "should be '%s'" % (xpkg,
metadata_xml_declaration))
- else:
- xml_version, xml_encoding,
xml_standalone = \
- xml_info["XML_DECLARATION"]
- if xml_encoding is None or \
- xml_encoding.upper() !=
metadata_xml_encoding:
- if xml_encoding is None:
- encoding_problem = "but
it is undefined"
- else:
- encoding_problem = "not
'%s'" % xml_encoding
- self.qatracker.add_error(
- "metadata.bad",
"%s/metadata.xml: "
- "xml declaration
encoding should be '%s', %s" %
- (xpkg,
metadata_xml_encoding, encoding_problem))
+ "xml declaration encoding
should be '%s', %s" %
+ (xpkg, metadata_xml_encoding,
encoding_problem))
- if "DOCTYPE" not in xml_info:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: %s" % (xpkg,
"DOCTYPE is missing"))
- else:
- doctype_name, doctype_system,
doctype_pubid = \
- xml_info["DOCTYPE"]
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 42b531b2810ec3e8e81348ebc4519a56e01a5a47
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Mon Jan 25 18:33:32 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=42b531b2
repoman/modules/.../pkgmetadata.py: Early return in metadata check code for
readability
pym/repoman/modules/scan/metadata/pkgmetadata.py | 190 ---
1 file changed, 96 insertions(+), 94 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 5c6452a..3c1c2d0 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -83,112 +83,114 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# metadata.xml file check
if "metadata.xml" not in checkdirlist:
self.qatracker.add_error("metadata.missing", xpkg +
"/metadata.xml")
+ self.muselist = frozenset(self.musedict)
+ return False
+
# metadata.xml parse check
+ metadata_bad = False
+ xml_info = {}
+ xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
+
+ # read metadata.xml into memory
+ try:
+ _metadata_xml = xml.etree.ElementTree.parse(
+ _unicode_encode(
+ os.path.join(checkdir, "metadata.xml"),
+ encoding=_encodings['fs'],
errors='strict'),
+ parser=xml_parser)
+ except (ExpatError, SyntaxError, EnvironmentError) as e:
+ metadata_bad = True
+ self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
+ del e
else:
- metadata_bad = False
- xml_info = {}
- xml_parser = _XMLParser(xml_info,
target=_MetadataTreeBuilder())
-
- # read metadata.xml into memory
- try:
- _metadata_xml = xml.etree.ElementTree.parse(
- _unicode_encode(
- os.path.join(checkdir,
"metadata.xml"),
- encoding=_encodings['fs'],
errors='strict'),
- parser=xml_parser)
- except (ExpatError, SyntaxError, EnvironmentError) as e:
- metadata_bad = True
- self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
- del e
+ if not hasattr(xml_parser, 'parser') or \
+ sys.hexversion < 0x207 or \
+ (sys.hexversion > 0x300 and sys.hexversion
< 0x302):
+ # doctype is not parsed with python 2.6 or 3.1
+ pass
else:
- if not hasattr(xml_parser, 'parser') or \
- sys.hexversion < 0x207 or \
- (sys.hexversion > 0x300 and
sys.hexversion < 0x302):
- # doctype is not parsed with python 2.6
or 3.1
- pass
+ if "XML_DECLARATION" not in xml_info:
+ self.qatracker.add_error(
+ "metadata.bad",
"%s/metadata.xml: "
+ "xml declaration is missing on
first line, "
+ "should be '%s'" % (xpkg,
metadata_xml_declaration))
else:
- if "XML_DECLARATION" not in xml_info:
+ xml_version, xml_encoding,
xml_standalone = \
+ xml_info["XML_DECLARATION"]
+ if xml_encoding is None or \
+ xml_encoding.upper() !=
metadata_xml_encoding:
+ if xml_encoding is None:
+ encoding_problem = "but
it is undefined"
+ else:
+ encoding_problem = "not
'%s'" % xml_encoding
self.qatracker.add_error(
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 1f4d8908748dc59c362d655db3f6fe3b4c913476
Author: Brian Dolbec gentoo org>
AuthorDate: Thu May 5 16:04:54 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1f4d8908
repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors
Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..433551a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
return uselist
def _add_validate_errors(self, xpkg, log):
+ listed = set()
for error in log:
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: line: %s, %s"
- % (xpkg, error.line, error.message))
+ msg_prefix = error.message.split(":",1)[0]
+ info = "%s %s" % (error.line, msg_prefix)
+ if info not in listed:
+ listed.add(info)
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: line: %s, %s"
+ % (xpkg, error.line, error.message))
@property
def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 64bb8e17a5c050671765626bbe45174e10b5a3a4
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 17:38:19 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=64bb8e17
repoman/modules/.../pkgmetadata.py: Improve whole document validation
Change to using assertValid() and add the error causing validation to fail to
the qa tracker
error.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 4921b6f..1594b27 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -206,8 +206,14 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# Only carry out if in package directory or check forced
if not metadata_bad:
validator = etree.XMLSchema(file=self.metadata_xsd)
- if not validator.validate(_metadata_xml):
- self.qatracker.add_error("metadata.bad", xpkg +
"/metadata.xml")
+ try:
+ validator.assertValid(_metadata_xml)
+ except etree.DocumentInvalid as error:
+ self.qatracker.add_error(
+ "metadata.bad",
+ xpkg + "/metadata.xml: %s"
+ % (str(error))
+ )
del metadata_bad
self.muselist = frozenset(self.musedict)
return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 2d2b99917ed054bea083a0e47f30aac34f0fefdd
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Tue May 3 09:01:29 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sat May 14 18:29:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d2b9991
repoman: Migrate from XmlLint to etree.XMLSchema for validation
Remove No longer used repoman._xml module
This change based on work by Dirkjan Ochtman gentoo.org>
Updated the change from XML.DTD to XMLSchema.
Additionally:
Move the metadata.xsd path determination code to metadata.py.
Add the DISTDIR backup location and fetching of the file if missing or
stale.
pym/repoman/_xml.py | 105 ---
pym/repoman/metadata.py | 21 +
pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 +-
pym/repoman/scanner.py | 10 +--
4 files changed, 26 insertions(+), 119 deletions(-)
diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
deleted file mode 100644
index 33a536a..000
--- a/pym/repoman/_xml.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
-import sys
-import xml
-
-# import our initialized portage instance
-from repoman._portage import portage
-
-from portage import os
-from portage.output import red
-from portage.process import find_binary
-
-from repoman.metadata import fetch_metadata_xsd
-from repoman._subprocess import repoman_getstatusoutput
-
-
-class _XMLParser(xml.etree.ElementTree.XMLParser):
-
- def __init__(self, data, **kwargs):
- xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
- self._portage_data = data
- if hasattr(self, 'parser'):
- self._base_XmlDeclHandler = self.parser.XmlDeclHandler
- self.parser.XmlDeclHandler =
self._portage_XmlDeclHandler
- self._base_StartDoctypeDeclHandler = \
- self.parser.StartDoctypeDeclHandler
- self.parser.StartDoctypeDeclHandler = \
- self._portage_StartDoctypeDeclHandler
-
- def _portage_XmlDeclHandler(self, version, encoding, standalone):
- if self._base_XmlDeclHandler is not None:
- self._base_XmlDeclHandler(version, encoding, standalone)
- self._portage_data["XML_DECLARATION"] = (version, encoding,
standalone)
-
- def _portage_StartDoctypeDeclHandler(
- self, doctypeName, systemId, publicId, has_internal_subset):
- if self._base_StartDoctypeDeclHandler is not None:
- self._base_StartDoctypeDeclHandler(
- doctypeName, systemId, publicId,
has_internal_subset)
- self._portage_data["DOCTYPE"] = (doctypeName, systemId,
publicId)
-
-
-class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
- """
- Implements doctype() as required to avoid deprecation warnings with
- >=python-2.7.
- """
- def doctype(self, name, pubid, system):
- pass
-
-
-class XmlLint(object):
-
- def __init__(self, options, repoman_settings, metadata_xsd=None):
- self.metadata_xsd = (metadata_xsd or
- os.path.join(repoman_settings["DISTDIR"],
'metadata.xsd'))
- self.options = options
- self.repoman_settings = repoman_settings
- self._is_capable = metadata_xsd is not None
- self.binary = None
- self._check_capable()
-
- def _check_capable(self):
- if self.options.mode == "manifest":
- return
- self.binary = find_binary('xmllint')
- if not self.binary:
- print(red("!!! xmllint not found. Can't check
metadata.xml.\n"))
- elif not self._is_capable:
- if not fetch_metadata_xsd(self.metadata_xsd,
self.repoman_settings):
- sys.exit(1)
- # this can be problematic if xmllint changes their
output
- self._is_capable = True
-
- @property
- def capable(self):
- return self._is_capable
-
- def check(self, checkdir, repolevel):
- '''Runs checks on the package metadata.xml file
-
- @param checkdir: string, path
- @param repolevel: integer
- @return boolean, False == bad metadata
- '''
- if not self.capable:
- if self.options.xml_parse or repolevel == 3:
- print("%s sorry, xmllint is needed. failing\n"
% red("!!!"))
- sys.exit(1)
- return True
- # xmlint can produce garbage output
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/checks/, pym/repoman/modules/scan/, ...
commit: f1b60313df4738c6535db31f0fc32ffcc5ab4dbd
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 10 08:49:48 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 10 08:49:48 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f1b60313
Bulk move of repoman to it's own namespace for better packaging
Edit repoman calls from portage setup.py, MANIFEST.in.
Copy setup.py to the new repoman pkg, begin edit for repoman.
Copy TEST-NOTES to repoman
MANIFEST.in| 1 -
repoman/MANIFEST.in| 3 ++
repoman/README | 49 +
repoman/TEST-NOTES | 45
{bin => repoman/bin}/repoman | 0
{cnf => repoman/cnf}/metadata.xsd | 0
{man => repoman/man}/repoman.1 | 0
{pym => repoman/pym}/repoman/__init__.py | 0
{pym => repoman/pym}/repoman/_portage.py | 0
{pym => repoman/pym}/repoman/_subprocess.py| 0
{pym => repoman/pym}/repoman/actions.py| 0
{pym => repoman/pym}/repoman/argparser.py | 0
{pym => repoman/pym}/repoman/check_missingslot.py | 0
{pym => repoman/pym}/repoman/checks/__init__.py| 0
.../pym}/repoman/checks/herds/__init__.py | 0
.../pym}/repoman/checks/herds/herdbase.py | 0
.../pym}/repoman/checks/herds/metadata.py | 0
{pym => repoman/pym}/repoman/copyrights.py | 0
{pym => repoman/pym}/repoman/errors.py | 0
{pym => repoman/pym}/repoman/gpg.py| 0
{pym => repoman/pym}/repoman/main.py | 0
{pym => repoman/pym}/repoman/metadata.py | 0
{pym => repoman/pym}/repoman/modules/__init__.py | 0
.../pym}/repoman/modules/commit/__init__.py| 0
.../pym}/repoman/modules/commit/repochecks.py | 0
.../pym}/repoman/modules/scan/__init__.py | 0
.../pym}/repoman/modules/scan/depend/__init__.py | 0
.../repoman/modules/scan/depend/_depend_checks.py | 0
.../repoman/modules/scan/depend/_gen_arches.py | 0
.../pym}/repoman/modules/scan/depend/profile.py| 0
.../repoman/modules/scan/directories/__init__.py | 0
.../pym}/repoman/modules/scan/directories/files.py | 0
.../pym}/repoman/modules/scan/directories/mtime.py | 0
.../pym}/repoman/modules/scan/eapi/__init__.py | 0
.../pym}/repoman/modules/scan/eapi/eapi.py | 0
.../pym}/repoman/modules/scan/ebuild/__init__.py | 0
.../pym}/repoman/modules/scan/ebuild/checks.py | 0
.../pym}/repoman/modules/scan/ebuild/ebuild.py | 0
.../pym}/repoman/modules/scan/ebuild/errors.py | 0
.../pym}/repoman/modules/scan/ebuild/multicheck.py | 0
.../pym}/repoman/modules/scan/eclasses/__init__.py | 0
.../pym}/repoman/modules/scan/eclasses/live.py | 0
.../pym}/repoman/modules/scan/eclasses/ruby.py | 0
.../pym}/repoman/modules/scan/fetch/__init__.py| 0
.../pym}/repoman/modules/scan/fetch/fetches.py | 0
.../pym}/repoman/modules/scan/keywords/__init__.py | 0
.../pym}/repoman/modules/scan/keywords/keywords.py | 0
.../pym}/repoman/modules/scan/manifest/__init__.py | 0
.../repoman/modules/scan/manifest/manifests.py | 0
.../pym}/repoman/modules/scan/metadata/__init__.py | 0
.../repoman/modules/scan/metadata/description.py | 0
.../modules/scan/metadata/ebuild_metadata.py | 0
.../repoman/modules/scan/metadata/pkgmetadata.py | 0
.../pym}/repoman/modules/scan/metadata/restrict.py | 0
.../repoman/modules/scan/metadata/use_flags.py | 0
.../pym}/repoman/modules/scan/options/__init__.py | 0
.../pym}/repoman/modules/scan/options/options.py | 0
{pym => repoman/pym}/repoman/modules/scan/scan.py | 0
.../pym}/repoman/modules/scan/scanbase.py | 0
.../pym}/repoman/modules/vcs/None/__init__.py | 0
.../pym}/repoman/modules/vcs/None/changes.py | 0
.../pym}/repoman/modules/vcs/None/status.py| 0
.../pym}/repoman/modules/vcs/__init__.py | 0
.../pym}/repoman/modules/vcs/bzr/__init__.py | 0
.../pym}/repoman/modules/vcs/bzr/changes.py| 0
.../pym}/repoman/modules/vcs/bzr/status.py | 0
.../pym}/repoman/modules/vcs/changes.py| 0
.../pym}/repoman/modules/vcs/cvs/__init__.py | 0
.../pym}/repoman/modules/vcs/cvs/changes.py| 0
.../pym}/repoman/modules/vcs/cvs/status.py | 0
.../pym}/repoman/modules/vcs/git/__init__.py | 0
.../pym}/repoman/modules/vcs/git/changes.py| 0
.../pym}/repoman/modules/vcs/git/status.py | 0
.../pym}/repoman/modules/vcs/hg/__init__.py| 0
.../pym}/repoman/modules/vcs/hg/changes.py | 0
.../pym}/repoman/modules/vcs/hg/status.py | 0
.../pym}/repoman/modules/vcs/settings.py | 0
.../pym}/repoman/modules/vcs/svn/__init__.py | 0
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 8075f0e91fc06125d6e2bcaaf655f66de694a568
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Tue May 3 07:18:05 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:41 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8075f0e9
repoman: Use lxml for parsing of metadata
Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).
Update import error message to lxml pkg.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++-
1 file changed, 13 insertions(+), 33 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index bcddb3e..317ab56 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,14 +7,14 @@ import sys
from itertools import chain
try:
- import xml.etree.ElementTree
- from xml.parsers.expat import ExpatError
+ from lxml import etree
+ from lxml.etree import ParserError
except (SystemExit, KeyboardInterrupt):
raise
except (ImportError, SystemError, RuntimeError, Exception):
# broken or missing xml support
# http://bugs.python.org/issue14988
- msg = ["Please enable python's \"xml\" USE flag in order to use
repoman."]
+ msg = ["Please emerge dev-python/lxml in order to use repoman."]
from portage.output import EOutput
out = EOutput()
for line in msg:
@@ -26,12 +26,11 @@ from repoman._portage import portage
from repoman.metadata import metadata_dtd_uri
from repoman.checks.herds.herdbase import get_herd_base
from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
-from portage import _encodings, _unicode_encode
from portage import exception
from portage.dep import Atom
@@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# metadata.xml parse check
metadata_bad = False
- xml_info = {}
- xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
# read metadata.xml into memory
try:
- _metadata_xml = xml.etree.ElementTree.parse(
- _unicode_encode(
- os.path.join(checkdir, "metadata.xml"),
- encoding=_encodings['fs'],
errors='strict'),
- parser=xml_parser)
- except (ExpatError, SyntaxError, EnvironmentError) as e:
+ _metadata_xml = etree.parse(os.path.join(checkdir,
'metadata.xml'))
+ except (ParserError, SyntaxError, EnvironmentError) as e:
metadata_bad = True
self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
del e
self.muselist = frozenset(self.musedict)
return False
- if "XML_DECLARATION" not in xml_info:
+ xml_encoding = _metadata_xml.docinfo.encoding
+ if xml_encoding.upper() != metadata_xml_encoding:
self.qatracker.add_error(
"metadata.bad", "%s/metadata.xml: "
- "xml declaration is missing on first line, "
- "should be '%s'" % (xpkg,
metadata_xml_declaration))
- else:
- xml_version, xml_encoding, xml_standalone = \
- xml_info["XML_DECLARATION"]
- if xml_encoding is None or \
- xml_encoding.upper() != metadata_xml_encoding:
- if xml_encoding is None:
- encoding_problem = "but it is undefined"
- else:
- encoding_problem = "not '%s'" %
xml_encoding
- self.qatracker.add_error(
- "metadata.bad", "%s/metadata.xml: "
- "xml declaration encoding should be
'%s', %s" %
- (xpkg, metadata_xml_encoding,
encoding_problem))
+ "xml declaration encoding should be '%s', not
'%s'" %
+ (xpkg, metadata_xml_encoding, xml_encoding))
- if "DOCTYPE" not in xml_info:
+ if not _metadata_xml.docinfo:
metadata_bad = True
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 20fd37b3ddb901f15135d05518a7d8a7bfb7c320
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Mon Jan 25 19:40:29 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:41 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=20fd37b3
repoman: Move parse_metadata_use function to the pkgmetadata.py
Add missing basestring compat code
pym/repoman/metadata.py | 50 -
pym/repoman/modules/scan/metadata/pkgmetadata.py | 56 +++-
2 files changed, 55 insertions(+), 51 deletions(-)
diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index a980184..7c64c8e 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -17,7 +17,6 @@ except ImportError:
# import our initialized portage instance
from repoman._portage import portage
-from portage import exception
from portage import os
from portage import shutil
from portage.output import green
@@ -34,55 +33,6 @@ metadata_xsd_uri =
'http://www.gentoo.org/xml-schema/metadata.xsd'
metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
-def parse_metadata_use(xml_tree):
- """
- Records are wrapped in XML as per GLEP 56
- returns a dict with keys constisting of USE flag names and values
- containing their respective descriptions
- """
- uselist = {}
-
- usetags = xml_tree.findall("use")
- if not usetags:
- return uselist
-
- # It's possible to have multiple 'use' elements.
- for usetag in usetags:
- flags = usetag.findall("flag")
- if not flags:
- # DTD allows use elements containing no flag elements.
- continue
-
- for flag in flags:
- pkg_flag = flag.get("name")
- if pkg_flag is None:
- raise exception.ParseError("missing 'name'
attribute for 'flag' tag")
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
- # (flag_restrict can be None)
- uselist[pkg_flag][flag_restrict] = "
".join("".join(inner_text).split())
-
- return uselist
-
-
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
Fetch metadata.xsd if it doesn't exist or the ctime is older than
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index a7150a9..af53f4b 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -23,7 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception):
# import our initialized portage instance
from repoman._portage import portage
-from repoman.metadata import metadata_dtd_uri, parse_metadata_use
+from repoman.metadata import metadata_dtd_uri
from repoman.checks.herds.herdbase import get_herd_base
from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
@@ -32,16 +32,70 @@ from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
from portage import _encodings, _unicode_encode
+from portage import exception
from portage.dep import Atom
from .use_flags import USEFlagChecks
+if sys.hexversion >= 0x300:
+ # pylint: disable=W0622
+ basestring = str
+
metadata_xml_encoding = 'UTF-8'
metadata_xml_declaration = '' \
% (metadata_xml_encoding,)
metadata_doctype_name = 'pkgmetadata'
+def parse_metadata_use(xml_tree):
+ """
+ Records are wrapped in XML as per GLEP 56
+ returns a dict with keys constisting of USE flag names and values
+ containing their respective descriptions
+ """
+ uselist = {}
+
+ usetags = xml_tree.findall("use")
+ if not usetags:
+ return uselist
+
+ # It's possible to have multiple 'use' elements.
+ for usetag in
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 0cd414979f4b5531fdd158b3c4766c436bd057eb
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 20:54:33 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:42 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0cd41497
repoman/modules/.../pkgmetadata.py: Have xml validation log all qatracker
errors
Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++-
1 file changed, 36 insertions(+), 44 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7117e7d..83ca93f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
(xpkg, metadata_doctype_name,
doctype_name))
# load USE flags from metadata.xml
- self.musedict, metadata_bad = self._parse_metadata_use(
- _metadata_xml, xpkg, metadata_bad)
+ self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
for atom in chain(*self.musedict.values()):
if atom is None:
continue
@@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# Only carry out if in package directory or check forced
if not metadata_bad:
validator = etree.XMLSchema(file=self.metadata_xsd)
- try:
- validator.assertValid(_metadata_xml)
- except etree.DocumentInvalid as error:
- self.qatracker.add_error(
- "metadata.bad",
- xpkg + "/metadata.xml: %s"
- % (str(error))
- )
- del metadata_bad
+ if not validator.validate(_metadata_xml):
+ self._add_validate_errors(xpkg,
validator.error_log)
self.muselist = frozenset(self.musedict)
return False
@@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
% (xpkg, myflag))
return False
- def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+ def _parse_metadata_use(self, xml_tree, xpkg):
"""
Records are wrapped in XML as per GLEP 56
returns a dict with keys constisting of USE flag names and
values
@@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
usetags = xml_tree.findall("use")
if not usetags:
- return uselist, metadata_bad
+ return uselist
# It's possible to have multiple 'use' elements.
for usetag in usetags:
@@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
for flag in flags:
pkg_flag = flag.get("name")
- if pkg_flag is None:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: line: %s,
'%s', missing attribute: name"
- % (xpkg, flag.sourceline,
flag.text))
- continue
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from
python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 8c8ddcfa78eee27a823bc83dc67b34fda77387d6
Author: Brian Dolbec gentoo org>
AuthorDate: Thu May 5 16:04:54 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:42 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c8ddcfa
repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors
Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..433551a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
return uselist
def _add_validate_errors(self, xpkg, log):
+ listed = set()
for error in log:
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: line: %s, %s"
- % (xpkg, error.line, error.message))
+ msg_prefix = error.message.split(":",1)[0]
+ info = "%s %s" % (error.line, msg_prefix)
+ if info not in listed:
+ listed.add(info)
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: line: %s, %s"
+ % (xpkg, error.line, error.message))
@property
def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/metadata/
commit: 6121c08bda7fe4bb96befcfb8fe3ad4263f0b770 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 18:42:41 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6121c08b repoman: Move some metdata_* variables to module it's used pym/repoman/metadata.py | 4 pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index 7a514dc..a980184 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -28,10 +28,6 @@ if sys.hexversion >= 0x300: if sys.hexversion >= 0x300: basestring = str -metadata_xml_encoding = 'UTF-8' -metadata_xml_declaration = '' \ - % (metadata_xml_encoding,) -metadata_doctype_name = 'pkgmetadata' metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' # force refetch if the local copy creation time is older than this diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3b48b8e..a7150a9 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -23,9 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage -from repoman.metadata import ( - metadata_xml_encoding, metadata_doctype_name, - metadata_dtd_uri, metadata_xml_declaration, parse_metadata_use) +from repoman.metadata import metadata_dtd_uri, parse_metadata_use from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint @@ -38,6 +36,11 @@ from portage.dep import Atom from .use_flags import USEFlagChecks +metadata_xml_encoding = 'UTF-8' +metadata_xml_declaration = '' \ + % (metadata_xml_encoding,) +metadata_doctype_name = 'pkgmetadata' + class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks'''
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 278f13dfd0308f2d7d80fb332034611e0fa9f6ae
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 18:46:22 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:42 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=278f13df
repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the
PkgMetadata class
This allows for a complete Q/A processing of all the use flags defined in the
xml.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++
1 file changed, 68 insertions(+), 70 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 1594b27..7117e7d 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
-from portage import exception
from portage.dep import Atom
from .use_flags import USEFlagChecks
@@ -43,55 +42,6 @@ metadata_xml_declaration = '' \
metadata_doctype_name = 'pkgmetadata'
-def parse_metadata_use(xml_tree):
- """
- Records are wrapped in XML as per GLEP 56
- returns a dict with keys constisting of USE flag names and values
- containing their respective descriptions
- """
- uselist = {}
-
- usetags = xml_tree.findall("use")
- if not usetags:
- return uselist
-
- # It's possible to have multiple 'use' elements.
- for usetag in usetags:
- flags = usetag.findall("flag")
- if not flags:
- # DTD allows use elements containing no flag elements.
- continue
-
- for flag in flags:
- pkg_flag = flag.get("name")
- if pkg_flag is None:
- raise exception.ParseError("missing 'name'
attribute for 'flag' tag")
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
- # (flag_restrict can be None)
- uselist[pkg_flag][flag_restrict] = "
".join("".join(inner_text).split())
-
- return uselist
-
-
class PkgMetadata(ScanBase, USEFlagChecks):
'''Package metadata.xml checks'''
@@ -180,28 +130,23 @@ class PkgMetadata(ScanBase, USEFlagChecks):
(xpkg, metadata_doctype_name,
doctype_name))
# load USE flags from metadata.xml
- try:
- self.musedict = parse_metadata_use(_metadata_xml)
- except portage.exception.ParseError as e:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad", "%s/metadata.xml: %s" % (xpkg,
e))
- else:
- for atom in chain(*self.musedict.values()):
- if atom is None:
- continue
- try:
- atom = Atom(atom)
- except InvalidAtom as e:
+ self.musedict, metadata_bad = self._parse_metadata_use(
+ _metadata_xml, xpkg, metadata_bad)
+ for atom in chain(*self.musedict.values()):
+ if atom is None:
+ continue
+ try:
+ atom = Atom(atom)
+ except InvalidAtom as e:
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: Invalid atom: %s" %
(xpkg, e))
+ else:
+ if atom.cp != xpkg:
self.qatracker.add_error(
"metadata.bad",
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 5f15e81967f8dca211cbd87d8696059aeac9ada2
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Tue May 3 06:33:47 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:24 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5f15e819
repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors
pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 ---
1 file changed, 75 insertions(+), 73 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3c1c2d0..3b48b8e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks):
metadata_bad = True
self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
del e
+ self.muselist = frozenset(self.musedict)
+ return False
+
+ if not hasattr(xml_parser, 'parser') or \
+ sys.hexversion < 0x207 or \
+ (sys.hexversion > 0x300 and sys.hexversion <
0x302):
+ # doctype is not parsed with python 2.6 or 3.1
+ pass
else:
- if not hasattr(xml_parser, 'parser') or \
- sys.hexversion < 0x207 or \
- (sys.hexversion > 0x300 and sys.hexversion
< 0x302):
- # doctype is not parsed with python 2.6 or 3.1
- pass
+ if "XML_DECLARATION" not in xml_info:
+ self.qatracker.add_error(
+ "metadata.bad", "%s/metadata.xml: "
+ "xml declaration is missing on first
line, "
+ "should be '%s'" % (xpkg,
metadata_xml_declaration))
else:
- if "XML_DECLARATION" not in xml_info:
+ xml_version, xml_encoding, xml_standalone = \
+ xml_info["XML_DECLARATION"]
+ if xml_encoding is None or \
+ xml_encoding.upper() !=
metadata_xml_encoding:
+ if xml_encoding is None:
+ encoding_problem = "but it is
undefined"
+ else:
+ encoding_problem = "not '%s'" %
xml_encoding
self.qatracker.add_error(
"metadata.bad",
"%s/metadata.xml: "
- "xml declaration is missing on
first line, "
- "should be '%s'" % (xpkg,
metadata_xml_declaration))
- else:
- xml_version, xml_encoding,
xml_standalone = \
- xml_info["XML_DECLARATION"]
- if xml_encoding is None or \
- xml_encoding.upper() !=
metadata_xml_encoding:
- if xml_encoding is None:
- encoding_problem = "but
it is undefined"
- else:
- encoding_problem = "not
'%s'" % xml_encoding
- self.qatracker.add_error(
- "metadata.bad",
"%s/metadata.xml: "
- "xml declaration
encoding should be '%s', %s" %
- (xpkg,
metadata_xml_encoding, encoding_problem))
+ "xml declaration encoding
should be '%s', %s" %
+ (xpkg, metadata_xml_encoding,
encoding_problem))
- if "DOCTYPE" not in xml_info:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: %s" % (xpkg,
"DOCTYPE is missing"))
- else:
- doctype_name, doctype_system,
doctype_pubid = \
- xml_info["DOCTYPE"]
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: b7eec0cd3bb59c2b21e91a71619212dbff8a5b0b
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Tue May 3 09:01:29 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:41 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7eec0cd
repoman: Migrate from XmlLint to etree.XMLSchema for validation
Remove No longer used repoman._xml module
This change based on work by Dirkjan Ochtman gentoo.org>
Updated the change from XML.DTD to XMLSchema.
Additionally:
Move the metadata.xsd path determination code to metadata.py.
Add the DISTDIR backup location and fetching of the file if missing or
stale.
pym/repoman/_xml.py | 105 ---
pym/repoman/metadata.py | 21 +
pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 +-
pym/repoman/scanner.py | 10 +--
4 files changed, 26 insertions(+), 119 deletions(-)
diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
deleted file mode 100644
index 33a536a..000
--- a/pym/repoman/_xml.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
-import sys
-import xml
-
-# import our initialized portage instance
-from repoman._portage import portage
-
-from portage import os
-from portage.output import red
-from portage.process import find_binary
-
-from repoman.metadata import fetch_metadata_xsd
-from repoman._subprocess import repoman_getstatusoutput
-
-
-class _XMLParser(xml.etree.ElementTree.XMLParser):
-
- def __init__(self, data, **kwargs):
- xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
- self._portage_data = data
- if hasattr(self, 'parser'):
- self._base_XmlDeclHandler = self.parser.XmlDeclHandler
- self.parser.XmlDeclHandler =
self._portage_XmlDeclHandler
- self._base_StartDoctypeDeclHandler = \
- self.parser.StartDoctypeDeclHandler
- self.parser.StartDoctypeDeclHandler = \
- self._portage_StartDoctypeDeclHandler
-
- def _portage_XmlDeclHandler(self, version, encoding, standalone):
- if self._base_XmlDeclHandler is not None:
- self._base_XmlDeclHandler(version, encoding, standalone)
- self._portage_data["XML_DECLARATION"] = (version, encoding,
standalone)
-
- def _portage_StartDoctypeDeclHandler(
- self, doctypeName, systemId, publicId, has_internal_subset):
- if self._base_StartDoctypeDeclHandler is not None:
- self._base_StartDoctypeDeclHandler(
- doctypeName, systemId, publicId,
has_internal_subset)
- self._portage_data["DOCTYPE"] = (doctypeName, systemId,
publicId)
-
-
-class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
- """
- Implements doctype() as required to avoid deprecation warnings with
- >=python-2.7.
- """
- def doctype(self, name, pubid, system):
- pass
-
-
-class XmlLint(object):
-
- def __init__(self, options, repoman_settings, metadata_xsd=None):
- self.metadata_xsd = (metadata_xsd or
- os.path.join(repoman_settings["DISTDIR"],
'metadata.xsd'))
- self.options = options
- self.repoman_settings = repoman_settings
- self._is_capable = metadata_xsd is not None
- self.binary = None
- self._check_capable()
-
- def _check_capable(self):
- if self.options.mode == "manifest":
- return
- self.binary = find_binary('xmllint')
- if not self.binary:
- print(red("!!! xmllint not found. Can't check
metadata.xml.\n"))
- elif not self._is_capable:
- if not fetch_metadata_xsd(self.metadata_xsd,
self.repoman_settings):
- sys.exit(1)
- # this can be problematic if xmllint changes their
output
- self._is_capable = True
-
- @property
- def capable(self):
- return self._is_capable
-
- def check(self, checkdir, repolevel):
- '''Runs checks on the package metadata.xml file
-
- @param checkdir: string, path
- @param repolevel: integer
- @return boolean, False == bad metadata
- '''
- if not self.capable:
- if self.options.xml_parse or repolevel == 3:
- print("%s sorry, xmllint is needed. failing\n"
% red("!!!"))
- sys.exit(1)
- return True
- # xmlint can produce garbage output
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 82846cb35a214c51731fa701fab26119c1b3832d Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 07:39:09 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:41 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=82846cb3 repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks This change based on original work done by Dirkjan Ochtman ochtman.nl> pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 1 file changed, 12 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3ca7897..4921b6f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage from repoman.metadata import metadata_dtd_uri -from repoman.checks.herds.herdbase import get_herd_base -from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom @@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks): "%s/metadata.xml: Atom contains " "unexpected cat/pn: %s" % (xpkg, atom)) - # Run other metadata.xml checkers - try: - check_metadata(_metadata_xml, get_herd_base( - self.repoman_settings)) - except (UnknownHerdsError, ) as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 4d2cb398e48cdd90499dee48d52072e147c64e0f
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 17:38:19 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:42 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4d2cb398
repoman/modules/.../pkgmetadata.py: Improve whole document validation
Change to using assertValid() and add the error causing validation to fail to
the qa tracker
error.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 4921b6f..1594b27 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -206,8 +206,14 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# Only carry out if in package directory or check forced
if not metadata_bad:
validator = etree.XMLSchema(file=self.metadata_xsd)
- if not validator.validate(_metadata_xml):
- self.qatracker.add_error("metadata.bad", xpkg +
"/metadata.xml")
+ try:
+ validator.assertValid(_metadata_xml)
+ except etree.DocumentInvalid as error:
+ self.qatracker.add_error(
+ "metadata.bad",
+ xpkg + "/metadata.xml: %s"
+ % (str(error))
+ )
del metadata_bad
self.muselist = frozenset(self.musedict)
return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: b1a831bffd28fc1bb889cc90bb7690c7e5e2b8d9 Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:08:50 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:42 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b1a831bf repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection The docinfo is always created, so is not valid to use like a boolean to detect a missing doctype. doctype is a null string or equivalent and as such the correct choice for the if statement. Failure discoverd by: Göktürk Yüksek binghamton.edu> Thank you pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 83ca93f..d8344c2 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -106,7 +106,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): "xml declaration encoding should be '%s', not '%s'" % (xpkg, metadata_xml_encoding, xml_encoding)) - if not _metadata_xml.docinfo: + if not _metadata_xml.docinfo.doctype: metadata_bad = True self.qatracker.add_error( "metadata.bad",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/
commit: 8f738e14e6710481d1dc81ec3fa707b2d4ba9919
Author: Brian Dolbec gentoo org>
AuthorDate: Sun May 8 20:07:02 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 21:18:43 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8f738e14
repoman Comment out the $ID header check for now. bug 579460
Leave the code in place for now, it may be repo config enabled later.
pym/repoman/modules/scan/ebuild/checks.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/modules/scan/ebuild/checks.py
b/pym/repoman/modules/scan/ebuild/checks.py
index 8cdc230..fb3e019 100644
--- a/pym/repoman/modules/scan/ebuild/checks.py
+++ b/pym/repoman/modules/scan/ebuild/checks.py
@@ -108,8 +108,8 @@ class EbuildHeader(LineCheck):
return errors.COPYRIGHT_ERROR
elif num == 1 and line.rstrip('\n') != self.gentoo_license:
return errors.LICENSE_ERROR
- elif num == 2 and line.rstrip('\n') != self.id_header:
- return errors.ID_HEADER_ERROR
+ #elif num == 2 and line.rstrip('\n') != self.id_header:
+ # return errors.ID_HEADER_ERROR
class EbuildWhitespace(LineCheck):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 119a9e27ef5e3442205f2e9be8f379a10177d673 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:45:31 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:41 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=119a9e27 repoman: mandate use of python 2.7 or 3.2+ for simplicity pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index af53f4b..bcddb3e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -162,54 +162,48 @@ class PkgMetadata(ScanBase, USEFlagChecks): self.muselist = frozenset(self.musedict) return False - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True + if "DOCTYPE" not in xml_info: + metadata_bad = True + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) + else: + doctype_name, doctype_system, doctype_pubid = \ + xml_info["DOCTYPE"] + if doctype_system != metadata_dtd_uri: + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] - if doctype_system != metadata_dtd_uri: - if doctype_system is None: -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/
commit: c7900bdfe83de7f1279f1dc9c1f11a73fd597fd9
Author: Brian Dolbec gentoo org>
AuthorDate: Sun May 8 20:07:02 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 8 20:07:02 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c7900bdf
repoman Comment out the $ID header check for now. bug 579460
Leave the code in place for now, it may be repo config enabled later.
pym/repoman/modules/scan/ebuild/checks.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/modules/scan/ebuild/checks.py
b/pym/repoman/modules/scan/ebuild/checks.py
index 8cdc230..fb3e019 100644
--- a/pym/repoman/modules/scan/ebuild/checks.py
+++ b/pym/repoman/modules/scan/ebuild/checks.py
@@ -108,8 +108,8 @@ class EbuildHeader(LineCheck):
return errors.COPYRIGHT_ERROR
elif num == 1 and line.rstrip('\n') != self.gentoo_license:
return errors.LICENSE_ERROR
- elif num == 2 and line.rstrip('\n') != self.id_header:
- return errors.ID_HEADER_ERROR
+ #elif num == 2 and line.rstrip('\n') != self.id_header:
+ # return errors.ID_HEADER_ERROR
class EbuildWhitespace(LineCheck):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 6a61a12b827a0fbcdf6c111f936d8f17f6b10a33
Author: Brian Dolbec gentoo org>
AuthorDate: Thu May 5 16:04:54 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu May 5 16:04:54 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a61a12b
repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors
Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..433551a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
return uselist
def _add_validate_errors(self, xpkg, log):
+ listed = set()
for error in log:
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: line: %s, %s"
- % (xpkg, error.line, error.message))
+ msg_prefix = error.message.split(":",1)[0]
+ info = "%s %s" % (error.line, msg_prefix)
+ if info not in listed:
+ listed.add(info)
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: line: %s, %s"
+ % (xpkg, error.line, error.message))
@property
def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: afefcbc188ea25540d72224034668207d9f4e024
Author: Brian Dolbec gentoo org>
AuthorDate: Thu May 5 16:01:02 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu May 5 16:01:02 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=afefcbc1
repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors
Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..81e435e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
return uselist
def _add_validate_errors(self, xpkg, log):
+ listed = set()
for error in log:
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: line: %s, %s"
- % (xpkg, error.line, error.message))
+ msg_prefix = error.message.split(":",1)[0]
+ info = "%s %s" % (error.line, msg_prefix)
+ if info not in listed:
+ listed.add(info)
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: line: %s, %s, %s"
+ % (xpkg, error.line, error.type,
error.message))
@property
def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 9357614944df633df54871bc732d37abfe3a035f Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:23:05 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed May 4 06:23:05 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=93576149 repoman/modules/.../pkgmetadata.py: Add missing basestring compat code pym/repoman/modules/scan/metadata/pkgmetadata.py | 4 1 file changed, 4 insertions(+) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 8b31e3f..d8344c2 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -32,6 +32,10 @@ from portage.dep import Atom from .use_flags import USEFlagChecks +if sys.hexversion >= 0x300: + # pylint: disable=W0622 + basestring = str + metadata_xml_encoding = 'UTF-8' metadata_xml_declaration = '' \ % (metadata_xml_encoding,)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 55943b69a42c292bc0922a22d71f9d992ea035e7 Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:08:50 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed May 4 06:08:50 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=55943b69 repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection The docinfo is always created, so is not valid to use like a boolean to detect a missing doctype. doctype is a null string or equivalent and as such the correct choice for the if statement. Failure discoverd by: Göktürk Yüksek binghamton.edu> Thank you pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index c76922b..8b31e3f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -102,7 +102,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): "xml declaration encoding should be '%s', not '%s'" % (xpkg, metadata_xml_encoding, xml_encoding)) - if not _metadata_xml.docinfo: + if not _metadata_xml.docinfo.doctype: metadata_bad = True self.qatracker.add_error( "metadata.bad",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 378e5458a9a68231c8b7abc5df375eff8062a402 Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 03:44:17 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed May 4 03:44:17 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=378e5458 repoman/modules/.../pkgmetadata.py: Update import error message for lxml pkg pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 44b5edd..c76922b 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -14,7 +14,7 @@ except (SystemExit, KeyboardInterrupt): except (ImportError, SystemError, RuntimeError, Exception): # broken or missing xml support # http://bugs.python.org/issue14988 - msg = ["Please enable python's \"xml\" USE flag in order to use repoman."] + msg = ["Please emerge dev-python/lxml in order to use repoman."] from portage.output import EOutput out = EOutput() for line in msg:
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: bf7dc3159db2c15ff7aa61c2c72b143bbd420be0
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 20:54:33 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 20:54:33 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf7dc315
repoman/modules/.../pkgmetadata.py: Have xml validation log all qatracker
errors
Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++-
1 file changed, 36 insertions(+), 44 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d6d8557..44b5edd 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -126,8 +126,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
(xpkg, metadata_doctype_name,
doctype_name))
# load USE flags from metadata.xml
- self.musedict, metadata_bad = self._parse_metadata_use(
- _metadata_xml, xpkg, metadata_bad)
+ self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
for atom in chain(*self.musedict.values()):
if atom is None:
continue
@@ -147,15 +146,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# Only carry out if in package directory or check forced
if not metadata_bad:
validator = etree.XMLSchema(file=self.metadata_xsd)
- try:
- validator.assertValid(_metadata_xml)
- except etree.DocumentInvalid as error:
- self.qatracker.add_error(
- "metadata.bad",
- xpkg + "/metadata.xml: %s"
- % (str(error))
- )
- del metadata_bad
+ if not validator.validate(_metadata_xml):
+ self._add_validate_errors(xpkg,
validator.error_log)
self.muselist = frozenset(self.musedict)
return False
@@ -178,7 +170,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
% (xpkg, myflag))
return False
- def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+ def _parse_metadata_use(self, xml_tree, xpkg):
"""
Records are wrapped in XML as per GLEP 56
returns a dict with keys constisting of USE flag names and
values
@@ -188,7 +180,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
usetags = xml_tree.findall("use")
if not usetags:
- return uselist, metadata_bad
+ return uselist
# It's possible to have multiple 'use' elements.
for usetag in usetags:
@@ -199,37 +191,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
for flag in flags:
pkg_flag = flag.get("name")
- if pkg_flag is None:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: line: %s,
'%s', missing attribute: name"
- % (xpkg, flag.sourceline,
flag.text))
- continue
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from
python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 50de92c080ac8e5bc3f0d6cd2bc69fb919afc186
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 18:46:22 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 18:46:22 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=50de92c0
repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the
PkgMetadata class
This allows for a complete Q/A processing of all the use flags defined in the
xml.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++
1 file changed, 68 insertions(+), 70 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7390b5b..d6d8557 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
-from portage import exception
from portage.dep import Atom
from .use_flags import USEFlagChecks
@@ -39,55 +38,6 @@ metadata_xml_declaration = '' \
metadata_doctype_name = 'pkgmetadata'
-def parse_metadata_use(xml_tree):
- """
- Records are wrapped in XML as per GLEP 56
- returns a dict with keys constisting of USE flag names and values
- containing their respective descriptions
- """
- uselist = {}
-
- usetags = xml_tree.findall("use")
- if not usetags:
- return uselist
-
- # It's possible to have multiple 'use' elements.
- for usetag in usetags:
- flags = usetag.findall("flag")
- if not flags:
- # DTD allows use elements containing no flag elements.
- continue
-
- for flag in flags:
- pkg_flag = flag.get("name")
- if pkg_flag is None:
- raise exception.ParseError("missing 'name'
attribute for 'flag' tag")
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
- # (flag_restrict can be None)
- uselist[pkg_flag][flag_restrict] = "
".join("".join(inner_text).split())
-
- return uselist
-
-
class PkgMetadata(ScanBase, USEFlagChecks):
'''Package metadata.xml checks'''
@@ -176,28 +126,23 @@ class PkgMetadata(ScanBase, USEFlagChecks):
(xpkg, metadata_doctype_name,
doctype_name))
# load USE flags from metadata.xml
- try:
- self.musedict = parse_metadata_use(_metadata_xml)
- except portage.exception.ParseError as e:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad", "%s/metadata.xml: %s" % (xpkg,
e))
- else:
- for atom in chain(*self.musedict.values()):
- if atom is None:
- continue
- try:
- atom = Atom(atom)
- except InvalidAtom as e:
+ self.musedict, metadata_bad = self._parse_metadata_use(
+ _metadata_xml, xpkg, metadata_bad)
+ for atom in chain(*self.musedict.values()):
+ if atom is None:
+ continue
+ try:
+ atom = Atom(atom)
+ except InvalidAtom as e:
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: Invalid atom: %s" %
(xpkg, e))
+ else:
+ if atom.cp != xpkg:
self.qatracker.add_error(
"metadata.bad",
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 235235c69343e95793525ee2104a82977a5a6729
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 17:42:22 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 17:42:22 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=235235c6
repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the
PkgMetadata class
This allows for a complete Q/A processing of all the use flags defined in the
xml.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 139 +++
1 file changed, 68 insertions(+), 71 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7390b5b..ef57a5d 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
-from portage import exception
from portage.dep import Atom
from .use_flags import USEFlagChecks
@@ -39,55 +38,6 @@ metadata_xml_declaration = '' \
metadata_doctype_name = 'pkgmetadata'
-def parse_metadata_use(xml_tree):
- """
- Records are wrapped in XML as per GLEP 56
- returns a dict with keys constisting of USE flag names and values
- containing their respective descriptions
- """
- uselist = {}
-
- usetags = xml_tree.findall("use")
- if not usetags:
- return uselist
-
- # It's possible to have multiple 'use' elements.
- for usetag in usetags:
- flags = usetag.findall("flag")
- if not flags:
- # DTD allows use elements containing no flag elements.
- continue
-
- for flag in flags:
- pkg_flag = flag.get("name")
- if pkg_flag is None:
- raise exception.ParseError("missing 'name'
attribute for 'flag' tag")
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
- # (flag_restrict can be None)
- uselist[pkg_flag][flag_restrict] = "
".join("".join(inner_text).split())
-
- return uselist
-
-
class PkgMetadata(ScanBase, USEFlagChecks):
'''Package metadata.xml checks'''
@@ -176,32 +126,26 @@ class PkgMetadata(ScanBase, USEFlagChecks):
(xpkg, metadata_doctype_name,
doctype_name))
# load USE flags from metadata.xml
- try:
- self.musedict = parse_metadata_use(_metadata_xml)
- except portage.exception.ParseError as e:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad", "%s/metadata.xml: %s" % (xpkg,
e))
- else:
- for atom in chain(*self.musedict.values()):
- if atom is None:
- continue
- try:
- atom = Atom(atom)
- except InvalidAtom as e:
+ self.musedict, metadata_bad = self._parse_metadata_use(
+ _metadata_xml, xpkg, metadata_bad)
+ for atom in chain(*self.musedict.values()):
+ if atom is None:
+ continue
+ try:
+ atom = Atom(atom)
+ except InvalidAtom as e:
+ self.qatracker.add_error(
+ "metadata.bad",
+ "%s/metadata.xml: Invalid atom: %s" %
(xpkg, e))
+ else:
+ if atom.cp != xpkg:
self.qatracker.add_error(
"metadata.bad",
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 621a6852099848474ed2c0880eae52ebd85f1703
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 17:38:19 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 17:38:19 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=621a6852
repoman/modules/.../pkgmetadata.py: Improve whole document validation
Change to using assertValid() and add the error causing validation to fail to
the qa tracker
error.
pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index b231370..7390b5b 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -202,8 +202,14 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# Only carry out if in package directory or check forced
if not metadata_bad:
validator = etree.XMLSchema(file=self.metadata_xsd)
- if not validator.validate(_metadata_xml):
- self.qatracker.add_error("metadata.bad", xpkg +
"/metadata.xml")
+ try:
+ validator.assertValid(_metadata_xml)
+ except etree.DocumentInvalid as error:
+ self.qatracker.add_error(
+ "metadata.bad",
+ xpkg + "/metadata.xml: %s"
+ % (str(error))
+ )
del metadata_bad
self.muselist = frozenset(self.musedict)
return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 57ad076b08d417bf5d2642131b76392a990f3c35 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:45:31 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 06:55:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=57ad076b repoman: mandate use of python 2.7 or 3.2+ for simplicity pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index cde2ba0..e8db92f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -158,54 +158,48 @@ class PkgMetadata(ScanBase, USEFlagChecks): self.muselist = frozenset(self.musedict) return False - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True + if "DOCTYPE" not in xml_info: + metadata_bad = True + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) + else: + doctype_name, doctype_system, doctype_pubid = \ + xml_info["DOCTYPE"] + if doctype_system != metadata_dtd_uri: + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] - if doctype_system != metadata_dtd_uri: - if doctype_system is None: -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 75b897d91874c505e14dc2c0808c4cad4268ad76
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 07:18:05 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 09:12:30 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=75b897d9
repoman: Use lxml for parsing of metadata
Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).
pym/repoman/modules/scan/metadata/pkgmetadata.py | 44 +++-
1 file changed, 12 insertions(+), 32 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index e8db92f..22afddf 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,8 +7,8 @@ import sys
from itertools import chain
try:
- import xml.etree.ElementTree
- from xml.parsers.expat import ExpatError
+ from lxml import etree
+ from lxml.etree import ParserError
except (SystemExit, KeyboardInterrupt):
raise
except (ImportError, SystemError, RuntimeError, Exception):
@@ -26,12 +26,11 @@ from repoman._portage import portage
from repoman.metadata import metadata_dtd_uri
from repoman.checks.herds.herdbase import get_herd_base
from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
-from portage import _encodings, _unicode_encode
from portage import exception
from portage.dep import Atom
@@ -141,50 +140,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# metadata.xml parse check
metadata_bad = False
- xml_info = {}
- xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
# read metadata.xml into memory
try:
- _metadata_xml = xml.etree.ElementTree.parse(
- _unicode_encode(
- os.path.join(checkdir, "metadata.xml"),
- encoding=_encodings['fs'],
errors='strict'),
- parser=xml_parser)
- except (ExpatError, SyntaxError, EnvironmentError) as e:
+ _metadata_xml = etree.parse(os.path.join(checkdir,
'metadata.xml'))
+ except (ParserError, SyntaxError, EnvironmentError) as e:
metadata_bad = True
self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
del e
self.muselist = frozenset(self.musedict)
return False
- if "XML_DECLARATION" not in xml_info:
+ xml_encoding = _metadata_xml.docinfo.encoding
+ if xml_encoding.upper() != metadata_xml_encoding:
self.qatracker.add_error(
"metadata.bad", "%s/metadata.xml: "
- "xml declaration is missing on first line, "
- "should be '%s'" % (xpkg,
metadata_xml_declaration))
- else:
- xml_version, xml_encoding, xml_standalone = \
- xml_info["XML_DECLARATION"]
- if xml_encoding is None or \
- xml_encoding.upper() != metadata_xml_encoding:
- if xml_encoding is None:
- encoding_problem = "but it is undefined"
- else:
- encoding_problem = "not '%s'" %
xml_encoding
- self.qatracker.add_error(
- "metadata.bad", "%s/metadata.xml: "
- "xml declaration encoding should be
'%s', %s" %
- (xpkg, metadata_xml_encoding,
encoding_problem))
+ "xml declaration encoding should be '%s', not
'%s'" %
+ (xpkg, metadata_xml_encoding, xml_encoding))
- if "DOCTYPE" not in xml_info:
+ if not _metadata_xml.docinfo:
metadata_bad = True
self.qatracker.add_error(
"metadata.bad",
"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is
missing"))
else:
- doctype_name, doctype_system, doctype_pubid = \
- xml_info["DOCTYPE"]
+ doctype_system = _metadata_xml.docinfo.system_url
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 4ef6bc69a3fe62d980c285ac0b9bb5bbadeefe9b
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 06:33:47 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 06:33:47 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4ef6bc69
repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors
pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 ---
1 file changed, 75 insertions(+), 73 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3c1c2d0..3b48b8e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks):
metadata_bad = True
self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
del e
+ self.muselist = frozenset(self.musedict)
+ return False
+
+ if not hasattr(xml_parser, 'parser') or \
+ sys.hexversion < 0x207 or \
+ (sys.hexversion > 0x300 and sys.hexversion <
0x302):
+ # doctype is not parsed with python 2.6 or 3.1
+ pass
else:
- if not hasattr(xml_parser, 'parser') or \
- sys.hexversion < 0x207 or \
- (sys.hexversion > 0x300 and sys.hexversion
< 0x302):
- # doctype is not parsed with python 2.6 or 3.1
- pass
+ if "XML_DECLARATION" not in xml_info:
+ self.qatracker.add_error(
+ "metadata.bad", "%s/metadata.xml: "
+ "xml declaration is missing on first
line, "
+ "should be '%s'" % (xpkg,
metadata_xml_declaration))
else:
- if "XML_DECLARATION" not in xml_info:
+ xml_version, xml_encoding, xml_standalone = \
+ xml_info["XML_DECLARATION"]
+ if xml_encoding is None or \
+ xml_encoding.upper() !=
metadata_xml_encoding:
+ if xml_encoding is None:
+ encoding_problem = "but it is
undefined"
+ else:
+ encoding_problem = "not '%s'" %
xml_encoding
self.qatracker.add_error(
"metadata.bad",
"%s/metadata.xml: "
- "xml declaration is missing on
first line, "
- "should be '%s'" % (xpkg,
metadata_xml_declaration))
- else:
- xml_version, xml_encoding,
xml_standalone = \
- xml_info["XML_DECLARATION"]
- if xml_encoding is None or \
- xml_encoding.upper() !=
metadata_xml_encoding:
- if xml_encoding is None:
- encoding_problem = "but
it is undefined"
- else:
- encoding_problem = "not
'%s'" % xml_encoding
- self.qatracker.add_error(
- "metadata.bad",
"%s/metadata.xml: "
- "xml declaration
encoding should be '%s', %s" %
- (xpkg,
metadata_xml_encoding, encoding_problem))
+ "xml declaration encoding
should be '%s', %s" %
+ (xpkg, metadata_xml_encoding,
encoding_problem))
- if "DOCTYPE" not in xml_info:
- metadata_bad = True
- self.qatracker.add_error(
- "metadata.bad",
- "%s/metadata.xml: %s" % (xpkg,
"DOCTYPE is missing"))
- else:
- doctype_name, doctype_system,
doctype_pubid = \
- xml_info["DOCTYPE"]
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/metadata/
commit: 52bce5a348c68a5af0def40f399943d3b1d4f509
Author: Brian Dolbec gentoo org>
AuthorDate: Tue May 3 09:01:29 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 09:13:56 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=52bce5a3
repoman: Migrate from XmlLint to etree.XMLSchema for validation
This change based on work by Dirkjan Ochtman gentoo.org>
Updated the change from XML.DTD to XMLSchema.
Additionally:
Move the metadata.xsd path determination code to metadata.py.
Add the DISTDIR backup location and fetching of the file if missing or
stale.
pym/repoman/metadata.py | 21 +
pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 +++--
pym/repoman/scanner.py | 10 ++
3 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index 7c64c8e..a9ad3e8 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -99,3 +99,24 @@ def fetch_metadata_xsd(metadata_xsd, repoman_settings):
pass
return True
+
+
+def get_metadata_xsd(repo_settings):
+ '''Locate and or fetch the metadata.xsd file
+
+ @param repo_settings: RepoSettings instance
+ @returns: path to the metadata.xsd file
+ '''
+ metadata_xsd = None
+ for path in reversed(repo_settings.repo_config.eclass_db.porttrees):
+ path = os.path.join(path, 'metadata/xml-schema/metadata.xsd')
+ if os.path.exists(path):
+ metadata_xsd = path
+ break
+ if metadata_xsd is None:
+ metadata_xsd = os.path.join(
+ repo_settings.repoman_settings["DISTDIR"],
'metadata.xsd'
+ )
+
+ fetch_metadata_xsd(metadata_xsd, repo_settings.repoman_settings)
+ return metadata_xsd
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index c744c13..b231370 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -24,7 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception):
# import our initialized portage instance
from repoman._portage import portage
from repoman.metadata import metadata_dtd_uri
-from repoman._xml import XmlLint
from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
@@ -104,13 +103,11 @@ class PkgMetadata(ScanBase, USEFlagChecks):
repo_settings = kwargs.get('repo_settings')
self.qatracker = kwargs.get('qatracker')
self.options = kwargs.get('options')
- metadata_xsd = kwargs.get('metadata_xsd')
+ self.metadata_xsd = kwargs.get('metadata_xsd')
self.globalUseFlags = kwargs.get('uselist')
self.repoman_settings = repo_settings.repoman_settings
self.musedict = {}
self.muselist = set()
- self.xmllint = XmlLint(self.options, self.repoman_settings,
- metadata_xsd=metadata_xsd)
def check(self, **kwargs):
'''Performs the checks on the metadata.xml for the package
@@ -123,7 +120,6 @@ class PkgMetadata(ScanBase, USEFlagChecks):
xpkg = kwargs.get('xpkg')
checkdir = kwargs.get('checkdir')
checkdirlist = kwargs.get('checkdirlist').get()
- repolevel = kwargs.get('repolevel')
self.musedict = {}
if self.options.mode in ['manifest']:
@@ -205,7 +201,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# Only carry out if in package directory or check forced
if not metadata_bad:
- if not self.xmllint.check(checkdir, repolevel):
+ validator = etree.XMLSchema(file=self.metadata_xsd)
+ if not validator.validate(_metadata_xml):
self.qatracker.add_error("metadata.bad", xpkg +
"/metadata.xml")
del metadata_bad
self.muselist = frozenset(self.musedict)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index fd07209..48d9001 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -10,6 +10,7 @@ from portage import normalize_path
from portage import os
from portage.output import green
from portage.util.futures.extendedfutures import ExtendedFuture
+from repoman.metadata import get_metadata_xsd
from repoman.modules.commit import repochecks
from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
from repoman.repos import repo_metadata
@@ -56,13 +57,6 @@ class Scanner(object):
portage.util.stack_lists([self.categories],
incremental=1))
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: c26a6f10faed17ba08bbed946e50b8492ed975e0
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Mon Jan 25 20:37:13 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 09:15:26 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c26a6f10
repoman: Remove No longer used repoman._xml module
pym/repoman/_xml.py | 105
1 file changed, 105 deletions(-)
diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
deleted file mode 100644
index 33a536a..000
--- a/pym/repoman/_xml.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
-import sys
-import xml
-
-# import our initialized portage instance
-from repoman._portage import portage
-
-from portage import os
-from portage.output import red
-from portage.process import find_binary
-
-from repoman.metadata import fetch_metadata_xsd
-from repoman._subprocess import repoman_getstatusoutput
-
-
-class _XMLParser(xml.etree.ElementTree.XMLParser):
-
- def __init__(self, data, **kwargs):
- xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
- self._portage_data = data
- if hasattr(self, 'parser'):
- self._base_XmlDeclHandler = self.parser.XmlDeclHandler
- self.parser.XmlDeclHandler =
self._portage_XmlDeclHandler
- self._base_StartDoctypeDeclHandler = \
- self.parser.StartDoctypeDeclHandler
- self.parser.StartDoctypeDeclHandler = \
- self._portage_StartDoctypeDeclHandler
-
- def _portage_XmlDeclHandler(self, version, encoding, standalone):
- if self._base_XmlDeclHandler is not None:
- self._base_XmlDeclHandler(version, encoding, standalone)
- self._portage_data["XML_DECLARATION"] = (version, encoding,
standalone)
-
- def _portage_StartDoctypeDeclHandler(
- self, doctypeName, systemId, publicId, has_internal_subset):
- if self._base_StartDoctypeDeclHandler is not None:
- self._base_StartDoctypeDeclHandler(
- doctypeName, systemId, publicId,
has_internal_subset)
- self._portage_data["DOCTYPE"] = (doctypeName, systemId,
publicId)
-
-
-class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
- """
- Implements doctype() as required to avoid deprecation warnings with
- >=python-2.7.
- """
- def doctype(self, name, pubid, system):
- pass
-
-
-class XmlLint(object):
-
- def __init__(self, options, repoman_settings, metadata_xsd=None):
- self.metadata_xsd = (metadata_xsd or
- os.path.join(repoman_settings["DISTDIR"],
'metadata.xsd'))
- self.options = options
- self.repoman_settings = repoman_settings
- self._is_capable = metadata_xsd is not None
- self.binary = None
- self._check_capable()
-
- def _check_capable(self):
- if self.options.mode == "manifest":
- return
- self.binary = find_binary('xmllint')
- if not self.binary:
- print(red("!!! xmllint not found. Can't check
metadata.xml.\n"))
- elif not self._is_capable:
- if not fetch_metadata_xsd(self.metadata_xsd,
self.repoman_settings):
- sys.exit(1)
- # this can be problematic if xmllint changes their
output
- self._is_capable = True
-
- @property
- def capable(self):
- return self._is_capable
-
- def check(self, checkdir, repolevel):
- '''Runs checks on the package metadata.xml file
-
- @param checkdir: string, path
- @param repolevel: integer
- @return boolean, False == bad metadata
- '''
- if not self.capable:
- if self.options.xml_parse or repolevel == 3:
- print("%s sorry, xmllint is needed. failing\n"
% red("!!!"))
- sys.exit(1)
- return True
- # xmlint can produce garbage output even on success, so only
dump
- # the ouput when it fails.
- st, out = repoman_getstatusoutput(
- self.binary + " --nonet --noout --schema %s %s" % (
- portage._shell_quote(self.metadata_xsd),
- portage._shell_quote(
- os.path.join(checkdir,
"metadata.xml"
- if st != os.EX_OK:
- print(red("!!!") + " metadata.xml is invalid:")
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: e66ee7b35405a650c517939e503443948b660ca9
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Mon Jan 25 18:33:32 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 06:25:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e66ee7b3
repoman/modules/.../pkgmetadata.py: Early return in metadata check code for
readability
pym/repoman/modules/scan/metadata/pkgmetadata.py | 190 ---
1 file changed, 96 insertions(+), 94 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 5c6452a..3c1c2d0 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -83,112 +83,114 @@ class PkgMetadata(ScanBase, USEFlagChecks):
# metadata.xml file check
if "metadata.xml" not in checkdirlist:
self.qatracker.add_error("metadata.missing", xpkg +
"/metadata.xml")
+ self.muselist = frozenset(self.musedict)
+ return False
+
# metadata.xml parse check
+ metadata_bad = False
+ xml_info = {}
+ xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
+
+ # read metadata.xml into memory
+ try:
+ _metadata_xml = xml.etree.ElementTree.parse(
+ _unicode_encode(
+ os.path.join(checkdir, "metadata.xml"),
+ encoding=_encodings['fs'],
errors='strict'),
+ parser=xml_parser)
+ except (ExpatError, SyntaxError, EnvironmentError) as e:
+ metadata_bad = True
+ self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
+ del e
else:
- metadata_bad = False
- xml_info = {}
- xml_parser = _XMLParser(xml_info,
target=_MetadataTreeBuilder())
-
- # read metadata.xml into memory
- try:
- _metadata_xml = xml.etree.ElementTree.parse(
- _unicode_encode(
- os.path.join(checkdir,
"metadata.xml"),
- encoding=_encodings['fs'],
errors='strict'),
- parser=xml_parser)
- except (ExpatError, SyntaxError, EnvironmentError) as e:
- metadata_bad = True
- self.qatracker.add_error("metadata.bad",
"%s/metadata.xml: %s" % (xpkg, e))
- del e
+ if not hasattr(xml_parser, 'parser') or \
+ sys.hexversion < 0x207 or \
+ (sys.hexversion > 0x300 and sys.hexversion
< 0x302):
+ # doctype is not parsed with python 2.6 or 3.1
+ pass
else:
- if not hasattr(xml_parser, 'parser') or \
- sys.hexversion < 0x207 or \
- (sys.hexversion > 0x300 and
sys.hexversion < 0x302):
- # doctype is not parsed with python 2.6
or 3.1
- pass
+ if "XML_DECLARATION" not in xml_info:
+ self.qatracker.add_error(
+ "metadata.bad",
"%s/metadata.xml: "
+ "xml declaration is missing on
first line, "
+ "should be '%s'" % (xpkg,
metadata_xml_declaration))
else:
- if "XML_DECLARATION" not in xml_info:
+ xml_version, xml_encoding,
xml_standalone = \
+ xml_info["XML_DECLARATION"]
+ if xml_encoding is None or \
+ xml_encoding.upper() !=
metadata_xml_encoding:
+ if xml_encoding is None:
+ encoding_problem = "but
it is undefined"
+ else:
+ encoding_problem = "not
'%s'" % xml_encoding
self.qatracker.add_error(
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/metadata/
commit: 2ec2486fd56b4e673dbf544c5667dd7cf9d683bf Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 18:42:41 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 06:38:34 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ec2486f repoman: Move some metdata_* variables to module it's used pym/repoman/metadata.py | 4 pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index 7a514dc..a980184 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -28,10 +28,6 @@ if sys.hexversion >= 0x300: if sys.hexversion >= 0x300: basestring = str -metadata_xml_encoding = 'UTF-8' -metadata_xml_declaration = '' \ - % (metadata_xml_encoding,) -metadata_doctype_name = 'pkgmetadata' metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' # force refetch if the local copy creation time is older than this diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3b48b8e..a7150a9 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -23,9 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage -from repoman.metadata import ( - metadata_xml_encoding, metadata_doctype_name, - metadata_dtd_uri, metadata_xml_declaration, parse_metadata_use) +from repoman.metadata import metadata_dtd_uri, parse_metadata_use from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint @@ -38,6 +36,11 @@ from portage.dep import Atom from .use_flags import USEFlagChecks +metadata_xml_encoding = 'UTF-8' +metadata_xml_declaration = '' \ + % (metadata_xml_encoding,) +metadata_doctype_name = 'pkgmetadata' + class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks'''
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: fdb0fd395a582996cf88df2819bf246a2ddcf914
Author: Dirkjan Ochtman ochtman nl>
AuthorDate: Mon Jan 25 19:40:29 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue May 3 06:54:24 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fdb0fd39
repoman: Move parse_metadata_use function to the pkgmetadata.py
pym/repoman/metadata.py | 50 ---
pym/repoman/modules/scan/metadata/pkgmetadata.py | 52 +++-
2 files changed, 51 insertions(+), 51 deletions(-)
diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index a980184..7c64c8e 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -17,7 +17,6 @@ except ImportError:
# import our initialized portage instance
from repoman._portage import portage
-from portage import exception
from portage import os
from portage import shutil
from portage.output import green
@@ -34,55 +33,6 @@ metadata_xsd_uri =
'http://www.gentoo.org/xml-schema/metadata.xsd'
metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
-def parse_metadata_use(xml_tree):
- """
- Records are wrapped in XML as per GLEP 56
- returns a dict with keys constisting of USE flag names and values
- containing their respective descriptions
- """
- uselist = {}
-
- usetags = xml_tree.findall("use")
- if not usetags:
- return uselist
-
- # It's possible to have multiple 'use' elements.
- for usetag in usetags:
- flags = usetag.findall("flag")
- if not flags:
- # DTD allows use elements containing no flag elements.
- continue
-
- for flag in flags:
- pkg_flag = flag.get("name")
- if pkg_flag is None:
- raise exception.ParseError("missing 'name'
attribute for 'flag' tag")
- flag_restrict = flag.get("restrict")
-
- # emulate the Element.itertext() method from python-2.7
- inner_text = []
- stack = []
- stack.append(flag)
- while stack:
- obj = stack.pop()
- if isinstance(obj, basestring):
- inner_text.append(obj)
- continue
- if isinstance(obj.text, basestring):
- inner_text.append(obj.text)
- if isinstance(obj.tail, basestring):
- stack.append(obj.tail)
- stack.extend(reversed(obj))
-
- if pkg_flag not in uselist:
- uselist[pkg_flag] = {}
-
- # (flag_restrict can be None)
- uselist[pkg_flag][flag_restrict] = "
".join("".join(inner_text).split())
-
- return uselist
-
-
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
Fetch metadata.xsd if it doesn't exist or the ctime is older than
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index a7150a9..cde2ba0 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -23,7 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception):
# import our initialized portage instance
from repoman._portage import portage
-from repoman.metadata import metadata_dtd_uri, parse_metadata_use
+from repoman.metadata import metadata_dtd_uri
from repoman.checks.herds.herdbase import get_herd_base
from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
@@ -32,6 +32,7 @@ from repoman.modules.scan.scanbase import ScanBase
from portage.exception import InvalidAtom
from portage import os
from portage import _encodings, _unicode_encode
+from portage import exception
from portage.dep import Atom
from .use_flags import USEFlagChecks
@@ -42,6 +43,55 @@ metadata_xml_declaration = '' \
metadata_doctype_name = 'pkgmetadata'
+def parse_metadata_use(xml_tree):
+ """
+ Records are wrapped in XML as per GLEP 56
+ returns a dict with keys constisting of USE flag names and values
+ containing their respective descriptions
+ """
+ uselist = {}
+
+ usetags = xml_tree.findall("use")
+ if not usetags:
+ return uselist
+
+ # It's possible to have multiple 'use' elements.
+ for usetag in usetags:
+ flags = usetag.findall("flag")
+ if not flags:
+ # DTD allows use elements containing no flag elements.
+
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 6ca71f4c66e45d86503f1cc199e1547e19b35a2e Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 07:39:09 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 09:12:55 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ca71f4c repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks This change based on original work done by Dirkjan Ochtman ochtman.nl> pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 1 file changed, 12 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 22afddf..c744c13 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage from repoman.metadata import metadata_dtd_uri -from repoman.checks.herds.herdbase import get_herd_base -from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import XmlLint from repoman.modules.scan.scanbase import ScanBase @@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks): "%s/metadata.xml: Atom contains " "unexpected cat/pn: %s" % (xpkg, atom)) - # Run other metadata.xml checkers - try: - check_metadata(_metadata_xml, get_herd_base( - self.repoman_settings)) - except (UnknownHerdsError, ) as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - # Only carry out if in package directory or check forced if not metadata_bad: if not self.xmllint.check(checkdir, repolevel):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/vcs/cvs/, pym/repoman/modules/vcs/
commit: d715028a686513979e6fb2b270df5f825d30cb65
Author: Zac Medico gentoo org>
AuthorDate: Sun May 1 03:31:04 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun May 1 03:32:07 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d715028a
repoman: handle removed packages in vcs_files_to_cps (bug 546010)
X-Gentoo-Bug: 546010
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=546010
pym/repoman/actions.py | 1 +
pym/repoman/modules/vcs/cvs/changes.py | 1 +
pym/repoman/modules/vcs/vcs.py | 7 +--
pym/repoman/scanner.py | 1 +
4 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/actions.py b/pym/repoman/actions.py
index 643e24c..4144b45 100644
--- a/pym/repoman/actions.py
+++ b/pym/repoman/actions.py
@@ -370,6 +370,7 @@ class Actions(object):
committer_name =
utilities.get_committer_name(env=self.repoman_settings)
for x in sorted(vcs_files_to_cps(
chain(myupdates, mymanifests, myremoved),
+ self.repo_settings.repodir,
self.scanner.repolevel, self.scanner.reposplit,
self.scanner.categories)):
catdir, pkgdir = x.split("/")
checkdir = self.repo_settings.repodir + "/" + x
diff --git a/pym/repoman/modules/vcs/cvs/changes.py
b/pym/repoman/modules/vcs/cvs/changes.py
index 3e2f97a..c3d880b 100644
--- a/pym/repoman/modules/vcs/cvs/changes.py
+++ b/pym/repoman/modules/vcs/cvs/changes.py
@@ -112,6 +112,7 @@ class Changes(ChangesBase):
if updates or removed:
for x in sorted(vcs_files_to_cps(
chain(updates, removed, manifests),
+ self.repo_settings.repodir,
scanner.repolevel, scanner.reposplit,
scanner.categories)):
self.repoman_settings["O"] =
os.path.join(self.repo_settings.repodir, x)
digestgen(mysettings=self.repoman_settings,
myportdb=self.repo_settings.portdb)
diff --git a/pym/repoman/modules/vcs/vcs.py b/pym/repoman/modules/vcs/vcs.py
index c8cb55d..e9d45d4 100644
--- a/pym/repoman/modules/vcs/vcs.py
+++ b/pym/repoman/modules/vcs/vcs.py
@@ -96,7 +96,7 @@ def FindVCS(cwd=None):
return outvcs
-def vcs_files_to_cps(vcs_file_iter, repolevel, reposplit, categories):
+def vcs_files_to_cps(vcs_file_iter, repodir, repolevel, reposplit, categories):
"""
Iterate over the given modified file paths returned from the vcs,
and return a frozenset containing category/pn strings for each
@@ -127,7 +127,10 @@ def vcs_files_to_cps(vcs_file_iter, repolevel, reposplit,
categories):
if len(f_split) > 3 and f_split[1] in categories:
modified_cps.append("/".join(f_split[1:3]))
- return frozenset(modified_cps)
+ # Exclude packages that have been removed, since calling
+ # code assumes that the packages exist.
+ return frozenset(x for x in frozenset(modified_cps)
+ if os.path.exists(os.path.join(repodir, x)))
def vcs_new_changed(relative_path, mychanged, mynew):
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index e383c8d..fd07209 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -177,6 +177,7 @@ class Scanner(object):
if self.options.if_modified == "y":
self.effective_scanlist = sorted(vcs_files_to_cps(
chain(self.changed.changed, self.changed.new,
self.changed.removed),
+ self.repo_settings.repodir,
self.repolevel, self.reposplit,
self.categories))
# Create our kwargs dict here to initialize the plugins with
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/modules/scan/keywords/, ...
commit: c65d9bc746b4c6774eb71332ab851ec3dc6f3675
Author: Brian Dolbec gentoo org>
AuthorDate: Thu Apr 28 05:11:57 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 28 14:50:42 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c65d9bc7
repoman: Move the LiveEclasses.is_live() to the Ebuild class
This removes another dynaic_data variable.
It also belongs with the ebuild eclass as a primary source for that type of
data.
pym/repoman/modules/scan/ebuild/ebuild.py | 9 +
pym/repoman/modules/scan/eclasses/__init__.py | 1 -
pym/repoman/modules/scan/eclasses/live.py | 22 +++---
pym/repoman/modules/scan/keywords/__init__.py | 1 -
pym/repoman/modules/scan/keywords/keywords.py | 8 +++-
pym/repoman/modules/scan/metadata/__init__.py | 1 -
.../modules/scan/metadata/ebuild_metadata.py | 3 +--
7 files changed, 16 insertions(+), 29 deletions(-)
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index c247a7f..0277aa9 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -13,6 +13,7 @@ from repoman.qa_data import no_exec, allvars
# import our initialized portage instance
from repoman._portage import portage
from portage import os
+from portage.const import LIVE_ECLASSES
pv_toolong_re = re.compile(r'[0-9]{19,}')
@@ -220,6 +221,14 @@ class Ebuild(ScanBase):
return self.continue_
@property
+ def is_live(self):
+ '''Test if the ebuild inherits a live eclass
+
+ @returns: boolean
+ '''
+ return set(LIVE_ECLASSES.intersection(self.inherited))
+
+ @property
def runInPkgs(self):
'''Package level scans'''
return (True, [self.check_isebuild])
diff --git a/pym/repoman/modules/scan/eclasses/__init__.py
b/pym/repoman/modules/scan/eclasses/__init__.py
index 63bb86f..78d46e4 100644
--- a/pym/repoman/modules/scan/eclasses/__init__.py
+++ b/pym/repoman/modules/scan/eclasses/__init__.py
@@ -22,7 +22,6 @@ module_spec = {
],
'func_kwargs': {
'ebuild': (None, None),
- 'live_ebuild': ('Future', 'UNSET'),
'pkg': (None, None),
'xpkg': (None, None),
'y_ebuild': (None, None),
diff --git a/pym/repoman/modules/scan/eclasses/live.py
b/pym/repoman/modules/scan/eclasses/live.py
index 842cbab..1ce33c0 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -6,8 +6,6 @@ Performs Live eclass checks
from repoman._portage import portage
from repoman.modules.scan.scanbase import ScanBase
-from portage.const import LIVE_ECLASSES
-
class LiveEclassChecks(ScanBase):
'''Performs checks for the usage of Live eclasses in ebuilds'''
@@ -20,27 +18,14 @@ class LiveEclassChecks(ScanBase):
self.pmaskdict = kwargs.get('repo_metadata')['pmaskdict']
self.repo_settings = kwargs.get('repo_settings')
- def is_live(self, **kwargs):
- '''Test if the ebuild inherits a live eclass
-
- @returns: dictionary, including {live_ebuild}
- '''
- ebuild = kwargs.get('ebuild').get()
- # update the dynamic data
- dyn_live = kwargs.get('live_ebuild')
- dyn_live.set(LIVE_ECLASSES.intersection(ebuild.inherited))
- return False
-
def check(self, **kwargs):
'''Ebuilds that inherit a "Live" eclass (darcs, subversion,
git, cvs,
etc..) should not be allowed to be marked stable
@param pkg: Package in which we check (object).
- @param package: Package in which we check (string).
+ @param xpkg: Package in which we check (string).
@param ebuild: Ebuild which we check (object).
@param y_ebuild: Ebuild which we check (string).
- @param keywords: The keywords of the ebuild.
- @param global_pmaskdict: A global dictionary of all the masks.
@returns: dictionary
'''
pkg = kwargs.get("pkg").result()
@@ -48,9 +33,8 @@ class LiveEclassChecks(ScanBase):
ebuild = kwargs.get('ebuild').get()
y_ebuild = kwargs.get('y_ebuild')
keywords = ebuild.keywords
- live_ebuild = kwargs.get('live_ebuild').get()
- if not live_ebuild and self.repo_settings.repo_config.name ==
"gentoo":
+ if not ebuild.is_live and self.repo_settings.repo_config.name
== "gentoo":
return False
is_stable = lambda kw: not
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 5cb92a836ed539b7dd5952b7404b5c7f473b844a
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 27 03:22:37 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 28 14:50:41 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5cb92a83
repoman/scanner.py: Clean up some debug prints
pym/repoman/scanner.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index f1a0231..e5e5717 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -260,17 +260,14 @@ class Scanner(object):
@param dynamic_data: dictionary
@param key: tuple of (dictionary-key, default-value)
'''
- #print("***", key, data)
if data[0] in ['Future', 'ExtendedFuture']:
if data[1] in ['UNSET']:
- #print("adding unset default")
dynamic_data[key] = ExtendedFuture()
else:
if data[1] in DATA_TYPES:
default = DATA_TYPES[data[1]]()
else:
default = data[1]
- #print("adding default:", default)
dynamic_data[key] = ExtendedFuture(default)
def scan_pkgs(self, can_force):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: dc684ee6e4b1f0b8bef7babdf13a1143046c87c5
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 27 03:20:45 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 28 14:50:41 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dc684ee6
repoman/scanner.py: Mark self.ext_futures private
pym/repoman/scanner.py | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a5c54b1..f1a0231 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -200,7 +200,7 @@ class Scanner(object):
}
# initialize the plugin checks here
self.modules = {}
- self.ext_futures = {}
+ self._ext_futures = {}
self.pkg_level_futures = None
def set_kwargs(self, mod):
@@ -232,10 +232,10 @@ class Scanner(object):
logging.debug("set_func_kwargs(); adding: %s, %s",
key, func_kwargs[key])
if func_kwargs[key][0] in ['Future', 'ExtendedFuture']:
- if key not in self.ext_futures:
+ if key not in self._ext_futures:
logging.debug(
"Adding a new key: %s to the
ExtendedFuture dict", key)
- self.ext_futures[key] = func_kwargs[key]
+ self._ext_futures[key] =
func_kwargs[key]
self._set_future(dynamic_data, key,
func_kwargs[key])
else: # builtin python data type
dynamic_data[key] =
DATA_TYPES[func_kwargs[key][0]]()
@@ -249,9 +249,9 @@ class Scanner(object):
@param dynamic_data: dictionary
'''
- for key in list(self.ext_futures):
+ for key in list(self._ext_futures):
if key not in self.pkg_level_futures:
- self._set_future(dynamic_data, key,
self.ext_futures[key])
+ self._set_future(dynamic_data, key,
self._ext_futures[key])
@staticmethod
def _set_future(dynamic_data, key, data):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/use/, pym/repoman/, pym/repoman/modules/scan/metadata/, ...
commit: 89a452505ce089a44f389e924fca233a1d0dc2fe
Author: Brian Dolbec gentoo org>
AuthorDate: Thu Apr 28 04:44:55 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 28 14:50:41 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=89a45250
repoman: Merge USEFlagChecks and UnusedCheck into PkgMetadata class
This eliminates muselist and used_useflags from dynamic_data.
pym/repoman/modules/scan/metadata/__init__.py | 25 +++-
pym/repoman/modules/scan/metadata/pkgmetadata.py | 44 ++
pym/repoman/modules/scan/metadata/unused.py| 40
.../modules/scan/{use => metadata}/use_flags.py| 23 +++
pym/repoman/modules/scan/scanbase.py | 2 +-
pym/repoman/modules/scan/use/__init__.py | 34 -
pym/repoman/scanner.py | 8 ++--
7 files changed, 52 insertions(+), 124 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/__init__.py
b/pym/repoman/modules/scan/metadata/__init__.py
index c297676..b090d6b 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -18,14 +18,18 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
- 'mod_kwargs': ['repo_settings', 'qatracker', 'options',
'metadata_xsd',
+ 'mod_kwargs': ['repo_settings', 'qatracker', 'options',
+ 'metadata_xsd', 'uselist',
],
'func_kwargs': {
'checkdir': (None, None),
'checkdirlist': (None, None),
- 'muselist': ('Future', 'set'),
+ 'ebuild': (None, None),
+ 'pkg': (None, None),
'repolevel': (None, None),
+ 'validity_future': (None, None),
'xpkg': (None, None),
+ 'y_ebuild': (None, None),
},
},
'ebuild-metadata': {
@@ -77,23 +81,6 @@ module_spec = {
'y_ebuild': (None, None),
},
},
- 'unused-metadata': {
- 'name': "unused",
- 'sourcefile': "unused",
- 'class': "UnusedCheck",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['qatracker',
- ],
- 'func_kwargs': {
- 'muselist': (None, None),
- 'used_useflags': (None, None),
- 'validity_future': (None, None),
- 'xpkg': (None, None),
- },
- },
}
}
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 25907b3..5c6452a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -36,8 +36,10 @@ from portage import os
from portage import _encodings, _unicode_encode
from portage.dep import Atom
+from .use_flags import USEFlagChecks
-class PkgMetadata(ScanBase):
+
+class PkgMetadata(ScanBase, USEFlagChecks):
'''Package metadata.xml checks'''
def __init__(self, **kwargs):
@@ -53,8 +55,10 @@ class PkgMetadata(ScanBase):
self.qatracker = kwargs.get('qatracker')
self.options = kwargs.get('options')
metadata_xsd = kwargs.get('metadata_xsd')
+ self.globalUseFlags = kwargs.get('uselist')
self.repoman_settings = repo_settings.repoman_settings
self.musedict = {}
+ self.muselist = set()
self.xmllint = XmlLint(self.options, self.repoman_settings,
metadata_xsd=metadata_xsd)
@@ -64,7 +68,7 @@ class PkgMetadata(ScanBase):
@param checkdir: string, directory path
@param checkdirlist: list of checkdir's
@param repolevel: integer
- @returns: dictionary, including {muselist}
+ @returns: boolean
'''
xpkg = kwargs.get('xpkg')
checkdir = kwargs.get('checkdir')
@@ -73,9 +77,7 @@ class PkgMetadata(ScanBase):
self.musedict = {}
if self.options.mode in ['manifest']:
- # update the dynamic data
- dyn_muselist = kwargs.get('muselist')
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: e5d298ccb05b630f17035f31a79c2fe55b90bf07
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 27 03:12:53 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 28 14:48:47 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e5d298cc
repoman/scanner.py: Make some variables local instead of classwide
Some were not needed outside of __init__.
Some are included self.kwargs.
Several classwide variables repolevel, reposplit, categories are accessed
outside of the scanner class. So therefore can not be made local.
pym/repoman/scanner.py | 20 ++--
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index d966513..a5c54b1 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -107,13 +107,13 @@ class Scanner(object):
self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = '
'.join(sorted(kwlist))
self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')
- self.profiles = setup_profile(profile_list)
+ profiles = setup_profile(profile_list)
- check_profiles(self.profiles,
self.repo_settings.repoman_settings.archlist())
+ check_profiles(profiles,
self.repo_settings.repoman_settings.archlist())
scanlist = scan(self.repolevel, self.reposplit, startdir,
self.categories, self.repo_settings)
- self.dev_keywords = dev_profile_keywords(self.profiles)
+ self.dev_keywords = dev_profile_keywords(profiles)
self.qatracker = self.vcs_settings.qatracker
@@ -123,7 +123,7 @@ class Scanner(object):
if self.vcs_settings.vcs is None:
self.options.echangelog = 'n'
- self.checks = {}
+ checks = {}
# The --echangelog option causes automatic ChangeLog generation,
# which invalidates changelog.ebuildadded and changelog.missing
# checks.
@@ -135,7 +135,7 @@ class Scanner(object):
# TODO: shouldn't this just be switched on the repo, iso the
VCS?
is_echangelog_enabled = self.options.echangelog in ('y',
'force')
self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in
('cvs', 'svn')
- self.checks['changelog'] = not is_echangelog_enabled and
self.vcs_settings.vcs_is_cvs_or_svn
+ checks['changelog'] = not is_echangelog_enabled and
self.vcs_settings.vcs_is_cvs_or_svn
if self.options.mode == "manifest" or self.options.quiet:
pass
@@ -170,7 +170,7 @@ class Scanner(object):
# Disable the "self.modules['Ebuild'].notadded" check when not
in commit mode and
# running `svn status` in every package dir will be too
expensive.
- self.checks['ebuild_notadded'] = not \
+ checks['ebuild_notadded'] = not \
(self.vcs_settings.vcs == "svn" and self.repolevel < 3
and self.options.mode != "commit")
self.effective_scanlist = scanlist
@@ -188,9 +188,9 @@ class Scanner(object):
"options": self.options,
"metadata_xsd": metadata_xsd,
"uselist": uselist,
- "checks": self.checks,
+ "checks": checks,
"repo_metadata": self.repo_metadata,
- "profiles": self.profiles,
+ "profiles": profiles,
"include_arches": self.include_arches,
"caches": self.caches,
"repoman_incrementals": self.repoman_incrementals,
@@ -291,7 +291,7 @@ class Scanner(object):
checkdirlist = os.listdir(checkdir)
# Run the status check
- if self.checks['ebuild_notadded']:
+ if self.kwargs['checks']['ebuild_notadded']:
self.vcs_settings.status.check(checkdir,
checkdir_relative, xpkg)
dynamic_data = {
@@ -350,7 +350,7 @@ class Scanner(object):
ebuildlist = sorted(pkgs.values())
ebuildlist = [pkg.pf for pkg in ebuildlist]
- if self.checks['changelog'] and "ChangeLog" not in
checkdirlist:
+ if self.kwargs['checks']['changelog'] and "ChangeLog"
not in checkdirlist:
self.qatracker.add_error("changelog.missing",
xpkg + "/ChangeLog")
changelog_path = os.path.join(checkdir_relative,
"ChangeLog")
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/modules/scan/eclasses/, ...
commit: 1521249252e81eb4eff9c37f498b51c52199592d
Author: Brian Dolbec gentoo org>
AuthorDate: Thu Apr 28 05:11:57 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 28 05:11:57 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=15212492
repoman: Move the LiveEclasses.is_live() to the Ebuild class
This removes another dynaic_data variable.
It also belongs with the ebuild eclass as a primary source for that type of
data.
pym/repoman/modules/scan/ebuild/ebuild.py | 9 +
pym/repoman/modules/scan/eclasses/__init__.py | 1 -
pym/repoman/modules/scan/eclasses/live.py | 22 +++---
pym/repoman/modules/scan/keywords/__init__.py | 1 -
pym/repoman/modules/scan/keywords/keywords.py | 8 +++-
pym/repoman/modules/scan/metadata/__init__.py | 1 -
.../modules/scan/metadata/ebuild_metadata.py | 3 +--
7 files changed, 16 insertions(+), 29 deletions(-)
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index c247a7f..0277aa9 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -13,6 +13,7 @@ from repoman.qa_data import no_exec, allvars
# import our initialized portage instance
from repoman._portage import portage
from portage import os
+from portage.const import LIVE_ECLASSES
pv_toolong_re = re.compile(r'[0-9]{19,}')
@@ -220,6 +221,14 @@ class Ebuild(ScanBase):
return self.continue_
@property
+ def is_live(self):
+ '''Test if the ebuild inherits a live eclass
+
+ @returns: boolean
+ '''
+ return set(LIVE_ECLASSES.intersection(self.inherited))
+
+ @property
def runInPkgs(self):
'''Package level scans'''
return (True, [self.check_isebuild])
diff --git a/pym/repoman/modules/scan/eclasses/__init__.py
b/pym/repoman/modules/scan/eclasses/__init__.py
index 63bb86f..78d46e4 100644
--- a/pym/repoman/modules/scan/eclasses/__init__.py
+++ b/pym/repoman/modules/scan/eclasses/__init__.py
@@ -22,7 +22,6 @@ module_spec = {
],
'func_kwargs': {
'ebuild': (None, None),
- 'live_ebuild': ('Future', 'UNSET'),
'pkg': (None, None),
'xpkg': (None, None),
'y_ebuild': (None, None),
diff --git a/pym/repoman/modules/scan/eclasses/live.py
b/pym/repoman/modules/scan/eclasses/live.py
index 842cbab..1ce33c0 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -6,8 +6,6 @@ Performs Live eclass checks
from repoman._portage import portage
from repoman.modules.scan.scanbase import ScanBase
-from portage.const import LIVE_ECLASSES
-
class LiveEclassChecks(ScanBase):
'''Performs checks for the usage of Live eclasses in ebuilds'''
@@ -20,27 +18,14 @@ class LiveEclassChecks(ScanBase):
self.pmaskdict = kwargs.get('repo_metadata')['pmaskdict']
self.repo_settings = kwargs.get('repo_settings')
- def is_live(self, **kwargs):
- '''Test if the ebuild inherits a live eclass
-
- @returns: dictionary, including {live_ebuild}
- '''
- ebuild = kwargs.get('ebuild').get()
- # update the dynamic data
- dyn_live = kwargs.get('live_ebuild')
- dyn_live.set(LIVE_ECLASSES.intersection(ebuild.inherited))
- return False
-
def check(self, **kwargs):
'''Ebuilds that inherit a "Live" eclass (darcs, subversion,
git, cvs,
etc..) should not be allowed to be marked stable
@param pkg: Package in which we check (object).
- @param package: Package in which we check (string).
+ @param xpkg: Package in which we check (string).
@param ebuild: Ebuild which we check (object).
@param y_ebuild: Ebuild which we check (string).
- @param keywords: The keywords of the ebuild.
- @param global_pmaskdict: A global dictionary of all the masks.
@returns: dictionary
'''
pkg = kwargs.get("pkg").result()
@@ -48,9 +33,8 @@ class LiveEclassChecks(ScanBase):
ebuild = kwargs.get('ebuild').get()
y_ebuild = kwargs.get('y_ebuild')
keywords = ebuild.keywords
- live_ebuild = kwargs.get('live_ebuild').get()
- if not live_ebuild and self.repo_settings.repo_config.name ==
"gentoo":
+ if not ebuild.is_live and self.repo_settings.repo_config.name
== "gentoo":
return False
is_stable = lambda kw: not
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/, pym/repoman/modules/scan/use/, ...
commit: 8f332cc2744b41dc0fccfbf4f194df3b97684a86
Author: Brian Dolbec gentoo org>
AuthorDate: Thu Apr 28 04:44:55 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 28 04:44:55 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8f332cc2
repoman: Merge USEFlagChecks and UnusedCheck into PkgMetadata class
This eliminates muselist and used_useflags from dynamic_data.
pym/repoman/modules/scan/metadata/__init__.py | 25 +++-
pym/repoman/modules/scan/metadata/pkgmetadata.py | 44 ++
pym/repoman/modules/scan/metadata/unused.py| 40
.../modules/scan/{use => metadata}/use_flags.py| 23 +++
pym/repoman/modules/scan/scanbase.py | 2 +-
pym/repoman/modules/scan/use/__init__.py | 34 -
pym/repoman/scanner.py | 8 ++--
7 files changed, 52 insertions(+), 124 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/__init__.py
b/pym/repoman/modules/scan/metadata/__init__.py
index c297676..b090d6b 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -18,14 +18,18 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
- 'mod_kwargs': ['repo_settings', 'qatracker', 'options',
'metadata_xsd',
+ 'mod_kwargs': ['repo_settings', 'qatracker', 'options',
+ 'metadata_xsd', 'uselist',
],
'func_kwargs': {
'checkdir': (None, None),
'checkdirlist': (None, None),
- 'muselist': ('Future', 'set'),
+ 'ebuild': (None, None),
+ 'pkg': (None, None),
'repolevel': (None, None),
+ 'validity_future': (None, None),
'xpkg': (None, None),
+ 'y_ebuild': (None, None),
},
},
'ebuild-metadata': {
@@ -77,23 +81,6 @@ module_spec = {
'y_ebuild': (None, None),
},
},
- 'unused-metadata': {
- 'name': "unused",
- 'sourcefile': "unused",
- 'class': "UnusedCheck",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['qatracker',
- ],
- 'func_kwargs': {
- 'muselist': (None, None),
- 'used_useflags': (None, None),
- 'validity_future': (None, None),
- 'xpkg': (None, None),
- },
- },
}
}
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 25907b3..5c6452a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -36,8 +36,10 @@ from portage import os
from portage import _encodings, _unicode_encode
from portage.dep import Atom
+from .use_flags import USEFlagChecks
-class PkgMetadata(ScanBase):
+
+class PkgMetadata(ScanBase, USEFlagChecks):
'''Package metadata.xml checks'''
def __init__(self, **kwargs):
@@ -53,8 +55,10 @@ class PkgMetadata(ScanBase):
self.qatracker = kwargs.get('qatracker')
self.options = kwargs.get('options')
metadata_xsd = kwargs.get('metadata_xsd')
+ self.globalUseFlags = kwargs.get('uselist')
self.repoman_settings = repo_settings.repoman_settings
self.musedict = {}
+ self.muselist = set()
self.xmllint = XmlLint(self.options, self.repoman_settings,
metadata_xsd=metadata_xsd)
@@ -64,7 +68,7 @@ class PkgMetadata(ScanBase):
@param checkdir: string, directory path
@param checkdirlist: list of checkdir's
@param repolevel: integer
- @returns: dictionary, including {muselist}
+ @returns: boolean
'''
xpkg = kwargs.get('xpkg')
checkdir = kwargs.get('checkdir')
@@ -73,9 +77,7 @@ class PkgMetadata(ScanBase):
self.musedict = {}
if self.options.mode in ['manifest']:
- # update the dynamic data
- dyn_muselist = kwargs.get('muselist')
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 125d92736ef9c87077cf32b8fb254fef4a52fbc1
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 27 03:12:53 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Wed Apr 27 05:20:31 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=125d9273
repoman/scanner.py: Make some variables local instead of classwide
Some were not needed outside of __init__.
Some are included self.kwargs.
pym/repoman/scanner.py | 30 +++---
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index d966513..46e9d85 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -75,11 +75,11 @@ class Scanner(object):
self.portdb._aux_cache_keys.update(
["EAPI", "IUSE", "KEYWORDS", "repository",
"SLOT"])
- self.reposplit = myreporoot.split(os.path.sep)
- self.repolevel = len(self.reposplit)
+ reposplit = myreporoot.split(os.path.sep)
+ self.repolevel = len(reposplit)
if self.options.mode == 'commit':
- repochecks.commit_check(self.repolevel, self.reposplit)
+ repochecks.commit_check(self.repolevel, reposplit)
repochecks.conflict_check(self.vcs_settings,
self.options)
# Make startdir relative to the canonical repodir, so that we
can pass
@@ -107,13 +107,13 @@ class Scanner(object):
self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = '
'.join(sorted(kwlist))
self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')
- self.profiles = setup_profile(profile_list)
+ profiles = setup_profile(profile_list)
- check_profiles(self.profiles,
self.repo_settings.repoman_settings.archlist())
+ check_profiles(profiles,
self.repo_settings.repoman_settings.archlist())
- scanlist = scan(self.repolevel, self.reposplit, startdir,
self.categories, self.repo_settings)
+ scanlist = scan(self.repolevel, reposplit, startdir,
self.categories, self.repo_settings)
- self.dev_keywords = dev_profile_keywords(self.profiles)
+ self.dev_keywords = dev_profile_keywords(profiles)
self.qatracker = self.vcs_settings.qatracker
@@ -123,7 +123,7 @@ class Scanner(object):
if self.vcs_settings.vcs is None:
self.options.echangelog = 'n'
- self.checks = {}
+ checks = {}
# The --echangelog option causes automatic ChangeLog generation,
# which invalidates changelog.ebuildadded and changelog.missing
# checks.
@@ -135,7 +135,7 @@ class Scanner(object):
# TODO: shouldn't this just be switched on the repo, iso the
VCS?
is_echangelog_enabled = self.options.echangelog in ('y',
'force')
self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in
('cvs', 'svn')
- self.checks['changelog'] = not is_echangelog_enabled and
self.vcs_settings.vcs_is_cvs_or_svn
+ checks['changelog'] = not is_echangelog_enabled and
self.vcs_settings.vcs_is_cvs_or_svn
if self.options.mode == "manifest" or self.options.quiet:
pass
@@ -170,14 +170,14 @@ class Scanner(object):
# Disable the "self.modules['Ebuild'].notadded" check when not
in commit mode and
# running `svn status` in every package dir will be too
expensive.
- self.checks['ebuild_notadded'] = not \
+ checks['ebuild_notadded'] = not \
(self.vcs_settings.vcs == "svn" and self.repolevel < 3
and self.options.mode != "commit")
self.effective_scanlist = scanlist
if self.options.if_modified == "y":
self.effective_scanlist = sorted(vcs_files_to_cps(
chain(self.changed.changed, self.changed.new,
self.changed.removed),
- self.repolevel, self.reposplit,
self.categories))
+ self.repolevel, reposplit, self.categories))
# Create our kwargs dict here to initialize the plugins with
self.kwargs = {
@@ -188,9 +188,9 @@ class Scanner(object):
"options": self.options,
"metadata_xsd": metadata_xsd,
"uselist": uselist,
- "checks": self.checks,
+ "checks": checks,
"repo_metadata": self.repo_metadata,
- "profiles": self.profiles,
+ "profiles": profiles,
"include_arches": self.include_arches,
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 8618be15d921e2de7abbdfe1a67c7b9c1ff224bd
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 27 03:22:37 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Wed Apr 27 05:20:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8618be15
repoman/scanner.py: Clean up some debug prints
pym/repoman/scanner.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index ab1339a..c2e64a5 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -260,17 +260,14 @@ class Scanner(object):
@param dynamic_data: dictionary
@param key: tuple of (dictionary-key, default-value)
'''
- #print("***", key, data)
if data[0] in ['Future', 'ExtendedFuture']:
if data[1] in ['UNSET']:
- #print("adding unset default")
dynamic_data[key] = ExtendedFuture()
else:
if data[1] in DATA_TYPES:
default = DATA_TYPES[data[1]]()
else:
default = data[1]
- #print("adding default:", default)
dynamic_data[key] = ExtendedFuture(default)
def scan_pkgs(self, can_force):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 5b1499d1c8a15cfcbf51fe55bf412f3d0583c41d
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 27 03:20:45 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Wed Apr 27 05:20:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5b1499d1
repoman/scanner.py: Mark self.ext_futures private
pym/repoman/scanner.py | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 46e9d85..ab1339a 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -200,7 +200,7 @@ class Scanner(object):
}
# initialize the plugin checks here
self.modules = {}
- self.ext_futures = {}
+ self._ext_futures = {}
self.pkg_level_futures = None
def set_kwargs(self, mod):
@@ -232,10 +232,10 @@ class Scanner(object):
logging.debug("set_func_kwargs(); adding: %s, %s",
key, func_kwargs[key])
if func_kwargs[key][0] in ['Future', 'ExtendedFuture']:
- if key not in self.ext_futures:
+ if key not in self._ext_futures:
logging.debug(
"Adding a new key: %s to the
ExtendedFuture dict", key)
- self.ext_futures[key] = func_kwargs[key]
+ self._ext_futures[key] =
func_kwargs[key]
self._set_future(dynamic_data, key,
func_kwargs[key])
else: # builtin python data type
dynamic_data[key] =
DATA_TYPES[func_kwargs[key][0]]()
@@ -249,9 +249,9 @@ class Scanner(object):
@param dynamic_data: dictionary
'''
- for key in list(self.ext_futures):
+ for key in list(self._ext_futures):
if key not in self.pkg_level_futures:
- self._set_future(dynamic_data, key,
self.ext_futures[key])
+ self._set_future(dynamic_data, key,
self._ext_futures[key])
@staticmethod
def _set_future(dynamic_data, key, data):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/use/, pym/repoman/, pym/repoman/modules/scan/metadata/, ...
commit: 239754be33f46b9f75208e4e61d49cb8d364749e
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 27 05:19:07 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Wed Apr 27 05:20:40 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=239754be
repoman: Merge USEFlagChecks and UnusedCheck into PkgMetadata class
This eliminates muselist and used_useflags from dynamic_data.
pym/repoman/modules/scan/metadata/__init__.py | 25 +++-
pym/repoman/modules/scan/metadata/pkgmetadata.py | 44 ++
pym/repoman/modules/scan/metadata/unused.py| 40
.../modules/scan/{use => metadata}/use_flags.py| 23 +++
pym/repoman/modules/scan/scanbase.py | 2 +-
pym/repoman/modules/scan/use/__init__.py | 34 -
pym/repoman/scanner.py | 6 +--
7 files changed, 51 insertions(+), 123 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/__init__.py
b/pym/repoman/modules/scan/metadata/__init__.py
index c297676..b090d6b 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -18,14 +18,18 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
- 'mod_kwargs': ['repo_settings', 'qatracker', 'options',
'metadata_xsd',
+ 'mod_kwargs': ['repo_settings', 'qatracker', 'options',
+ 'metadata_xsd', 'uselist',
],
'func_kwargs': {
'checkdir': (None, None),
'checkdirlist': (None, None),
- 'muselist': ('Future', 'set'),
+ 'ebuild': (None, None),
+ 'pkg': (None, None),
'repolevel': (None, None),
+ 'validity_future': (None, None),
'xpkg': (None, None),
+ 'y_ebuild': (None, None),
},
},
'ebuild-metadata': {
@@ -77,23 +81,6 @@ module_spec = {
'y_ebuild': (None, None),
},
},
- 'unused-metadata': {
- 'name': "unused",
- 'sourcefile': "unused",
- 'class': "UnusedCheck",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['qatracker',
- ],
- 'func_kwargs': {
- 'muselist': (None, None),
- 'used_useflags': (None, None),
- 'validity_future': (None, None),
- 'xpkg': (None, None),
- },
- },
}
}
diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py
b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 25907b3..5c6452a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -36,8 +36,10 @@ from portage import os
from portage import _encodings, _unicode_encode
from portage.dep import Atom
+from .use_flags import USEFlagChecks
-class PkgMetadata(ScanBase):
+
+class PkgMetadata(ScanBase, USEFlagChecks):
'''Package metadata.xml checks'''
def __init__(self, **kwargs):
@@ -53,8 +55,10 @@ class PkgMetadata(ScanBase):
self.qatracker = kwargs.get('qatracker')
self.options = kwargs.get('options')
metadata_xsd = kwargs.get('metadata_xsd')
+ self.globalUseFlags = kwargs.get('uselist')
self.repoman_settings = repo_settings.repoman_settings
self.musedict = {}
+ self.muselist = set()
self.xmllint = XmlLint(self.options, self.repoman_settings,
metadata_xsd=metadata_xsd)
@@ -64,7 +68,7 @@ class PkgMetadata(ScanBase):
@param checkdir: string, directory path
@param checkdirlist: list of checkdir's
@param repolevel: integer
- @returns: dictionary, including {muselist}
+ @returns: boolean
'''
xpkg = kwargs.get('xpkg')
checkdir = kwargs.get('checkdir')
@@ -73,9 +77,7 @@ class PkgMetadata(ScanBase):
self.musedict = {}
if self.options.mode in ['manifest']:
- # update the dynamic data
- dyn_muselist = kwargs.get('muselist')
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 8443e66ae5ffce5fb0daa73a3eeb0b18077ee942
Author: Zac Medico gentoo org>
AuthorDate: Tue Apr 26 18:07:48 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Apr 26 18:07:48 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8443e66a
Scanner: remove self.pkgs attribute
pym/repoman/scanner.py | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 6ded7ee..d966513 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -179,8 +179,6 @@ class Scanner(object):
chain(self.changed.changed, self.changed.new,
self.changed.removed),
self.repolevel, self.reposplit,
self.categories))
- self.pkgs = None
-
# Create our kwargs dict here to initialize the plugins with
self.kwargs = {
"repo_settings": self.repo_settings,
@@ -348,8 +346,8 @@ class Scanner(object):
continue
# Sort ebuilds in ascending order for the
KEYWORDS.dropped check.
- self.pkgs = dynamic_data['pkgs'].get()
- ebuildlist = sorted(self.pkgs.values())
+ pkgs = dynamic_data['pkgs'].get()
+ ebuildlist = sorted(pkgs.values())
ebuildlist = [pkg.pf for pkg in ebuildlist]
if self.checks['changelog'] and "ChangeLog" not in
checkdirlist:
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 67b7d60d949002ecf72a13dcda81e34362df8c2c
Author: Brian Dolbec gentoo org>
AuthorDate: Tue Apr 26 14:09:39 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue Apr 26 14:09:39 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=67b7d60d
repoman, scanner.py: Remove a rebase error duplicate line.
pym/repoman/scanner.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 23d666d..6ded7ee 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -233,8 +233,6 @@ class Scanner(object):
for key in new:
logging.debug("set_func_kwargs(); adding: %s, %s",
key, func_kwargs[key])
- logging.debug("set_func_kwargs(); adding: %s, %s",
- key, func_kwargs[key])
if func_kwargs[key][0] in ['Future', 'ExtendedFuture']:
if key not in self.ext_futures:
logging.debug(
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/
commit: 695492024a9ca40fd14148a980a960a8551dab29
Author: Brian Dolbec gentoo org>
AuthorDate: Tue Apr 26 14:08:41 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue Apr 26 14:08:41 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=69549202
Repoman: Merge IsEbuild class into ebuild
This reduces the need to import the pkgs data, it is now an internal class
instance variable.
'pkgs' is still needed, but only to assign it the new pkgs dictionary which the
scanner
requires for the ebuild loop.
pym/repoman/modules/scan/ebuild/__init__.py | 27 ++--
pym/repoman/modules/scan/ebuild/ebuild.py | 84 ++--
pym/repoman/modules/scan/ebuild/isebuild.py | 99 -
pym/repoman/scanner.py | 4 +-
4 files changed, 88 insertions(+), 126 deletions(-)
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py
b/pym/repoman/modules/scan/ebuild/__init__.py
index b243314..8666e78 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -10,25 +10,6 @@ module_spec = {
'name': 'ebuild',
'description': doc,
'provides':{
- 'isebuild-module': {
- 'name': "isebuild",
- 'sourcefile': "isebuild",
- 'class': "IsEbuild",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
- ],
- 'func_kwargs': {
- 'can_force': (None, None),
- 'checkdir': (None, None),
- 'checkdirlist': (None, None),
- 'pkgs': ('Future', 'dict'),
- 'validity_future': ('Future', True),
- 'xpkg': (None, None),
- },
- },
'ebuild-module': {
'name': "ebuild",
'sourcefile': "ebuild",
@@ -37,16 +18,20 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
- 'mod_kwargs': ['qatracker', 'repo_settings',
'vcs_settings', 'checks',
+ 'mod_kwargs': ['qatracker', 'repo_settings',
'vcs_settings',
+ 'checks', 'portdb'
],
'func_kwargs': {
+ 'can_force': (None, None),
'catdir': (None, None),
'changed': (None, None),
'changelog_modified': (None, None),
+ 'checkdir': (None, None),
+ 'checkdirlist': (None, None),
'ebuild': ('Future', 'UNSET'),
'pkg': ('Future', 'UNSET'),
'pkgdir': (None, None),
- 'pkgs': (None, None),
+ 'pkgs': ('Future', 'dict'),
'repolevel': (None, None),
'validity_future': (None, None),
'xpkg': (None, None),
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index 92b1ea4..c247a7f 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -3,8 +3,13 @@
from __future__ import print_function, unicode_literals
import re
+import stat
+
+from _emerge.Package import Package
+from _emerge.RootConfig import RootConfig
from repoman.modules.scan.scanbase import ScanBase
+from repoman.qa_data import no_exec, allvars
# import our initialized portage instance
from repoman._portage import portage
from portage import os
@@ -19,16 +24,19 @@ class Ebuild(ScanBase):
'''Class init
@param qatracker: QATracker instance
+ @param portdb: portdb instance
@param repo_settings: repository settings instance
@param vcs_settings: VCSSettings instance
- @param changed: changes dictionary
@param checks: checks dictionary
'''
super(Ebuild, self).__init__(**kwargs)
self.qatracker = kwargs.get('qatracker')
+ self.portdb = kwargs.get('portdb')
self.repo_settings = kwargs.get('repo_settings')
self.vcs_settings = kwargs.get('vcs_settings')
self.checks = kwargs.get('checks')
+ self.root_config =
RootConfig(self.repo_settings.repoman_settings,
+
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/fetch/, pym/repoman/, pym/repoman/modules/scan/mirrors/
commit: 64b58270e5dc1c9f4aa2267a6bf3ab8117d0d44c
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 25 17:14:16 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 17:15:11 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=64b58270
repoman: Move thirdpartymirror check to the fetches module
This check should be part of the FetchChecks class.
This removes 'src_uri_error' Future need.
pym/repoman/modules/scan/fetch/__init__.py | 2 +-
pym/repoman/modules/scan/fetch/fetches.py | 45 ++--
pym/repoman/modules/scan/mirrors/__init__.py | 30 ---
.../modules/scan/mirrors/thirdpartymirrors.py | 61 --
pym/repoman/scanner.py | 2 +-
5 files changed, 43 insertions(+), 97 deletions(-)
diff --git a/pym/repoman/modules/scan/fetch/__init__.py
b/pym/repoman/modules/scan/fetch/__init__.py
index 1228435..3c8e600 100644
--- a/pym/repoman/modules/scan/fetch/__init__.py
+++ b/pym/repoman/modules/scan/fetch/__init__.py
@@ -24,7 +24,7 @@ module_spec = {
'changed': (None, None),
'checkdir': (None, None),
'checkdir_relative': (None, None),
- 'src_uri_error': ('Future', 'UNSET'),
+ 'ebuild': (None, None),
'xpkg': (None, None),
},
},
diff --git a/pym/repoman/modules/scan/fetch/fetches.py
b/pym/repoman/modules/scan/fetch/fetches.py
index ddbf6bd..555f34f 100644
--- a/pym/repoman/modules/scan/fetch/fetches.py
+++ b/pym/repoman/modules/scan/fetch/fetches.py
@@ -32,13 +32,22 @@ class FetchChecks(ScanBase):
self.vcs_settings = kwargs.get('vcs_settings')
self._src_uri_error = False
+ # TODO: Build a regex instead here, for the SRC_URI.mirror
check.
+ self.thirdpartymirrors = {}
+ profile_thirdpartymirrors =
self.repo_settings.repoman_settings.thirdpartymirrors().items()
+ for mirror_alias, mirrors in profile_thirdpartymirrors:
+ for mirror in mirrors:
+ if not mirror.endswith("/"):
+ mirror += "/"
+ self.thirdpartymirrors[mirror] = mirror_alias
+
def check(self, **kwargs):
'''Checks the ebuild sources and files for errors
@param xpkg: the pacakge being checked
@param checkdir: string, directory path
@param checkdir_relative: repolevel determined path
- @returns: dictionary, including {src_uri_error}
+ @returns: boolean
'''
xpkg = kwargs.get('xpkg')
checkdir = kwargs.get('checkdir')
@@ -130,9 +139,6 @@ class FetchChecks(ScanBase):
self.qatracker.add_error(
"file.name",
"%s/files/%s: char '%s'" %
(checkdir, y, y[index]))
- # update the dynamic data
- dyn_src_uri_error = kwargs.get('src_uri_error')
- dyn_src_uri_error.set(self._src_uri_error)
return False
def digests(self, checkdir):
@@ -147,7 +153,38 @@ class FetchChecks(ScanBase):
del mf
return _digests
+ def check_mirrors(self, **kwargs):
+ '''Check that URIs don't reference a server from
thirdpartymirrors
+
+ @param ebuild: Ebuild which we check (object).
+ @returns: boolean
+ '''
+ ebuild = kwargs.get('ebuild').get()
+
+ for uri in portage.dep.use_reduce(
+ ebuild.metadata["SRC_URI"], matchall=True,
is_src_uri=True,
+ eapi=ebuild.eapi, flat=True):
+ contains_mirror = False
+ for mirror, mirror_alias in
self.thirdpartymirrors.items():
+ if uri.startswith(mirror):
+ contains_mirror = True
+ break
+ if not contains_mirror:
+ continue
+
+ new_uri = "mirror://%s/%s" % (mirror_alias,
uri[len(mirror):])
+ self.qatracker.add_error(
+ "SRC_URI.mirror",
+ "%s: '%s' found in thirdpartymirrors, use '%s'"
% (
+ ebuild.relative_path, mirror, new_uri))
+ return False
+
@property
def runInPkgs(self):
'''Package level scans'''
return (True, [self.check])
+
+ @property
+ def
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/depend/
commit: 260574004ff6c4dc5187fdf579851d920788fff4
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 25 17:02:26 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 25 17:02:26 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=26057400
repoman: remove DependUnknown (part of ProfileDependsChecks now)
pym/repoman/modules/scan/depend/unknown.py | 41 --
1 file changed, 41 deletions(-)
diff --git a/pym/repoman/modules/scan/depend/unknown.py
b/pym/repoman/modules/scan/depend/unknown.py
deleted file mode 100644
index 6667d4f..000
--- a/pym/repoman/modules/scan/depend/unknown.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from repoman.modules.scan.scanbase import ScanBase
-
-
-class DependUnknown(ScanBase):
- '''Perform checks to determine unknown dependencies'''
-
- def __init__(self, **kwargs):
- '''Class init
-
- @param qatracker: QATracker instance
- '''
- self.qatracker = kwargs.get('qatracker')
-
- def check(self, **kwargs):
- '''Perform unknown dependancy checks
-
- @param ebuild: Future.result == Ebuild which we check (object).
- @param baddepsyntax: Future.result == boolean
- @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom)
- @returns: dictionary
- '''
- ebuild = kwargs.get('ebuild').get()
- baddepsyntax = kwargs.get('baddepsyntax').get()
- unknown_pkgs = kwargs.get('unknown_pkgs').get()
-
- if not baddepsyntax and unknown_pkgs:
- type_map = {}
- for mytype, atom in unknown_pkgs:
- type_map.setdefault(mytype, set()).add(atom)
- for mytype, atoms in type_map.items():
- self.qatracker.add_error(
- "dependency.unknown", "%s: %s: %s"
- % (ebuild.relative_path, mytype, ",
".join(sorted(atoms
- return False
-
- @property
- def runInEbuilds(self):
- '''Ebuild level scans'''
- return (True, [self.check])
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 836cc3ffc55376348315577958e7212fefa38a2a
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 25 16:50:59 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 25 16:58:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=836cc3ff
ProfileDependsChecks: encapsulate DependChecks and LicenseChecks
pym/repoman/modules/scan/depend/__init__.py | 38 +-
pym/repoman/modules/scan/depend/_depend_checks.py | 150
pym/repoman/modules/scan/depend/depend.py | 158 --
pym/repoman/modules/scan/depend/profile.py| 17 ++-
pym/repoman/modules/scan/metadata/__init__.py | 17 ---
pym/repoman/modules/scan/metadata/license.py | 54
pym/repoman/scanner.py| 4 +-
7 files changed, 167 insertions(+), 271 deletions(-)
diff --git a/pym/repoman/modules/scan/depend/__init__.py
b/pym/repoman/modules/scan/depend/__init__.py
index 01bd116..6d12286 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -10,24 +10,6 @@ module_spec = {
'name': 'depend',
'description': doc,
'provides':{
- 'depend-module': {
- 'name': "depend",
- 'sourcefile': "depend",
- 'class': "DependChecks",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['qatracker', 'portdb'
- ],
- 'func_kwargs': {
- 'baddepsyntax': ('Future', False),
- 'badlicsyntax': ('Future', False),
- 'ebuild': (None, None),
- 'pkg': (None, None),
- 'unknown_pkgs': ('Future', 'UNSET'),
- },
- },
'profile-module': {
'name': "profile",
'sourcefile': "profile",
@@ -37,30 +19,12 @@ module_spec = {
'func_desc': {
},
'mod_kwargs': ['qatracker', 'portdb', 'profiles',
'options',
- 'repo_settings', 'include_arches', 'caches',
+ 'repo_metadata', 'repo_settings',
'include_arches', 'caches',
'repoman_incrementals', 'env', 'have',
'dev_keywords'
],
'func_kwargs': {
- 'baddepsyntax': (None, None),
'ebuild': (None, None),
'pkg': (None, None),
- 'unknown_pkgs': (None, None),
- },
- },
- 'unknown-module': {
- 'name': "unknown",
- 'sourcefile': "unknown",
- 'class': "DependUnknown",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['qatracker',
- ],
- 'func_kwargs': {
- 'baddepsyntax': (None, None),
- 'ebuild': (None, None),
- 'unknown_pkgs': ('Future', 'UNSET'),
},
},
}
diff --git a/pym/repoman/modules/scan/depend/_depend_checks.py
b/pym/repoman/modules/scan/depend/_depend_checks.py
new file mode 100644
index 000..4e1d216
--- /dev/null
+++ b/pym/repoman/modules/scan/depend/_depend_checks.py
@@ -0,0 +1,150 @@
+# -*- coding:utf-8 -*-
+
+
+from _emerge.Package import Package
+
+from repoman.check_missingslot import check_missingslot
+# import our initialized portage instance
+from repoman._portage import portage
+from repoman.qa_data import suspect_virtual, suspect_rdepend
+
+
+def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata):
+ '''Checks the ebuild dependencies for errors
+
+ @param pkg: Package in which we check (object).
+ @param ebuild: Ebuild which we check (object).
+ @param portdb: portdb instance
+ @param qatracker: QATracker instance
+ @param repo_metadata: dictionary of various repository items.
+ @returns: (unknown_pkgs, badlicsyntax)
+ '''
+
+ unknown_pkgs = set()
+
+ inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \
+ "java-pkg-opt-2" in ebuild.inherited,
+ inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited
+ # operator_tokens = set(["||", "(", ")"])
+ type_list, badsyntax = [], []
+ for mytype in Package._dep_keys + ("LICENSE",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/status/
commit: ac6c5aa13a48bc50b1aa5cfc8cb46ad9c8c42a2b
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 25 16:32:11 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 16:32:11 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ac6c5aa1
repoman: Migrate the vcsstatus code back to scanner.py
This module had only 2 lines of useful code, the rest was boiler plate module
handling code.
It was also not dependant on other checks.
pym/repoman/modules/scan/status/__init__.py | 31 --
pym/repoman/modules/scan/status/vcsstatus.py | 39
pym/repoman/scanner.py | 6 -
3 files changed, 5 insertions(+), 71 deletions(-)
diff --git a/pym/repoman/modules/scan/status/__init__.py
b/pym/repoman/modules/scan/status/__init__.py
deleted file mode 100644
index 1abf580..000
--- a/pym/repoman/modules/scan/status/__init__.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Copyright 2015-2016 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-doc = """fetches plug-in module for repoman.
-Performs fetch related checks on ebuilds."""
-__doc__ = doc[:]
-
-
-module_spec = {
- 'name': 'status',
- 'description': doc,
- 'provides':{
- 'status-module': {
- 'name': "vcsstatus",
- 'sourcefile': "vcsstatus",
- 'class': "VCSStatus",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['vcs_settings', 'checks',
- ],
- 'func_kwargs': {
- 'checkdir': (None, None),
- 'checkdir_relative': (None, None),
- 'xpkg': (None, None),
- },
- },
- }
-}
-
diff --git a/pym/repoman/modules/scan/status/vcsstatus.py
b/pym/repoman/modules/scan/status/vcsstatus.py
deleted file mode 100644
index ab81a11..000
--- a/pym/repoman/modules/scan/status/vcsstatus.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from repoman.modules.scan.scanbase import ScanBase
-
-
-class VCSStatus(ScanBase):
- '''Determines the status of the vcs repositories
- to determine if files are not added'''
-
- def __init__(self, **kwargs):
- '''Class init
-
- @param vcs_settings: VCSSettings instance
- '''
- super(VCSStatus, self).__init__(**kwargs)
- self.vcs_settings = kwargs.get('vcs_settings')
- self.check_not_added = kwargs.get("checks")['ebuild_notadded']
-
- def check(self, **kwargs):
- '''Performs an indirect status check via the
- correct vcs plugin Status class
-
- @param check_not_added: boolean
- @param checkdir: string, directory path
- @param checkdir_relative: repolevel determined path
- @param xpkg: the current package being checked
- @returns: boolean
- '''
- checkdir = kwargs.get('checkdir')
- checkdir_relative = kwargs.get('checkdir_relative')
- xpkg = kwargs.get('xpkg')
- if self.check_not_added:
- self.vcs_settings.status.check(checkdir,
checkdir_relative, xpkg)
- return False
-
- @property
- def runInPkgs(self):
- '''Package level scans'''
- return (True, [self.check])
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 73187e9..2224fa3 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -294,6 +294,10 @@ class Scanner(object):
checkdir_relative = os.path.join(".", checkdir_relative)
checkdirlist = os.listdir(checkdir)
+ # Run the status check
+ if self.checks['ebuild_notadded']:
+ self.vcs_settings.status.check(checkdir,
checkdir_relative, xpkg)
+
dynamic_data = {
'checkdirlist': ExtendedFuture(checkdirlist),
'checkdir': checkdir,
@@ -322,7 +326,7 @@ class Scanner(object):
# need to set it up for ==> self.modules or some other
ordered list
for mod in [('manifests', 'Manifests'), ('isebuild',
'IsEbuild'),
('keywords', 'KeywordChecks'),
('files', 'FileChecks'),
- ('vcsstatus', 'VCSStatus'),
('fetches', 'FetchChecks'),
+ ('fetches', 'FetchChecks'),
('pkgmetadata',
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/status/, pym/repoman/
commit: 4a0072779d465cb7b0bd9e396e78772c8a45d051
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 25 16:11:42 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 16:11:42 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a007277
repoman: Clean up some no longer used eadded variables
pym/repoman/modules/scan/status/__init__.py | 1 -
pym/repoman/modules/scan/status/vcsstatus.py | 5 +
pym/repoman/scanner.py | 2 --
3 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/pym/repoman/modules/scan/status/__init__.py
b/pym/repoman/modules/scan/status/__init__.py
index 5f30639..1abf580 100644
--- a/pym/repoman/modules/scan/status/__init__.py
+++ b/pym/repoman/modules/scan/status/__init__.py
@@ -23,7 +23,6 @@ module_spec = {
'func_kwargs': {
'checkdir': (None, None),
'checkdir_relative': (None, None),
- 'eadded': ('Future', 'UNSET'),
'xpkg': (None, None),
},
},
diff --git a/pym/repoman/modules/scan/status/vcsstatus.py
b/pym/repoman/modules/scan/status/vcsstatus.py
index fd56560..ab81a11 100644
--- a/pym/repoman/modules/scan/status/vcsstatus.py
+++ b/pym/repoman/modules/scan/status/vcsstatus.py
@@ -24,16 +24,13 @@ class VCSStatus(ScanBase):
@param checkdir: string, directory path
@param checkdir_relative: repolevel determined path
@param xpkg: the current package being checked
- @returns: dictionary including {eadded}
+ @returns: boolean
'''
checkdir = kwargs.get('checkdir')
checkdir_relative = kwargs.get('checkdir_relative')
xpkg = kwargs.get('xpkg')
if self.check_not_added:
self.vcs_settings.status.check(checkdir,
checkdir_relative, xpkg)
- # update the dynamic data
- dyn_eadded = kwargs.get('eadded')
- dyn_eadded.set(self.vcs_settings.status.eadded)
return False
@property
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 508fedb..73187e9 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -180,7 +180,6 @@ class Scanner(object):
self.repolevel, self.reposplit,
self.categories))
self.pkgs = None
- self.eadded = False
# Create our kwargs dict here to initialize the plugins with
self.kwargs = {
@@ -285,7 +284,6 @@ class Scanner(object):
logging.info("checking package %s", xpkg)
# save memory by discarding xmatch caches from previous
package(s)
self.caches['arch_xmatch'].clear()
- self.eadded = []
catdir, pkgdir = xpkg.split("/")
checkdir = self.repo_settings.repodir + "/" + xpkg
checkdir_relative = ""
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/
commit: baa93beabed3da1186683c0a4ea9e6fed4d3fbda
Author: Brian Dolbec gentoo org>
AuthorDate: Fri Mar 11 00:11:13 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=baa93bea
repoman: Move changelog_modified to dynamic_data for use in the scan/ebuild
module
pym/repoman/modules/scan/ebuild/ebuild.py | 3 ++-
pym/repoman/scanner.py| 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index 0ae416b..e9a2cdd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -74,9 +74,10 @@ class Ebuild(ScanBase):
self.xpkg = kwargs.get('xpkg')
self.y_ebuild = kwargs.get('y_ebuild')
self.changed = kwargs.get('changed')
+ changelog_modified = kwargs.get('changelog_modified')
self._set_paths(**kwargs)
- if self.checks['changelog'] and not self.changelog_modified \
+ if self.checks['changelog'] and not changelog_modified \
and self.ebuild_path in self.changed.new_ebuilds:
self.qatracker.add_error('changelog.ebuildadded',
self.relative_path)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 3d23725..86e389a 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -263,7 +263,7 @@ class Scanner(object):
self.qatracker.add_error("changelog.missing",
xpkg + "/ChangeLog")
changelog_path = os.path.join(checkdir_relative,
"ChangeLog")
- self.changelog_modified = changelog_path in
self.changed.changelogs
+ dynamic_data["changelog_modified"] = changelog_path in
self.changed.changelogs
self._scan_ebuilds(ebuildlist, dynamic_data)
return dynamic_data['can_force']
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/metadata/
commit: 13197c7ce03e7fa88899589bfcfe6f3ebaac9112
Author: Michał Górny gentoo org>
AuthorDate: Sun Apr 17 08:06:28 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=13197c7c
repoman: Use XML Schema for metadata.xml validation
pym/repoman/_xml.py | 16 +-
pym/repoman/metadata.py | 39
pym/repoman/modules/scan/metadata/__init__.py| 2 +-
pym/repoman/modules/scan/metadata/pkgmetadata.py | 6 ++--
pym/repoman/scanner.py | 8 ++---
5 files changed, 36 insertions(+), 35 deletions(-)
diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
index d55dda5..33a536a 100644
--- a/pym/repoman/_xml.py
+++ b/pym/repoman/_xml.py
@@ -12,7 +12,7 @@ from portage import os
from portage.output import red
from portage.process import find_binary
-from repoman.metadata import fetch_metadata_dtd
+from repoman.metadata import fetch_metadata_xsd
from repoman._subprocess import repoman_getstatusoutput
@@ -53,12 +53,12 @@ class
_MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
class XmlLint(object):
- def __init__(self, options, repoman_settings, metadata_dtd=None):
- self.metadata_dtd = (metadata_dtd or
- os.path.join(repoman_settings["DISTDIR"],
'metadata.dtd'))
+ def __init__(self, options, repoman_settings, metadata_xsd=None):
+ self.metadata_xsd = (metadata_xsd or
+ os.path.join(repoman_settings["DISTDIR"],
'metadata.xsd'))
self.options = options
self.repoman_settings = repoman_settings
- self._is_capable = metadata_dtd is not None
+ self._is_capable = metadata_xsd is not None
self.binary = None
self._check_capable()
@@ -69,7 +69,7 @@ class XmlLint(object):
if not self.binary:
print(red("!!! xmllint not found. Can't check
metadata.xml.\n"))
elif not self._is_capable:
- if not fetch_metadata_dtd(self.metadata_dtd,
self.repoman_settings):
+ if not fetch_metadata_xsd(self.metadata_xsd,
self.repoman_settings):
sys.exit(1)
# this can be problematic if xmllint changes their
output
self._is_capable = True
@@ -93,8 +93,8 @@ class XmlLint(object):
# xmlint can produce garbage output even on success, so only
dump
# the ouput when it fails.
st, out = repoman_getstatusoutput(
- self.binary + " --nonet --noout --dtdvalid %s %s" % (
- portage._shell_quote(self.metadata_dtd),
+ self.binary + " --nonet --noout --schema %s %s" % (
+ portage._shell_quote(self.metadata_xsd),
portage._shell_quote(
os.path.join(checkdir,
"metadata.xml"
if st != os.EX_OK:
diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index e95ad41..7a514dc 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -33,8 +33,9 @@ metadata_xml_declaration = '' \
% (metadata_xml_encoding,)
metadata_doctype_name = 'pkgmetadata'
metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
+metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd'
# force refetch if the local copy creation time is older than this
-metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
+metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
def parse_metadata_use(xml_tree):
@@ -86,36 +87,36 @@ def parse_metadata_use(xml_tree):
return uselist
-def fetch_metadata_dtd(metadata_dtd, repoman_settings):
+def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
- Fetch metadata.dtd if it doesn't exist or the ctime is older than
- metadata_dtd_ctime_interval.
+ Fetch metadata.xsd if it doesn't exist or the ctime is older than
+ metadata_xsd_ctime_interval.
@rtype: bool
@return: True if successful, otherwise False
"""
must_fetch = True
- metadata_dtd_st = None
+ metadata_xsd_st = None
current_time = int(time.time())
try:
- metadata_dtd_st = os.stat(metadata_dtd)
+ metadata_xsd_st = os.stat(metadata_xsd)
except EnvironmentError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
else:
- # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
- if abs(current_time - metadata_dtd_st.st_ctime) \
- < metadata_dtd_ctime_interval:
+
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/arches/
commit: 44b8d01d71d5efffcce776a291acb6fef076dcb9
Author: Zac Medico gentoo org>
AuthorDate: Sat Apr 23 23:54:41 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=44b8d01d
ArchChecks: don't mix arches between ebuilds
Fix ArchChecks to not mix arches of ebuilds together, so that
errors are only reported for those arches that the ebuild has
keywords for.
pym/repoman/modules/scan/arches/arches.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 4df25a8..6e1c17d 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,7 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
- #dyn_arches.clear()
+ dyn_arches.clear()
dyn_arches.update(arches)
return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/vcs/bzr/, pym/repoman/modules/vcs/None/, ...
commit: 82c1160302e4d66e2ea6cf0afbacc2585e5ad965
Author: Brian Dolbec gentoo org>
AuthorDate: Tue Feb 16 20:06:30 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=82c11603
repoman: Migrate the commit code to the vcs modules
pym/repoman/actions.py | 84 +
pym/repoman/modules/vcs/None/changes.py | 13 +
pym/repoman/modules/vcs/bzr/changes.py | 1 -
pym/repoman/modules/vcs/changes.py | 20
pym/repoman/modules/vcs/cvs/changes.py | 1 -
pym/repoman/modules/vcs/git/changes.py | 10
pym/repoman/modules/vcs/hg/changes.py | 17 +++
7 files changed, 83 insertions(+), 63 deletions(-)
diff --git a/pym/repoman/actions.py b/pym/repoman/actions.py
index 4032e04..c0cd5cc 100644
--- a/pym/repoman/actions.py
+++ b/pym/repoman/actions.py
@@ -7,21 +7,19 @@ import io
import logging
import platform
import signal
-import subprocess
import sys
import tempfile
from itertools import chain
from _emerge.UserQuery import UserQuery
-import portage
+from repoman._portage import portage
from portage import os
from portage import _encodings
from portage import _unicode_encode
from portage.output import (
bold, create_color_func, green, red)
from portage.package.ebuild.digestgen import digestgen
-from portage.process import find_binary, spawn
from portage.util import writemsg_level
from repoman.gpg import gpgsign, need_signature
@@ -443,43 +441,18 @@ class Actions(object):
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
- commit_cmd = []
- if self.options.pretend and self.vcs_settings.vcs is None:
- # substitute a bogus value for pretend output
- commit_cmd.append("cvs")
- else:
- commit_cmd.append(self.vcs_settings.vcs)
- commit_cmd.extend(self.vcs_settings.vcs_global_opts)
- commit_cmd.append("commit")
- commit_cmd.extend(self.vcs_settings.vcs_local_opts)
- if self.vcs_settings.vcs == "hg":
- commit_cmd.extend(["--logfile", commitmessagefile])
- commit_cmd.extend(myfiles)
- else:
- commit_cmd.extend(["-F", commitmessagefile])
- commit_cmd.extend(f.lstrip("./") for f in myfiles)
-
+ retval = self.vcs_settings.changes.commit(myfiles,
commitmessagefile)
+ # cleanup the commit message before possibly exiting
try:
- if self.options.pretend:
- print("(%s)" % (" ".join(commit_cmd),))
- else:
- retval = spawn(commit_cmd,
env=self.repo_settings.commit_env)
- if retval != os.EX_OK:
- if
self.repo_settings.repo_config.sign_commit and not
self.vcs_settings.status.supports_gpg_sign():
- # Inform user that newer git is
needed (bug #403323).
- logging.error(
- "Git >=1.7.9 is
required for signed commits!")
-
- writemsg_level(
- "!!! Exiting on %s (shell) "
- "error code: %s\n" %
(self.vcs_settings.vcs, retval),
- level=logging.ERROR,
noiselevel=-1)
- sys.exit(retval)
- finally:
- try:
- os.unlink(commitmessagefile)
- except OSError:
- pass
+ os.unlink(commitmessagefile)
+ except OSError:
+ pass
+ if retval != os.EX_OK:
+ writemsg_level(
+ "!!! Exiting on %s (shell) "
+ "error code: %s\n" % (self.vcs_settings.vcs,
retval),
+ level=logging.ERROR, noiselevel=-1)
+ sys.exit(retval)
def priming_commit(self, myupdates, myremoved, commitmessage):
@@ -503,29 +476,18 @@ class Actions(object):
# so strip the prefix.
myfiles = [f.lstrip("./") for f in myfiles]
- commit_cmd = [self.vcs_settings.vcs]
- commit_cmd.extend(self.vcs_settings.vcs_global_opts)
- commit_cmd.append("commit")
- commit_cmd.extend(self.vcs_settings.vcs_local_opts)
- commit_cmd.extend(["-F", commitmessagefile])
- commit_cmd.extend(myfiles)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 3dc78ff91eccd81c972ceadf59d059aabe9ccdbc
Author: Brian Dolbec gentoo org>
AuthorDate: Tue Mar 15 18:40:06 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3dc78ff9
repoman: Create a new boolean Fuse type
Create a Fuse type which implememts a boolean as a one time fuse.
The Fuse is initialized True, then is pop()'d to become False.
Once the Fuse is blown, it can not be reset to True.
Convert the use of the dynamic_data variable 'allvalid' to a Fuse instance.
pym/repoman/fuse.py | 68 +
pym/repoman/modules/scan/ebuild/ebuild.py | 7 ++-
pym/repoman/modules/scan/ebuild/isebuild.py | 13 +++---
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/scanner.py | 2 +
5 files changed, 83 insertions(+), 9 deletions(-)
diff --git a/pym/repoman/fuse.py b/pym/repoman/fuse.py
new file mode 100644
index 000..ac864fd
--- /dev/null
+++ b/pym/repoman/fuse.py
@@ -0,0 +1,68 @@
+
+'''
+fuse.py
+
+A tiny one-time-fuse class that uses a boolean to mimic the property of
+an electrical fuse. IT's good (True) until it is popped (bad, False).
+It is not resetable.
+'''
+
+
+class Fuse(object):
+ '''A One time fuse style boolean instance'''
+
+ __slots__ = ('_state')
+
+ def __init__(self):
+ self._state = True
+
+ def pop(self):
+ '''Blow's the fuse state (makes it False)'''
+ self._state = False
+
+ def __repr__(self):
+ '''x.__repr__() <==> repr(x)'''
+ return repr(self._state>0)
+
+ def __str__(self):
+ '''x.__str__() <==> str(x)'''
+ return ['False', 'True'][self._state]
+
+ def __bool__(self):
+ '''self != 0'''
+ return self._state != 0
+
+ def __nonzero__(self):
+ '''self != 0'''
+ return self._state != 0
+
+ def __abs__(self):
+ '''x.__abs__() <==> abs(x)'''
+ return [0, 1] [self._state]
+
+ def __int__(self):
+ '''int(self)'''
+ return [0, 1][self._state]
+
+ def __eq__(self, value):
+ '''Return self==value.'''
+ return self._state == value
+
+ def __ne__(self, value):
+ '''Return self!=value.'''
+ return self._state != value
+
+ def __ge__(self, value):
+ '''Return self>=value.'''
+ return self._state >= value
+
+ def __gt__(self, value):
+ return self._state > value
+
+ def __le__(self, value):
+ '''Return self<=value.'''
+ return self._state <= value
+
+ def __lt__(self, value):
+ '''Return self
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 32399303c9c8b8965c9cd8a10fd6c87542d01bc3
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 1 07:47:49 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32399303
repoman/actions.py: fix copyright update (bug 405017)
The Actions.perform method failed to update copyright for new and
changed files with --echangelog=n, so fix it to handle this case.
X-Gentoo-Bug: 405017
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=405017
pym/repoman/actions.py | 9 +
pym/repoman/utilities.py | 7 ---
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/pym/repoman/actions.py b/pym/repoman/actions.py
index c0cd5cc..643e24c 100644
--- a/pym/repoman/actions.py
+++ b/pym/repoman/actions.py
@@ -9,6 +9,7 @@ import platform
import signal
import sys
import tempfile
+import time
from itertools import chain
from _emerge.UserQuery import UserQuery
@@ -22,6 +23,7 @@ from portage.output import (
from portage.package.ebuild.digestgen import digestgen
from portage.util import writemsg_level
+from repoman.copyrights import update_copyright
from repoman.gpg import gpgsign, need_signature
from repoman import utilities
from repoman.modules.vcs.vcs import vcs_files_to_cps
@@ -111,6 +113,13 @@ class Actions(object):
commitmessage = commitmessage.rstrip()
+ # Update copyright for new and changed files
+ year = time.strftime('%Y', time.gmtime())
+ for fn in chain(mynew, mychanged):
+ if fn.endswith('.diff') or fn.endswith('.patch'):
+ continue
+ update_copyright(fn, year, pretend=self.options.pretend)
+
myupdates, broken_changelog_manifests = self.changelogs(
myupdates, mymanifests, myremoved,
mychanged, myautoadd,
mynew, commitmessage)
diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py
index 47b5aab..8a757dc 100644
--- a/pym/repoman/utilities.py
+++ b/pym/repoman/utilities.py
@@ -395,13 +395,6 @@ def UpdateChangeLog(
year = time.strftime('%Y', gmtime)
date = time.strftime('%d %b %Y', gmtime)
- # check modified files and the ChangeLog for copyright updates
- # patches and diffs (identified by .patch and .diff) are excluded
- for fn in chain(new, changed):
- if fn.endswith('.diff') or fn.endswith('.patch'):
- continue
- update_copyright(os.path.join(pkgdir, fn), year,
pretend=pretend)
-
cl_path = os.path.join(pkgdir, 'ChangeLog')
clold_lines = []
clnew_lines = []
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/arches/, pym/repoman/modules/scan/depend/, pym/repoman/
commit: 80cfdec3df20d1a29d1e9eccc7f2ed666a72e1ea
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 25 03:57:03 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=80cfdec3
ProfileDependsChecks: convert ArchChecks to function
pym/repoman/modules/scan/arches/__init__.py| 30 --
pym/repoman/modules/scan/arches/arches.py | 78 --
pym/repoman/modules/scan/depend/__init__.py| 1 -
pym/repoman/modules/scan/depend/_gen_arches.py | 57 +++
pym/repoman/modules/scan/depend/profile.py | 5 +-
pym/repoman/scanner.py | 2 +-
6 files changed, 61 insertions(+), 112 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/__init__.py
b/pym/repoman/modules/scan/arches/__init__.py
deleted file mode 100644
index d66be1e..000
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2015-2016 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-doc = """Arches plug-in module for repoman.
-Performs archs checks on ebuilds."""
-__doc__ = doc[:]
-
-
-module_spec = {
- 'name': 'arches',
- 'description': doc,
- 'provides':{
- 'archs-module': {
- 'name': "arches",
- 'sourcefile': "arches",
- 'class': "ArchChecks",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['options', 'repo_settings', 'profiles'
- ],
- 'func_kwargs': {
- 'arches': ('Future', 'set'),
- 'ebuild': (None, None),
- },
- },
- }
-}
-
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
deleted file mode 100644
index b86848d..000
--- a/pym/repoman/modules/scan/arches/arches.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from repoman.modules.scan.scanbase import ScanBase
-
-
-class ArchChecks(ScanBase):
- '''Perform arch keyword checks'''
-
- def __init__(self, **kwargs):
- '''Class init
-
- @param options: the run time cli options
- @param repo_settings: repository settings instance
- @param profiles: dictionary
- '''
- self.options = kwargs.get('options')
- self.repo_settings = kwargs.get('repo_settings')
- self.profiles = kwargs.get('profiles')
-
- def check(self, **kwargs):
- '''Determines the arches for the ebuild following the profile
rules
-
- @param ebuild: Ebuild which we check (object).
- @returns: dictionary, including arches set
- '''
- ebuild = kwargs.get('ebuild').get()
- if self.options.ignore_arches:
- arches = [[
- self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ARCH"],
-
self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
- else:
- arches = set()
- for keyword in ebuild.keywords:
- if keyword[0] == "-":
- continue
- elif keyword[0] == "~":
- arch = keyword[1:]
- if arch == "*":
- for expanded_arch in
self.profiles:
- if expanded_arch ==
"**":
- continue
- arches.add(
- (keyword,
expanded_arch, (
-
expanded_arch, "~" + expanded_arch)))
- else:
- arches.add((keyword, arch,
(arch, keyword)))
- else:
- # For ebuilds with stable keywords,
check if the
- # dependencies are satisfiable for
unstable
- # configurations, since use.stable.mask
is not
- # applied for unstable configurations
(see bug
- # 563546).
- if keyword == "*":
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: edd6f6cd73839dad09ce7c65c09ac38c30e6b5eb
Author: Brian Dolbec gentoo org>
AuthorDate: Sat Apr 16 23:29:20 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=edd6f6cd
repoman: Limit the kwargs passed to the module's __init__()
This has the benefit of forcing the module to declare what data it requires.
Rather than possibly silently fail.
pym/repoman/scanner.py | 17 ++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index e9a8e20..5ac519e 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -202,7 +202,18 @@ class Scanner(object):
'fetches', 'pkgmetadata']:
mod_class = MODULE_CONTROLLER.get_class(mod)
logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod_class.__name__] =
mod_class(**self.kwargs)
+ self.modules[mod_class.__name__] =
mod_class(**self.set_kwargs(mod))
+
+ def set_kwargs(self, mod):
+ '''Creates a limited set of kwargs to pass to the module's
__init__()
+
+ @param mod: module name string
+ @returns: dictionary
+ '''
+ kwargs = {}
+ for key in MODULE_CONTROLLER.modules[mod]['mod_kwargs']:
+ kwargs[key] = self.kwargs[key]
+ return kwargs
def scan_pkgs(self, can_force):
for xpkg in self.effective_scanlist:
@@ -295,7 +306,7 @@ class Scanner(object):
if mod[0]:
mod_class =
MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name:
%s", mod_class.__name__)
- self.modules[mod[1]] =
mod_class(**self.kwargs)
+ self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod))
logging.debug("scan_ebuilds: module: %s",
mod[1])
do_it, functions =
self.modules[mod[1]].runInEbuilds
logging.debug("do_it: %s, functions: %s",
do_it, [x.__name__ for x in functions])
@@ -329,7 +340,7 @@ class Scanner(object):
if mod[0]:
mod_class = MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.kwargs)
+ self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod))
logging.debug("scan_ebuilds final checks: module: %s",
mod[1])
do_it, functions = self.modules[mod[1]].runInFinal
logging.debug("do_it: %s, functions: %s", do_it,
[x.__name__ for x in functions])
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: fa237acb3e15d18877237f034194cf879e879170 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 16 19:57:08 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Apr 25 15:28:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa237acb XmlLint: remove incorred self._is_capable setting This setting interferred with the fix for bug 567746. pym/repoman/_xml.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py index f7ff9fb..d55dda5 100644 --- a/pym/repoman/_xml.py +++ b/pym/repoman/_xml.py @@ -60,7 +60,6 @@ class XmlLint(object): self.repoman_settings = repoman_settings self._is_capable = metadata_dtd is not None self.binary = None - self._is_capable = False self._check_capable() def _check_capable(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/ebuild/
commit: 45f57e9b8dc5c582e786a7eb7d8679abd59958fe
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Mar 16 02:00:20 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=45f57e9b
repoman: Migrate 'can_force' to the new Fuse class
pym/repoman/main.py | 5 +++--
pym/repoman/modules/scan/ebuild/isebuild.py | 5 +++--
pym/repoman/scanner.py | 3 +--
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/pym/repoman/main.py b/pym/repoman/main.py
index 337e638..2ccda99 100755
--- a/pym/repoman/main.py
+++ b/pym/repoman/main.py
@@ -25,6 +25,7 @@ from portage.util import formatter
from repoman.actions import Actions
from repoman.argparser import parse_args
+from repoman.fuse import Fuse
from repoman.qa_data import (
format_qa_output, format_qa_output_column, qahelp,
qawarnings, qacats)
@@ -75,7 +76,7 @@ def repoman_main(argv):
# Set this to False when an extraordinary issue (generally
# something other than a QA issue) makes it impossible to
# commit (like if Manifest generation fails).
- can_force = True
+ can_force = Fuse()
portdir, portdir_overlay, mydir =
utilities.FindPortdir(repoman_settings)
if portdir is None:
@@ -110,7 +111,7 @@ def repoman_main(argv):
# Perform the main checks
scanner = Scanner(repo_settings, myreporoot, config_root, options,
vcs_settings, mydir, env)
- can_force = scanner.scan_pkgs(can_force)
+ scanner.scan_pkgs(can_force)
commitmessage = None
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py
b/pym/repoman/modules/scan/ebuild/isebuild.py
index 474a874..514d23e 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -42,6 +42,7 @@ class IsEbuild(ScanBase):
checkdir = kwargs.get('checkdir')
xpkg = kwargs.get('xpkg')
fuse = kwargs.get('validity_fuse')
+ can_force = kwargs.get('can_force')
self.continue_ = False
ebuildlist = []
pkgs = {}
@@ -85,9 +86,9 @@ class IsEbuild(ScanBase):
# metadata leads to false positives for several checks,
and false
# positives confuse users.
self.continue_ = True
+ can_force.pop()
- return {'continue': self.continue_, 'pkgs': pkgs,
- 'can_force': not self.continue_}
+ return {'continue': self.continue_, 'pkgs': pkgs}
@property
def runInPkgs(self):
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 9c6f5ac..a9f56e9 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -205,7 +205,6 @@ class Scanner(object):
self.modules[mod_class.__name__] =
mod_class(**self.kwargs)
def scan_pkgs(self, can_force):
- dynamic_data = {'can_force': can_force}
for xpkg in self.effective_scanlist:
xpkg_continue = False
# ebuilds and digests added to cvs respectively.
@@ -268,7 +267,7 @@ class Scanner(object):
dynamic_data["changelog_modified"] = changelog_path in
self.changed.changelogs
self._scan_ebuilds(ebuildlist, dynamic_data)
- return dynamic_data['can_force']
+ return
def _scan_ebuilds(self, ebuildlist, dynamic_data):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/depend/, pym/repoman/modules/scan/eclasses/, ...
commit: 7ec0953fd023a6285beaea0421b438de2b88a2e6
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 25 01:15:36 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7ec0953f
repoman: Convert all pointer type dynamic_data to the new ExtendedFuture class
pym/repoman/main.py| 13 ++---
pym/repoman/modules/scan/arches/__init__.py| 4 +-
pym/repoman/modules/scan/arches/arches.py | 5 +-
pym/repoman/modules/scan/depend/__init__.py| 22 +--
pym/repoman/modules/scan/depend/depend.py | 15 +++--
pym/repoman/modules/scan/depend/profile.py | 10 ++--
pym/repoman/modules/scan/depend/unknown.py | 6 +-
pym/repoman/modules/scan/directories/__init__.py | 12 +++-
pym/repoman/modules/scan/directories/files.py | 2 +-
pym/repoman/modules/scan/directories/mtime.py | 4 +-
pym/repoman/modules/scan/eapi/__init__.py | 3 +-
pym/repoman/modules/scan/eapi/eapi.py | 2 +-
pym/repoman/modules/scan/ebuild/__init__.py| 29 ++---
pym/repoman/modules/scan/ebuild/ebuild.py | 18 +++---
pym/repoman/modules/scan/ebuild/isebuild.py| 17 ++
pym/repoman/modules/scan/ebuild/multicheck.py | 4 +-
pym/repoman/modules/scan/eclasses/__init__.py | 12 +++-
pym/repoman/modules/scan/eclasses/live.py | 11 ++--
pym/repoman/modules/scan/eclasses/ruby.py | 4 +-
pym/repoman/modules/scan/fetch/__init__.py | 9 ++-
pym/repoman/modules/scan/fetch/fetches.py | 3 +-
pym/repoman/modules/scan/keywords/__init__.py | 9 ++-
pym/repoman/modules/scan/keywords/keywords.py | 6 +-
pym/repoman/modules/scan/manifest/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 39 +
pym/repoman/modules/scan/metadata/description.py | 4 +-
.../modules/scan/metadata/ebuild_metadata.py | 11 ++--
pym/repoman/modules/scan/metadata/license.py | 4 +-
pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 ++--
pym/repoman/modules/scan/metadata/restrict.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py| 6 +-
pym/repoman/modules/scan/mirrors/__init__.py | 4 +-
.../modules/scan/mirrors/thirdpartymirrors.py | 5 +-
pym/repoman/modules/scan/scanbase.py | 36
pym/repoman/modules/scan/status/__init__.py| 7 ++-
pym/repoman/modules/scan/status/vcsstatus.py | 5 +-
pym/repoman/modules/scan/use/__init__.py | 10 +++-
pym/repoman/modules/scan/use/use_flags.py | 10 +---
pym/repoman/scanner.py | 68 +-
39 files changed, 255 insertions(+), 192 deletions(-)
diff --git a/pym/repoman/main.py b/pym/repoman/main.py
index 62c3c2c..2009a44 100755
--- a/pym/repoman/main.py
+++ b/pym/repoman/main.py
@@ -22,8 +22,8 @@ import portage.repository.config
from portage.output import create_color_func, nocolor
from portage.output import ConsoleStyleFile, StyleWriter
from portage.util import formatter
-from portage.util.futures import (
- Future,
+from portage.util.futures.extendedfutures import (
+ ExtendedFuture,
InvalidStateError,
)
@@ -79,7 +79,7 @@ def repoman_main(argv):
# Set this to False when an extraordinary issue (generally
# something other than a QA issue) makes it impossible to
# commit (like if Manifest generation fails).
- can_force = Future()
+ can_force = ExtendedFuture(True)
portdir, portdir_overlay, mydir =
utilities.FindPortdir(repoman_settings)
if portdir is None:
@@ -174,14 +174,9 @@ def repoman_main(argv):
qa_output = qa_output.getvalue()
qa_output = qa_output.splitlines(True)
- try:
- can_force = can_force.result()
- except InvalidStateError:
- can_force = True
-
# output the results
actions = Actions(repo_settings, options, scanner, vcs_settings)
- if actions.inform(can_force, result):
+ if actions.inform(can_force.get(), result):
# perform any other actions
actions.perform(qa_output)
diff --git a/pym/repoman/modules/scan/arches/__init__.py
b/pym/repoman/modules/scan/arches/__init__.py
index e4a4806..d66be1e 100644
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ b/pym/repoman/modules/scan/arches/__init__.py
@@ -20,7 +20,9 @@ module_spec = {
},
'mod_kwargs': ['options', 'repo_settings', 'profiles'
],
- 'func_kwargs': {'ebuild': None, 'arches': 'set',
+ 'func_kwargs': {
+ 'arches': ('Future', 'set'),
+ 'ebuild': (None, None),
},
},
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 926e2dc2975d1bea54454014cda714b426b3d5b9
Author: Brian Dolbec gentoo org>
AuthorDate: Tue Mar 15 18:45:45 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=926e2dc2
repoman/modules.scan/metadata/unused.py: Add docstrings
Remove un-needed override functions
pym/repoman/modules/scan/metadata/unused.py | 21 +
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/pym/repoman/modules/scan/metadata/unused.py
b/pym/repoman/modules/scan/metadata/unused.py
index 9ff7e56..a58a614 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -3,11 +3,23 @@ from repoman.modules.scan.scanbase import ScanBase
class UnusedCheck(ScanBase):
+ '''Checks and reports any un-used metadata.xml use flag descriptions'''
def __init__(self, **kwargs):
+ '''UnusedCheck init function
+
+ @param qatracker: QATracker instance
+ '''
self.qatracker = kwargs.get('qatracker')
def check(self, **kwargs):
+ '''Reports on any unused metadata.xml use descriptions
+
+ @param xpkg: the pacakge being checked
+ @param muselist: use flag list
+ @param used_useflags: use flag list
+ @param validity_fuse: Fuse instance
+ '''
xpkg = kwargs.get('xpkg')
muselist = kwargs.get('muselist')
used_useflags = kwargs.get('used_useflags')
@@ -22,13 +34,6 @@ class UnusedCheck(ScanBase):
return {'continue': False}
@property
- def runInPkgs(self):
- return (False, [])
-
- @property
- def runInEbuilds(self):
- return (False, [])
-
- @property
def runInFinal(self):
+ '''Final scans at the package level'''
return (True, [self.check])
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/depend/
commit: 902abebcad19a0d598103664a6afcdb0fc211edc
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 18 07:32:54 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=902abebc
repoman: Remove 'type_list' from dynamic_data
IT was not used outside of the DependChecks.check()
pym/repoman/modules/scan/depend/__init__.py | 3 +--
pym/repoman/modules/scan/depend/depend.py | 2 --
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/pym/repoman/modules/scan/depend/__init__.py
b/pym/repoman/modules/scan/depend/__init__.py
index 735604d..cf5b415 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -21,8 +21,7 @@ module_spec = {
'mod_kwargs': ['qatracker', 'portdb'
],
'func_kwargs': {'ebuild': None, 'pkg': None,
'unknown_pkgs': 'set',
- 'type_list': 'list', 'badlicsyntax': 'Future',
- 'baddepsyntax': 'Future',
+ 'badlicsyntax': 'Future', 'baddepsyntax':
'Future',
},
},
'profile-module': {
diff --git a/pym/repoman/modules/scan/depend/depend.py
b/pym/repoman/modules/scan/depend/depend.py
index 0efe5de..a5f91d3 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -146,8 +146,6 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
dyn_unknown.update(unknown_pkgs)
- dyn_type_list = kwargs.get('type_list')
- dyn_type_list.extend(type_list)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
(kwargs.get('baddepsyntax'), baddepsyntax),
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/
commit: 28d182722ce4941f455b54bd47a58e317a189e2a Author: Brian Dolbec gentoo org> AuthorDate: Wed Mar 30 14:50:57 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Apr 25 15:28:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=28d18272 repoman/modules/scan.py: Add docstrings pym/repoman/modules/scan/scan.py | 14 ++ 1 file changed, 14 insertions(+) diff --git a/pym/repoman/modules/scan/scan.py b/pym/repoman/modules/scan/scan.py index 4901500..d2a5f51 100644 --- a/pym/repoman/modules/scan/scan.py +++ b/pym/repoman/modules/scan/scan.py @@ -1,5 +1,10 @@ # -*- coding:utf-8 -*- +''' +moudules/scan.py +Module specific package scan list generator +''' + import logging import os import sys @@ -8,6 +13,15 @@ from repoman.errors import caterror def scan(repolevel, reposplit, startdir, categories, repo_settings): + '''Generate a list of pkgs to scan + + @param repolevel: integer, number of subdirectories deep from the tree root + @param reposplit: list of the path subdirs + @param startdir: the top level directory to begin scanning from + @param categories: list of known categories + @param repo_settings: repository settings instance + @returns: scanlist, sorted list of pkgs to scan + ''' scanlist = [] if repolevel == 2: # we are inside a category directory
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/depend/, pym/repoman/modules/scan/arches/, ...
commit: a05acb46919638ef60a4303c2bc6d92e9e6c0771
Author: Brian Dolbec gentoo org>
AuthorDate: Sun Apr 24 01:34:37 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a05acb46
More new data structure change fixes
Fix arches not being cleared every ebuild.
pym/repoman/modules/scan/arches/arches.py | 1 +
pym/repoman/modules/scan/depend/depend.py | 1 +
pym/repoman/modules/scan/ebuild/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 2 +-
pym/repoman/modules/scan/ebuild/isebuild.py | 11 --
pym/repoman/modules/scan/eclasses/live.py | 5 ++-
pym/repoman/modules/scan/keywords/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/modules/scan/use/use_flags.py | 13 +++---
pym/repoman/scanner.py| 57 +--
11 files changed, 62 insertions(+), 38 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..4df25a8 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
+ #dyn_arches.clear()
dyn_arches.update(arches)
return False
diff --git a/pym/repoman/modules/scan/depend/depend.py
b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
+ dyn_unknown.clear()
dyn_unknown.update(unknown_pkgs)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py
b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
'func_kwargs': {'checkdirlist': 'list', 'checkdir':
None, 'xpkg': None,
- 'validity_fuse': None, 'can_force': None,
'pkgs': 'dict',
+ 'validity_future': 'Future', 'can_force': None,
'pkgs': 'dict',
},
},
'ebuild-module': {
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
self.qatracker.add_error(
"ebuild.notadded", self.xpkg + "/" +
self.y_ebuild + ".ebuild")
# update the dynamic data
- self.set_result_raise([('ebuild', self)])
+ self.set_result_raise([(kwargs.get('ebuild'), self)])
return False
def set_pkg_data(self, **kwargs):
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py
b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
from repoman.qa_data import no_exec, allvars
from repoman.modules.scan.scanbase import ScanBase
+
class IsEbuild(ScanBase):
'''Performs basic tests to confirm it is an ebuild'''
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
self.continue_ = True
self.set_result_pass([(can_force, False)])
# set our updated data
- self.set_result_raise([
- (kwargs.get('pkgs'), pkgs),
- ])
+ dyn_pkgs = kwargs.get('pkgs')
+ # clear() sets it to None,
+ # we don't want to kill the pointer reference
+ # just set it back to an empty dict()
+ for key in list(dyn_pkgs):
+ dyn_pkgs.pop(key)
+ dyn_pkgs.update(pkgs)
return self.continue_
@property
diff --git a/pym/repoman/modules/scan/eclasses/live.py
b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/, ...
commit: 93428acb8af7f5b90fb6a3ce5a21ee0e2110a4f5
Author: Zac Medico gentoo org>
AuthorDate: Sat Apr 16 20:24:06 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=93428acb
repoman: replace Fuse with Future
Replace Fuse with Future, which is similar more generic. The
code ends up being slightly more verbose, but more flexible.
The Future class will be useful elsewhere, including the
EventLoop class.
pym/portage/util/futures.py | 118
pym/repoman/fuse.py | 68
pym/repoman/main.py | 12 ++-
pym/repoman/modules/scan/ebuild/ebuild.py | 10 ++-
pym/repoman/modules/scan/ebuild/isebuild.py | 25 --
pym/repoman/modules/scan/metadata/unused.py | 10 ++-
pym/repoman/scanner.py | 4 +-
7 files changed, 164 insertions(+), 83 deletions(-)
diff --git a/pym/portage/util/futures.py b/pym/portage/util/futures.py
new file mode 100644
index 000..c648f10
--- /dev/null
+++ b/pym/portage/util/futures.py
@@ -0,0 +1,118 @@
+# Copyright 2016 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+#
+# For compatibility with python versions which do not have the
+# asyncio module (Python 3.3 and earlier), this module provides a
+# subset of the asyncio.futures.Futures interface.
+
+from __future__ import unicode_literals
+
+__all__ = (
+ 'CancelledError',
+ 'Future',
+ 'InvalidStateError',
+)
+
+try:
+ from asyncio import (
+ CancelledError,
+ Future,
+ InvalidStateError,
+ )
+except ImportError:
+
+ from portage.exception import PortageException
+
+ _PENDING = 'PENDING'
+ _CANCELLED = 'CANCELLED'
+ _FINISHED = 'FINISHED'
+
+ class Error(PortageException):
+ pass
+
+ class CancelledError(Error):
+ def __init__(self):
+ Error.__init__(self, "cancelled")
+
+ class InvalidStateError(Error):
+ pass
+
+ class Future(object):
+
+ # Class variables serving as defaults for instance variables.
+ _state = _PENDING
+ _result = None
+ _exception = None
+
+ def cancel(self):
+ """Cancel the future and schedule callbacks.
+
+ If the future is already done or cancelled, return
False. Otherwise,
+ change the future's state to cancelled, schedule the
callbacks and
+ return True.
+ """
+ if self._state != _PENDING:
+ return False
+ self._state = _CANCELLED
+ return True
+
+ def done(self):
+ """Return True if the future is done.
+
+ Done means either that a result / exception are
available, or that the
+ future was cancelled.
+ """
+ return self._state != _PENDING
+
+ def result(self):
+ """Return the result this future represents.
+
+ If the future has been cancelled, raises
CancelledError. If the
+ future's result isn't yet available, raises
InvalidStateError. If
+ the future is done and has an exception set, this
exception is raised.
+ """
+ if self._state == _CANCELLED:
+ raise CancelledError()
+ if self._state != _FINISHED:
+ raise InvalidStateError('Result is not ready.')
+ if self._exception is not None:
+ raise self._exception
+ return self._result
+
+ def exception(self):
+ """Return the exception that was set on this future.
+
+ The exception (or None if no exception was set) is
returned only if
+ the future is done. If the future has been cancelled,
raises
+ CancelledError. If the future isn't done yet, raises
+ InvalidStateError.
+ """
+ if self._state == _CANCELLED:
+ raise CancelledError
+ if self._state != _FINISHED:
+ raise InvalidStateError('Exception is not set.')
+ return self._exception
+
+ def set_result(self, result):
+ """Mark the future done and set its result.
+
+ If the future is already done when this method is
called,
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, ...
commit: c24cac30a56bd7922d977999752b19ed3f64a300
Author: Brian Dolbec gentoo org>
AuthorDate: Sun Apr 17 15:39:40 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c24cac30
repoman: Populate module_spec's func_kwargs for all scan modules
First step towards a propoerly controlled module data exchange interface.
pym/repoman/modules/scan/arches/__init__.py | 2 +-
pym/repoman/modules/scan/depend/__init__.py | 10 +++---
pym/repoman/modules/scan/directories/__init__.py | 5 +++--
pym/repoman/modules/scan/eapi/__init__.py| 2 +-
pym/repoman/modules/scan/ebuild/__init__.py | 10 +++---
pym/repoman/modules/scan/eclasses/__init__.py| 5 +++--
pym/repoman/modules/scan/fetch/__init__.py | 4 +++-
pym/repoman/modules/scan/keywords/__init__.py| 3 ++-
pym/repoman/modules/scan/manifest/__init__.py| 2 +-
pym/repoman/modules/scan/metadata/__init__.py| 16 ++--
pym/repoman/modules/scan/mirrors/__init__.py | 2 +-
pym/repoman/modules/scan/status/__init__.py | 3 ++-
pym/repoman/modules/scan/use/__init__.py | 4 +++-
13 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/__init__.py
b/pym/repoman/modules/scan/arches/__init__.py
index b87e07c..e4a4806 100644
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ b/pym/repoman/modules/scan/arches/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['options', 'repo_settings', 'profiles'
],
- 'func_kwargs': {'ebuild': None,
+ 'func_kwargs': {'ebuild': None, 'arches': 'set',
},
},
}
diff --git a/pym/repoman/modules/scan/depend/__init__.py
b/pym/repoman/modules/scan/depend/__init__.py
index a0f2f9a..a2b8534 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -20,7 +20,9 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'portdb'
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'pkg': None,
'unknown_pkgs': 'set',
+ 'type_list': [], 'badlicsyntax': 'Future',
+ 'baddepsyntax': 'Future',
},
},
'profile-module': {
@@ -35,7 +37,8 @@ module_spec = {
'repo_settings', 'include_arches', 'caches',
'repoman_incrementals', 'env', 'have',
'dev_keywords'
],
- 'func_kwargs': {
+ 'func_kwargs': {'arches': 'set', 'ebuild': None, 'pkg':
None,
+ 'unknown_pkgs': 'set', 'baddepsyntax': 'Future',
},
},
'unknown-module': {
@@ -48,7 +51,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'unknown_pkgs': 'set',
+ 'baddepsyntax': 'Future',
},
},
}
diff --git a/pym/repoman/modules/scan/directories/__init__.py
b/pym/repoman/modules/scan/directories/__init__.py
index d50ace5..0193843 100644
--- a/pym/repoman/modules/scan/directories/__init__.py
+++ b/pym/repoman/modules/scan/directories/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings',
'vcs_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'checkdir': None, 'checkdirlist': None,
+ 'checkdir_relative': None, 'changed': None,
},
},
'mtime-module': {
@@ -33,7 +34,7 @@ module_spec = {
},
'mod_kwargs': ['vcs_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'changed': None, 'pkg':
None,
},
},
}
diff --git a/pym/repoman/modules/scan/eapi/__init__.py
b/pym/repoman/modules/scan/eapi/__init__.py
index 63618ab..376e79a 100644
--- a/pym/repoman/modules/scan/eapi/__init__.py
+++ b/pym/repoman/modules/scan/eapi/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'repo_settings'
],
- 'func_kwargs': {
+ 'func_kwargs':
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/depend/, pym/repoman/modules/scan/arches/, ...
commit: aea571e7b772f4c1cf23399dc6e685a123d6982f
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 18 07:25:17 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=aea571e7
WIP: repoman: convert all modules to the new data transfer system
Use Future instances for all non-pointer type variables modified by the modules.
Create set_result_pass() and set_result_raise() in ScanBase to reduce code
duplication.
Create a get_result() in ScanBase which returns either a result or an
unset_value rather
than raise an InvalidStateError.
pym/repoman/modules/scan/arches/arches.py | 7 +++-
pym/repoman/modules/scan/depend/__init__.py| 8 ++--
pym/repoman/modules/scan/depend/depend.py | 19 ++---
pym/repoman/modules/scan/depend/unknown.py | 10 ++---
pym/repoman/modules/scan/directories/files.py | 2 +-
pym/repoman/modules/scan/directories/mtime.py | 6 +--
pym/repoman/modules/scan/eapi/eapi.py | 4 +-
pym/repoman/modules/scan/ebuild/__init__.py| 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 31 +--
pym/repoman/modules/scan/ebuild/isebuild.py| 28 +
pym/repoman/modules/scan/ebuild/multicheck.py | 4 +-
pym/repoman/modules/scan/eclasses/live.py | 16
pym/repoman/modules/scan/eclasses/ruby.py | 6 +--
pym/repoman/modules/scan/fetch/fetches.py | 4 +-
pym/repoman/modules/scan/keywords/keywords.py | 8 ++--
pym/repoman/modules/scan/manifest/manifests.py | 6 +--
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/description.py | 6 +--
.../modules/scan/metadata/ebuild_metadata.py | 16
pym/repoman/modules/scan/metadata/license.py | 6 +--
pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 +-
pym/repoman/modules/scan/metadata/restrict.py | 4 +-
pym/repoman/modules/scan/metadata/unused.py| 9 ++---
.../modules/scan/mirrors/thirdpartymirrors.py | 8 ++--
pym/repoman/modules/scan/options/options.py| 4 +-
pym/repoman/modules/scan/scanbase.py | 38 +-
pym/repoman/modules/scan/status/vcsstatus.py | 6 ++-
pym/repoman/modules/scan/use/use_flags.py | 15 ---
pym/repoman/scanner.py | 46 --
29 files changed, 207 insertions(+), 126 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 90ec04d..8bc7a24 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -23,7 +23,7 @@ class ArchChecks(ScanBase):
@param ebuild: Ebuild which we check (object).
@returns: dictionary, including arches set
'''
- ebuild = kwargs.get('ebuild')
+ ebuild = kwargs.get('ebuild').result()
if self.options.ignore_arches:
arches = [[
self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ARCH"],
@@ -67,7 +67,10 @@ class ArchChecks(ScanBase):
# Use an empty profile for checking
dependencies of
# packages that have empty KEYWORDS.
arches.add(('**', '**', ('**',)))
- return {'continue': False, 'arches': arches}
+ # update the dynamic data
+ dyn_arches = kwargs.get('arches')
+ dyn_arches.update(arches)
+ return False
@property
def runInEbuilds(self):
diff --git a/pym/repoman/modules/scan/depend/__init__.py
b/pym/repoman/modules/scan/depend/__init__.py
index a2b8534..735604d 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['qatracker', 'portdb'
],
'func_kwargs': {'ebuild': None, 'pkg': None,
'unknown_pkgs': 'set',
- 'type_list': [], 'badlicsyntax': 'Future',
+ 'type_list': 'list', 'badlicsyntax': 'Future',
'baddepsyntax': 'Future',
},
},
@@ -37,8 +37,8 @@ module_spec = {
'repo_settings', 'include_arches', 'caches',
'repoman_incrementals', 'env', 'have',
'dev_keywords'
],
- 'func_kwargs': {'arches': 'set', 'ebuild': None, 'pkg':
None,
- 'unknown_pkgs': 'set', 'baddepsyntax': 'Future',
+ 'func_kwargs': {'arches': None, 'ebuild': None,
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/keywords/, pym/repoman/modules/scan/mirrors/, ...
commit: d6c9768f1ad11e8222a04dcb3057c0b0d91abc94
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Mar 16 17:10:40 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d6c9768f
repoman: Rough out an initial new data requirements listing in module_spec
pym/repoman/modules/scan/arches/__init__.py | 4
pym/repoman/modules/scan/depend/__init__.py | 14 ++
pym/repoman/modules/scan/directories/__init__.py | 8
pym/repoman/modules/scan/eapi/__init__.py| 4
pym/repoman/modules/scan/ebuild/__init__.py | 12
pym/repoman/modules/scan/eclasses/__init__.py| 8
pym/repoman/modules/scan/fetch/__init__.py | 4
pym/repoman/modules/scan/keywords/__init__.py| 4
pym/repoman/modules/scan/manifest/__init__.py| 4
pym/repoman/modules/scan/metadata/__init__.py| 24
pym/repoman/modules/scan/mirrors/__init__.py | 4
pym/repoman/modules/scan/options/__init__.py | 4
pym/repoman/modules/scan/status/__init__.py | 4
pym/repoman/modules/scan/use/__init__.py | 4
14 files changed, 102 insertions(+)
diff --git a/pym/repoman/modules/scan/arches/__init__.py
b/pym/repoman/modules/scan/arches/__init__.py
index d080c30..b87e07c 100644
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ b/pym/repoman/modules/scan/arches/__init__.py
@@ -18,6 +18,10 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
+ 'mod_kwargs': ['options', 'repo_settings', 'profiles'
+ ],
+ 'func_kwargs': {'ebuild': None,
+ },
},
}
}
diff --git a/pym/repoman/modules/scan/depend/__init__.py
b/pym/repoman/modules/scan/depend/__init__.py
index 9fd7970..a0f2f9a 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -18,6 +18,10 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
+ 'mod_kwargs': ['qatracker', 'portdb'
+ ],
+ 'func_kwargs': {
+ },
},
'profile-module': {
'name': "profile",
@@ -27,6 +31,12 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
+ 'mod_kwargs': ['qatracker', 'portdb', 'profiles',
'options',
+ 'repo_settings', 'include_arches', 'caches',
+ 'repoman_incrementals', 'env', 'have',
'dev_keywords'
+ ],
+ 'func_kwargs': {
+ },
},
'unknown-module': {
'name': "unknown",
@@ -36,6 +46,10 @@ module_spec = {
'functions': ['check'],
'func_desc': {
},
+ 'mod_kwargs': ['qatracker',
+ ],
+ 'func_kwargs': {
+ },
},
}
}
diff --git a/pym/repoman/modules/scan/directories/__init__.py
b/pym/repoman/modules/scan/directories/__init__.py
index ec18d30..d50ace5 100644
--- a/pym/repoman/modules/scan/directories/__init__.py
+++ b/pym/repoman/modules/scan/directories/__init__.py
@@ -18,6 +18,10 @@ module_spec = {
'functions': ['check'],
'func_kwargs': {
},
+ 'mod_kwargs': ['portdb', 'qatracker', 'repo_settings',
'vcs_settings',
+ ],
+ 'func_kwargs': {
+ },
},
'mtime-module': {
'name': "mtime",
@@ -27,6 +31,10 @@ module_spec = {
'functions': ['check'],
'func_kwargs': {
},
+ 'mod_kwargs': ['vcs_settings',
+ ],
+ 'func_kwargs': {
+ },
},
}
}
diff --git a/pym/repoman/modules/scan/eapi/__init__.py
b/pym/repoman/modules/scan/eapi/__init__.py
index 39bab4d..63618ab 100644
--- a/pym/repoman/modules/scan/eapi/__init__.py
+++ b/pym/repoman/modules/scan/eapi/__init__.py
@@ -18,6 +18,10 @@ module_spec = {
'functions': ['check'],
'func_kwargs': {
},
+ 'mod_kwargs': ['qatracker', 'repo_settings'
+ ],
+ 'func_kwargs': {
+
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/status/, pym/repoman/modules/scan/use/, ...
commit: 8bb6fedaf4e5ccd163735d9997b0e67bd2038f0d
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Mar 7 21:04:01 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:28:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8bb6feda
repoman: Add docstrings to the final vcs and scan modules
pym/repoman/modules/scan/arches/arches.py | 13
pym/repoman/modules/scan/depend/depend.py | 13
pym/repoman/modules/scan/depend/profile.py | 5 +-
pym/repoman/modules/scan/depend/unknown.py | 13
pym/repoman/modules/scan/directories/files.py | 1 +
pym/repoman/modules/scan/directories/mtime.py | 8 ++
pym/repoman/modules/scan/eapi/eapi.py | 2 +
pym/repoman/modules/scan/ebuild/ebuild.py | 22 +-
pym/repoman/modules/scan/ebuild/isebuild.py| 3 +-
pym/repoman/modules/scan/ebuild/multicheck.py | 4 +-
pym/repoman/modules/scan/eclasses/live.py | 6 ++
pym/repoman/modules/scan/eclasses/ruby.py | 8 +-
pym/repoman/modules/scan/fetch/fetches.py | 1 +
pym/repoman/modules/scan/keywords/keywords.py | 1 +
pym/repoman/modules/scan/manifest/manifests.py | 25 ++
pym/repoman/modules/scan/metadata/pkgmetadata.py | 1 +
.../modules/scan/mirrors/thirdpartymirrors.py | 1 +
pym/repoman/modules/scan/options/options.py| 9 +++
pym/repoman/modules/scan/status/vcsstatus.py | 1 +
pym/repoman/modules/scan/use/use_flags.py | 1 +
pym/repoman/modules/vcs/None/changes.py| 15 +++-
pym/repoman/modules/vcs/bzr/changes.py | 13 +++-
pym/repoman/modules/vcs/bzr/status.py | 2 +-
pym/repoman/modules/vcs/changes.py | 90 +++---
pym/repoman/modules/vcs/cvs/changes.py | 54 +
pym/repoman/modules/vcs/cvs/status.py | 2 +-
pym/repoman/modules/vcs/git/changes.py | 24 +-
pym/repoman/modules/vcs/git/status.py | 5 ++
pym/repoman/modules/vcs/hg/changes.py | 21 -
pym/repoman/modules/vcs/hg/status.py | 2 +-
pym/repoman/modules/vcs/settings.py| 13
pym/repoman/modules/vcs/svn/changes.py | 42 ++
pym/repoman/modules/vcs/svn/status.py | 2 +-
pym/repoman/modules/vcs/vcs.py | 13 +++-
34 files changed, 358 insertions(+), 78 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 641a790..90ec04d 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -4,13 +4,25 @@ from repoman.modules.scan.scanbase import ScanBase
class ArchChecks(ScanBase):
+ '''Perform arch keyword checks'''
def __init__(self, **kwargs):
+ '''Class init
+
+ @param options: the run time cli options
+ @param repo_settings: repository settings instance
+ @param profiles: dictionary
+ '''
self.options = kwargs.get('options')
self.repo_settings = kwargs.get('repo_settings')
self.profiles = kwargs.get('profiles')
def check(self, **kwargs):
+ '''Determines the arches for the ebuild following the profile
rules
+
+ @param ebuild: Ebuild which we check (object).
+ @returns: dictionary, including arches set
+ '''
ebuild = kwargs.get('ebuild')
if self.options.ignore_arches:
arches = [[
@@ -59,4 +71,5 @@ class ArchChecks(ScanBase):
@property
def runInEbuilds(self):
+ '''Ebuild level scans'''
return (True, [self.check])
diff --git a/pym/repoman/modules/scan/depend/depend.py
b/pym/repoman/modules/scan/depend/depend.py
index 810a94a..eaafc95 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -11,12 +11,24 @@ from repoman.qa_data import suspect_virtual, suspect_rdepend
class DependChecks(ScanBase):
+ '''Perform dependency checks'''
def __init__(self, **kwargs):
+ '''
+ @param portdb: portdb instance
+ @param qatracker: QATracker instance
+ '''
self.qatracker = kwargs.get('qatracker')
self.portdb = kwargs.get('portdb')
def check(self, **kwargs):
+ '''Checks the ebuild dependencies for errors
+
+ @param pkg: Package in which we check (object).
+ @param ebuild: Ebuild which we check (object).
+ @returns: dictionary including {unknown_pkgs, type_list,
+
badlicsyntax, baddepsyntax}
+ '''
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: b275450533245b5097b5f6c08a84df999ed2c989
Author: Michał Górny gentoo org>
AuthorDate: Sun Apr 17 08:06:28 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 15:03:37 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2754505
repoman: Use XML Schema for metadata.xml validation
pym/repoman/_xml.py | 16 +-
pym/repoman/metadata.py | 39
pym/repoman/modules/scan/metadata/__init__.py| 2 +-
pym/repoman/modules/scan/metadata/pkgmetadata.py | 6 ++--
pym/repoman/scanner.py | 8 ++---
5 files changed, 36 insertions(+), 35 deletions(-)
diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
index d55dda5..33a536a 100644
--- a/pym/repoman/_xml.py
+++ b/pym/repoman/_xml.py
@@ -12,7 +12,7 @@ from portage import os
from portage.output import red
from portage.process import find_binary
-from repoman.metadata import fetch_metadata_dtd
+from repoman.metadata import fetch_metadata_xsd
from repoman._subprocess import repoman_getstatusoutput
@@ -53,12 +53,12 @@ class
_MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
class XmlLint(object):
- def __init__(self, options, repoman_settings, metadata_dtd=None):
- self.metadata_dtd = (metadata_dtd or
- os.path.join(repoman_settings["DISTDIR"],
'metadata.dtd'))
+ def __init__(self, options, repoman_settings, metadata_xsd=None):
+ self.metadata_xsd = (metadata_xsd or
+ os.path.join(repoman_settings["DISTDIR"],
'metadata.xsd'))
self.options = options
self.repoman_settings = repoman_settings
- self._is_capable = metadata_dtd is not None
+ self._is_capable = metadata_xsd is not None
self.binary = None
self._check_capable()
@@ -69,7 +69,7 @@ class XmlLint(object):
if not self.binary:
print(red("!!! xmllint not found. Can't check
metadata.xml.\n"))
elif not self._is_capable:
- if not fetch_metadata_dtd(self.metadata_dtd,
self.repoman_settings):
+ if not fetch_metadata_xsd(self.metadata_xsd,
self.repoman_settings):
sys.exit(1)
# this can be problematic if xmllint changes their
output
self._is_capable = True
@@ -93,8 +93,8 @@ class XmlLint(object):
# xmlint can produce garbage output even on success, so only
dump
# the ouput when it fails.
st, out = repoman_getstatusoutput(
- self.binary + " --nonet --noout --dtdvalid %s %s" % (
- portage._shell_quote(self.metadata_dtd),
+ self.binary + " --nonet --noout --schema %s %s" % (
+ portage._shell_quote(self.metadata_xsd),
portage._shell_quote(
os.path.join(checkdir,
"metadata.xml"
if st != os.EX_OK:
diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index e95ad41..7a514dc 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -33,8 +33,9 @@ metadata_xml_declaration = '' \
% (metadata_xml_encoding,)
metadata_doctype_name = 'pkgmetadata'
metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
+metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd'
# force refetch if the local copy creation time is older than this
-metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
+metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
def parse_metadata_use(xml_tree):
@@ -86,36 +87,36 @@ def parse_metadata_use(xml_tree):
return uselist
-def fetch_metadata_dtd(metadata_dtd, repoman_settings):
+def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
- Fetch metadata.dtd if it doesn't exist or the ctime is older than
- metadata_dtd_ctime_interval.
+ Fetch metadata.xsd if it doesn't exist or the ctime is older than
+ metadata_xsd_ctime_interval.
@rtype: bool
@return: True if successful, otherwise False
"""
must_fetch = True
- metadata_dtd_st = None
+ metadata_xsd_st = None
current_time = int(time.time())
try:
- metadata_dtd_st = os.stat(metadata_dtd)
+ metadata_xsd_st = os.stat(metadata_xsd)
except EnvironmentError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
else:
- # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
- if abs(current_time - metadata_dtd_st.st_ctime) \
- < metadata_dtd_ctime_interval:
+
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/depend/, pym/repoman/, pym/repoman/modules/scan/arches/
commit: 4d370b84a917b351eaaa55f0d181909f42add18d
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 25 03:57:03 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 25 04:04:35 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4d370b84
ProfileDependsChecks: convert ArchChecks to function
pym/repoman/modules/scan/arches/__init__.py| 30 --
pym/repoman/modules/scan/arches/arches.py | 78 --
pym/repoman/modules/scan/depend/__init__.py| 1 -
pym/repoman/modules/scan/depend/_gen_arches.py | 57 +++
pym/repoman/modules/scan/depend/profile.py | 5 +-
pym/repoman/scanner.py | 2 +-
6 files changed, 61 insertions(+), 112 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/__init__.py
b/pym/repoman/modules/scan/arches/__init__.py
deleted file mode 100644
index d66be1e..000
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2015-2016 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-doc = """Arches plug-in module for repoman.
-Performs archs checks on ebuilds."""
-__doc__ = doc[:]
-
-
-module_spec = {
- 'name': 'arches',
- 'description': doc,
- 'provides':{
- 'archs-module': {
- 'name': "arches",
- 'sourcefile': "arches",
- 'class': "ArchChecks",
- 'description': doc,
- 'functions': ['check'],
- 'func_desc': {
- },
- 'mod_kwargs': ['options', 'repo_settings', 'profiles'
- ],
- 'func_kwargs': {
- 'arches': ('Future', 'set'),
- 'ebuild': (None, None),
- },
- },
- }
-}
-
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
deleted file mode 100644
index b86848d..000
--- a/pym/repoman/modules/scan/arches/arches.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from repoman.modules.scan.scanbase import ScanBase
-
-
-class ArchChecks(ScanBase):
- '''Perform arch keyword checks'''
-
- def __init__(self, **kwargs):
- '''Class init
-
- @param options: the run time cli options
- @param repo_settings: repository settings instance
- @param profiles: dictionary
- '''
- self.options = kwargs.get('options')
- self.repo_settings = kwargs.get('repo_settings')
- self.profiles = kwargs.get('profiles')
-
- def check(self, **kwargs):
- '''Determines the arches for the ebuild following the profile
rules
-
- @param ebuild: Ebuild which we check (object).
- @returns: dictionary, including arches set
- '''
- ebuild = kwargs.get('ebuild').get()
- if self.options.ignore_arches:
- arches = [[
- self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ARCH"],
-
self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
- else:
- arches = set()
- for keyword in ebuild.keywords:
- if keyword[0] == "-":
- continue
- elif keyword[0] == "~":
- arch = keyword[1:]
- if arch == "*":
- for expanded_arch in
self.profiles:
- if expanded_arch ==
"**":
- continue
- arches.add(
- (keyword,
expanded_arch, (
-
expanded_arch, "~" + expanded_arch)))
- else:
- arches.add((keyword, arch,
(arch, keyword)))
- else:
- # For ebuilds with stable keywords,
check if the
- # dependencies are satisfiable for
unstable
- # configurations, since use.stable.mask
is not
- # applied for unstable configurations
(see bug
- # 563546).
- if keyword == "*":
-
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/arches/
commit: 1cafabf2b61ac43c0ff8694ec3d895fc68a0cac3
Author: Zac Medico gentoo org>
AuthorDate: Sat Apr 23 23:54:41 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 03:03:12 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1cafabf2
ArchChecks: don't mix arches between ebuilds
Fix ArchChecks to not mix arches of ebuilds together, so that
errors are only reported for those arches that the ebuild has
keywords for.
pym/repoman/modules/scan/arches/arches.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 4df25a8..6e1c17d 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,7 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
- #dyn_arches.clear()
+ dyn_arches.clear()
dyn_arches.update(arches)
return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, ...
commit: 4a19fdd05e2aad4c0c97202c784f9d2f49538c67
Author: Brian Dolbec gentoo org>
AuthorDate: Sun Apr 24 01:34:37 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 03:03:04 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a19fdd0
More new data structure change fixes
Fix arches not being cleared every ebuild.
pym/repoman/modules/scan/arches/arches.py | 1 +
pym/repoman/modules/scan/depend/depend.py | 1 +
pym/repoman/modules/scan/ebuild/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 2 +-
pym/repoman/modules/scan/ebuild/isebuild.py | 11 --
pym/repoman/modules/scan/eclasses/live.py | 5 ++-
pym/repoman/modules/scan/keywords/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/modules/scan/use/use_flags.py | 13 +++---
pym/repoman/scanner.py| 57 +--
11 files changed, 62 insertions(+), 38 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..4df25a8 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
+ #dyn_arches.clear()
dyn_arches.update(arches)
return False
diff --git a/pym/repoman/modules/scan/depend/depend.py
b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
+ dyn_unknown.clear()
dyn_unknown.update(unknown_pkgs)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py
b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
'func_kwargs': {'checkdirlist': 'list', 'checkdir':
None, 'xpkg': None,
- 'validity_fuse': None, 'can_force': None,
'pkgs': 'dict',
+ 'validity_future': 'Future', 'can_force': None,
'pkgs': 'dict',
},
},
'ebuild-module': {
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
self.qatracker.add_error(
"ebuild.notadded", self.xpkg + "/" +
self.y_ebuild + ".ebuild")
# update the dynamic data
- self.set_result_raise([('ebuild', self)])
+ self.set_result_raise([(kwargs.get('ebuild'), self)])
return False
def set_pkg_data(self, **kwargs):
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py
b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
from repoman.qa_data import no_exec, allvars
from repoman.modules.scan.scanbase import ScanBase
+
class IsEbuild(ScanBase):
'''Performs basic tests to confirm it is an ebuild'''
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
self.continue_ = True
self.set_result_pass([(can_force, False)])
# set our updated data
- self.set_result_raise([
- (kwargs.get('pkgs'), pkgs),
- ])
+ dyn_pkgs = kwargs.get('pkgs')
+ # clear() sets it to None,
+ # we don't want to kill the pointer reference
+ # just set it back to an empty dict()
+ for key in list(dyn_pkgs):
+ dyn_pkgs.pop(key)
+ dyn_pkgs.update(pkgs)
return self.continue_
@property
diff --git a/pym/repoman/modules/scan/eclasses/live.py
b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/keywords/, pym/repoman/modules/scan/metadata/, ...
commit: d0454110c604d328777f3391854299a449e66f89
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 25 01:15:36 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Apr 25 03:05:50 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d0454110
repoman: Convert all pointer type dynamic_data to the new ExtendedFuture class
pym/repoman/main.py| 13 ++---
pym/repoman/modules/scan/arches/__init__.py| 4 +-
pym/repoman/modules/scan/arches/arches.py | 5 +-
pym/repoman/modules/scan/depend/__init__.py| 22 +--
pym/repoman/modules/scan/depend/depend.py | 15 +++--
pym/repoman/modules/scan/depend/profile.py | 10 ++--
pym/repoman/modules/scan/depend/unknown.py | 6 +-
pym/repoman/modules/scan/directories/__init__.py | 12 +++-
pym/repoman/modules/scan/directories/files.py | 2 +-
pym/repoman/modules/scan/directories/mtime.py | 4 +-
pym/repoman/modules/scan/eapi/__init__.py | 3 +-
pym/repoman/modules/scan/eapi/eapi.py | 2 +-
pym/repoman/modules/scan/ebuild/__init__.py| 29 ++---
pym/repoman/modules/scan/ebuild/ebuild.py | 18 +++---
pym/repoman/modules/scan/ebuild/isebuild.py| 17 ++
pym/repoman/modules/scan/ebuild/multicheck.py | 4 +-
pym/repoman/modules/scan/eclasses/__init__.py | 12 +++-
pym/repoman/modules/scan/eclasses/live.py | 11 ++--
pym/repoman/modules/scan/eclasses/ruby.py | 4 +-
pym/repoman/modules/scan/fetch/__init__.py | 9 ++-
pym/repoman/modules/scan/fetch/fetches.py | 3 +-
pym/repoman/modules/scan/keywords/__init__.py | 9 ++-
pym/repoman/modules/scan/keywords/keywords.py | 6 +-
pym/repoman/modules/scan/manifest/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 39 +
pym/repoman/modules/scan/metadata/description.py | 4 +-
.../modules/scan/metadata/ebuild_metadata.py | 11 ++--
pym/repoman/modules/scan/metadata/license.py | 4 +-
pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 ++--
pym/repoman/modules/scan/metadata/restrict.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py| 6 +-
pym/repoman/modules/scan/mirrors/__init__.py | 4 +-
.../modules/scan/mirrors/thirdpartymirrors.py | 5 +-
pym/repoman/modules/scan/scanbase.py | 36
pym/repoman/modules/scan/status/__init__.py| 7 ++-
pym/repoman/modules/scan/status/vcsstatus.py | 5 +-
pym/repoman/modules/scan/use/__init__.py | 10 +++-
pym/repoman/modules/scan/use/use_flags.py | 10 +---
pym/repoman/scanner.py | 68 +-
39 files changed, 255 insertions(+), 192 deletions(-)
diff --git a/pym/repoman/main.py b/pym/repoman/main.py
index 62c3c2c..2009a44 100755
--- a/pym/repoman/main.py
+++ b/pym/repoman/main.py
@@ -22,8 +22,8 @@ import portage.repository.config
from portage.output import create_color_func, nocolor
from portage.output import ConsoleStyleFile, StyleWriter
from portage.util import formatter
-from portage.util.futures import (
- Future,
+from portage.util.futures.extendedfutures import (
+ ExtendedFuture,
InvalidStateError,
)
@@ -79,7 +79,7 @@ def repoman_main(argv):
# Set this to False when an extraordinary issue (generally
# something other than a QA issue) makes it impossible to
# commit (like if Manifest generation fails).
- can_force = Future()
+ can_force = ExtendedFuture(True)
portdir, portdir_overlay, mydir =
utilities.FindPortdir(repoman_settings)
if portdir is None:
@@ -174,14 +174,9 @@ def repoman_main(argv):
qa_output = qa_output.getvalue()
qa_output = qa_output.splitlines(True)
- try:
- can_force = can_force.result()
- except InvalidStateError:
- can_force = True
-
# output the results
actions = Actions(repo_settings, options, scanner, vcs_settings)
- if actions.inform(can_force, result):
+ if actions.inform(can_force.get(), result):
# perform any other actions
actions.perform(qa_output)
diff --git a/pym/repoman/modules/scan/arches/__init__.py
b/pym/repoman/modules/scan/arches/__init__.py
index e4a4806..d66be1e 100644
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ b/pym/repoman/modules/scan/arches/__init__.py
@@ -20,7 +20,9 @@ module_spec = {
},
'mod_kwargs': ['options', 'repo_settings', 'profiles'
],
- 'func_kwargs': {'ebuild': None, 'arches': 'set',
+ 'func_kwargs': {
+ 'arches': ('Future', 'set'),
+ 'ebuild': (None, None),
},
},
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/arches/
commit: 9c050581d75e4aebafd6b8f854877f6b6d621a38
Author: Zac Medico gentoo org>
AuthorDate: Sat Apr 23 23:54:41 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 24 01:43:53 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9c050581
ArchChecks: don't mix arches between ebuilds
Fix ArchChecks to not mix arches of ebuilds together, so that
errors are only reported for those arches that the ebuild has
keywords for.
pym/repoman/modules/scan/arches/arches.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 4df25a8..6e1c17d 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,7 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
- #dyn_arches.clear()
+ dyn_arches.clear()
dyn_arches.update(arches)
return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/use/, pym/repoman/modules/scan/ebuild/, ...
commit: 963ff0806c5ba897bd7ec5179989ab0576b6d9a1
Author: Brian Dolbec gentoo org>
AuthorDate: Sun Apr 24 01:34:37 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun Apr 24 01:34:37 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=963ff080
More new data structure change fixes
Fix arches not being cleared every ebuild.
pym/repoman/modules/scan/arches/arches.py | 1 +
pym/repoman/modules/scan/depend/depend.py | 1 +
pym/repoman/modules/scan/ebuild/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 2 +-
pym/repoman/modules/scan/ebuild/isebuild.py | 11 +++--
pym/repoman/modules/scan/eclasses/live.py | 5 ++-
pym/repoman/modules/scan/keywords/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/modules/scan/use/use_flags.py | 13 +++---
pym/repoman/scanner.py| 62 ++-
11 files changed, 67 insertions(+), 38 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..4df25a8 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
+ #dyn_arches.clear()
dyn_arches.update(arches)
return False
diff --git a/pym/repoman/modules/scan/depend/depend.py
b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
+ dyn_unknown.clear()
dyn_unknown.update(unknown_pkgs)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py
b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
'func_kwargs': {'checkdirlist': 'list', 'checkdir':
None, 'xpkg': None,
- 'validity_fuse': None, 'can_force': None,
'pkgs': 'dict',
+ 'validity_future': 'Future', 'can_force': None,
'pkgs': 'dict',
},
},
'ebuild-module': {
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
self.qatracker.add_error(
"ebuild.notadded", self.xpkg + "/" +
self.y_ebuild + ".ebuild")
# update the dynamic data
- self.set_result_raise([('ebuild', self)])
+ self.set_result_raise([(kwargs.get('ebuild'), self)])
return False
def set_pkg_data(self, **kwargs):
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py
b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
from repoman.qa_data import no_exec, allvars
from repoman.modules.scan.scanbase import ScanBase
+
class IsEbuild(ScanBase):
'''Performs basic tests to confirm it is an ebuild'''
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
self.continue_ = True
self.set_result_pass([(can_force, False)])
# set our updated data
- self.set_result_raise([
- (kwargs.get('pkgs'), pkgs),
- ])
+ dyn_pkgs = kwargs.get('pkgs')
+ # clear() sets it to None,
+ # we don't want to kill the pointer reference
+ # just set it back to an empty dict()
+ for key in list(dyn_pkgs):
+ dyn_pkgs.pop(key)
+ dyn_pkgs.update(pkgs)
return self.continue_
@property
diff --git a/pym/repoman/modules/scan/eclasses/live.py
b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, ...
commit: 8838c4317981315741df4334d5a1a172902f20a0
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Apr 20 16:52:21 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Sun Apr 24 00:45:21 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8838c431
More new data structure change fixes
Fix arches not being cleared every ebuild.
pym/repoman/modules/scan/arches/arches.py | 1 +
pym/repoman/modules/scan/depend/depend.py | 1 +
pym/repoman/modules/scan/ebuild/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 2 +-
pym/repoman/modules/scan/ebuild/isebuild.py | 11 +++--
pym/repoman/modules/scan/eclasses/live.py | 5 ++-
pym/repoman/modules/scan/keywords/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/modules/scan/use/use_flags.py | 13 +++---
pym/repoman/scanner.py| 62 ++-
11 files changed, 67 insertions(+), 38 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..6e1c17d 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
+ dyn_arches.clear()
dyn_arches.update(arches)
return False
diff --git a/pym/repoman/modules/scan/depend/depend.py
b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
+ dyn_unknown.clear()
dyn_unknown.update(unknown_pkgs)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py
b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
'func_kwargs': {'checkdirlist': 'list', 'checkdir':
None, 'xpkg': None,
- 'validity_fuse': None, 'can_force': None,
'pkgs': 'dict',
+ 'validity_future': 'Future', 'can_force': None,
'pkgs': 'dict',
},
},
'ebuild-module': {
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py
b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
self.qatracker.add_error(
"ebuild.notadded", self.xpkg + "/" +
self.y_ebuild + ".ebuild")
# update the dynamic data
- self.set_result_raise([('ebuild', self)])
+ self.set_result_raise([(kwargs.get('ebuild'), self)])
return False
def set_pkg_data(self, **kwargs):
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py
b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
from repoman.qa_data import no_exec, allvars
from repoman.modules.scan.scanbase import ScanBase
+
class IsEbuild(ScanBase):
'''Performs basic tests to confirm it is an ebuild'''
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
self.continue_ = True
self.set_result_pass([(can_force, False)])
# set our updated data
- self.set_result_raise([
- (kwargs.get('pkgs'), pkgs),
- ])
+ dyn_pkgs = kwargs.get('pkgs')
+ # clear() sets it to None,
+ # we don't want to kill the pointer reference
+ # just set it back to an empty dict()
+ for key in list(dyn_pkgs):
+ dyn_pkgs.pop(key)
+ dyn_pkgs.update(pkgs)
return self.continue_
@property
diff --git a/pym/repoman/modules/scan/eclasses/live.py
b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/
commit: 96dc91e77da08c7ba30974ee65c9f3ae876e5b39
Author: Brian Dolbec gentoo org>
AuthorDate: Sat Apr 16 23:29:20 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 21 16:51:34 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=96dc91e7
repoman: Limit the kwargs passed to the module's __init__()
This has the benefit of forcing the module to declare what data it requires.
Rather than possibly silently fail.
pym/repoman/scanner.py | 17 ++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index e9a8e20..5ac519e 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -202,7 +202,18 @@ class Scanner(object):
'fetches', 'pkgmetadata']:
mod_class = MODULE_CONTROLLER.get_class(mod)
logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod_class.__name__] =
mod_class(**self.kwargs)
+ self.modules[mod_class.__name__] =
mod_class(**self.set_kwargs(mod))
+
+ def set_kwargs(self, mod):
+ '''Creates a limited set of kwargs to pass to the module's
__init__()
+
+ @param mod: module name string
+ @returns: dictionary
+ '''
+ kwargs = {}
+ for key in MODULE_CONTROLLER.modules[mod]['mod_kwargs']:
+ kwargs[key] = self.kwargs[key]
+ return kwargs
def scan_pkgs(self, can_force):
for xpkg in self.effective_scanlist:
@@ -295,7 +306,7 @@ class Scanner(object):
if mod[0]:
mod_class =
MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name:
%s", mod_class.__name__)
- self.modules[mod[1]] =
mod_class(**self.kwargs)
+ self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod))
logging.debug("scan_ebuilds: module: %s",
mod[1])
do_it, functions =
self.modules[mod[1]].runInEbuilds
logging.debug("do_it: %s, functions: %s",
do_it, [x.__name__ for x in functions])
@@ -329,7 +340,7 @@ class Scanner(object):
if mod[0]:
mod_class = MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.kwargs)
+ self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod))
logging.debug("scan_ebuilds final checks: module: %s",
mod[1])
do_it, functions = self.modules[mod[1]].runInFinal
logging.debug("do_it: %s, functions: %s", do_it,
[x.__name__ for x in functions])
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/arches/, pym/repoman/modules/scan/ebuild/, ...
commit: 8ecc34680ea95b2f69242427c040986f76be3a27
Author: Brian Dolbec gentoo org>
AuthorDate: Sun Apr 17 15:39:40 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 21 16:51:35 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8ecc3468
repoman: Populate module_spec's func_kwargs for all scan modules
First step towards a propoerly controlled module data exchange interface.
pym/repoman/modules/scan/arches/__init__.py | 2 +-
pym/repoman/modules/scan/depend/__init__.py | 10 +++---
pym/repoman/modules/scan/directories/__init__.py | 5 +++--
pym/repoman/modules/scan/eapi/__init__.py| 2 +-
pym/repoman/modules/scan/ebuild/__init__.py | 10 +++---
pym/repoman/modules/scan/eclasses/__init__.py| 5 +++--
pym/repoman/modules/scan/fetch/__init__.py | 4 +++-
pym/repoman/modules/scan/keywords/__init__.py| 3 ++-
pym/repoman/modules/scan/manifest/__init__.py| 2 +-
pym/repoman/modules/scan/metadata/__init__.py| 16 ++--
pym/repoman/modules/scan/mirrors/__init__.py | 2 +-
pym/repoman/modules/scan/status/__init__.py | 3 ++-
pym/repoman/modules/scan/use/__init__.py | 4 +++-
13 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/__init__.py
b/pym/repoman/modules/scan/arches/__init__.py
index b87e07c..e4a4806 100644
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ b/pym/repoman/modules/scan/arches/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['options', 'repo_settings', 'profiles'
],
- 'func_kwargs': {'ebuild': None,
+ 'func_kwargs': {'ebuild': None, 'arches': 'set',
},
},
}
diff --git a/pym/repoman/modules/scan/depend/__init__.py
b/pym/repoman/modules/scan/depend/__init__.py
index a0f2f9a..a2b8534 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -20,7 +20,9 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'portdb'
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'pkg': None,
'unknown_pkgs': 'set',
+ 'type_list': [], 'badlicsyntax': 'Future',
+ 'baddepsyntax': 'Future',
},
},
'profile-module': {
@@ -35,7 +37,8 @@ module_spec = {
'repo_settings', 'include_arches', 'caches',
'repoman_incrementals', 'env', 'have',
'dev_keywords'
],
- 'func_kwargs': {
+ 'func_kwargs': {'arches': 'set', 'ebuild': None, 'pkg':
None,
+ 'unknown_pkgs': 'set', 'baddepsyntax': 'Future',
},
},
'unknown-module': {
@@ -48,7 +51,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'unknown_pkgs': 'set',
+ 'baddepsyntax': 'Future',
},
},
}
diff --git a/pym/repoman/modules/scan/directories/__init__.py
b/pym/repoman/modules/scan/directories/__init__.py
index d50ace5..0193843 100644
--- a/pym/repoman/modules/scan/directories/__init__.py
+++ b/pym/repoman/modules/scan/directories/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings',
'vcs_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'checkdir': None, 'checkdirlist': None,
+ 'checkdir_relative': None, 'changed': None,
},
},
'mtime-module': {
@@ -33,7 +34,7 @@ module_spec = {
},
'mod_kwargs': ['vcs_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'changed': None, 'pkg':
None,
},
},
}
diff --git a/pym/repoman/modules/scan/eapi/__init__.py
b/pym/repoman/modules/scan/eapi/__init__.py
index 63618ab..376e79a 100644
--- a/pym/repoman/modules/scan/eapi/__init__.py
+++ b/pym/repoman/modules/scan/eapi/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'repo_settings'
],
- 'func_kwargs': {
+ 'func_kwargs':
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/scan/status/, pym/repoman/modules/scan/, ...
commit: aa5607f80125f8096a9283b4323e460b40fa250e
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Apr 18 07:25:17 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 21 16:51:35 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=aa5607f8
WIP: repoman: convert all modules to the new data transfer system
Use Future instances for all non-pointer type variables modified by the modules.
Create set_result_pass() and set_result_raise() in ScanBase to reduce code
duplication.
Create a get_result() in ScanBase which returns either a result or an
unset_value rather
than raise an InvalidStateError.
pym/repoman/modules/scan/arches/arches.py | 7 +++-
pym/repoman/modules/scan/depend/__init__.py| 8 ++--
pym/repoman/modules/scan/depend/depend.py | 19 ++---
pym/repoman/modules/scan/depend/unknown.py | 10 ++---
pym/repoman/modules/scan/directories/files.py | 2 +-
pym/repoman/modules/scan/directories/mtime.py | 6 +--
pym/repoman/modules/scan/eapi/eapi.py | 4 +-
pym/repoman/modules/scan/ebuild/__init__.py| 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 31 +--
pym/repoman/modules/scan/ebuild/isebuild.py| 28 +
pym/repoman/modules/scan/ebuild/multicheck.py | 4 +-
pym/repoman/modules/scan/eclasses/live.py | 16
pym/repoman/modules/scan/eclasses/ruby.py | 6 +--
pym/repoman/modules/scan/fetch/fetches.py | 4 +-
pym/repoman/modules/scan/keywords/keywords.py | 8 ++--
pym/repoman/modules/scan/manifest/manifests.py | 6 +--
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/description.py | 6 +--
.../modules/scan/metadata/ebuild_metadata.py | 16
pym/repoman/modules/scan/metadata/license.py | 6 +--
pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 +-
pym/repoman/modules/scan/metadata/restrict.py | 4 +-
pym/repoman/modules/scan/metadata/unused.py| 9 ++---
.../modules/scan/mirrors/thirdpartymirrors.py | 8 ++--
pym/repoman/modules/scan/options/options.py| 4 +-
pym/repoman/modules/scan/scanbase.py | 38 +-
pym/repoman/modules/scan/status/vcsstatus.py | 6 ++-
pym/repoman/modules/scan/use/use_flags.py | 15 ---
pym/repoman/scanner.py | 46 --
29 files changed, 207 insertions(+), 126 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py
b/pym/repoman/modules/scan/arches/arches.py
index 90ec04d..8bc7a24 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -23,7 +23,7 @@ class ArchChecks(ScanBase):
@param ebuild: Ebuild which we check (object).
@returns: dictionary, including arches set
'''
- ebuild = kwargs.get('ebuild')
+ ebuild = kwargs.get('ebuild').result()
if self.options.ignore_arches:
arches = [[
self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ARCH"],
@@ -67,7 +67,10 @@ class ArchChecks(ScanBase):
# Use an empty profile for checking
dependencies of
# packages that have empty KEYWORDS.
arches.add(('**', '**', ('**',)))
- return {'continue': False, 'arches': arches}
+ # update the dynamic data
+ dyn_arches = kwargs.get('arches')
+ dyn_arches.update(arches)
+ return False
@property
def runInEbuilds(self):
diff --git a/pym/repoman/modules/scan/depend/__init__.py
b/pym/repoman/modules/scan/depend/__init__.py
index a2b8534..735604d 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['qatracker', 'portdb'
],
'func_kwargs': {'ebuild': None, 'pkg': None,
'unknown_pkgs': 'set',
- 'type_list': [], 'badlicsyntax': 'Future',
+ 'type_list': 'list', 'badlicsyntax': 'Future',
'baddepsyntax': 'Future',
},
},
@@ -37,8 +37,8 @@ module_spec = {
'repo_settings', 'include_arches', 'caches',
'repoman_incrementals', 'env', 'have',
'dev_keywords'
],
- 'func_kwargs': {'arches': 'set', 'ebuild': None, 'pkg':
None,
- 'unknown_pkgs': 'set', 'baddepsyntax': 'Future',
+ 'func_kwargs': {'arches': None, 'ebuild': None,
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/modules/vcs/, pym/repoman/modules/vcs/git/
commit: 9a3a6a7b129cd54230207a784cc0927a68f1b3ae
Author: Brian Dolbec gentoo org>
AuthorDate: Wed Feb 10 18:05:46 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Apr 21 16:49:28 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9a3a6a7b
repoman: Migrate vcs index update code to the vcs modules
pym/repoman/actions.py | 22 +-
pym/repoman/modules/vcs/changes.py | 4
pym/repoman/modules/vcs/git/changes.py | 27 +++
3 files changed, 32 insertions(+), 21 deletions(-)
diff --git a/pym/repoman/actions.py b/pym/repoman/actions.py
index 4d06555..6c9de57 100644
--- a/pym/repoman/actions.py
+++ b/pym/repoman/actions.py
@@ -164,27 +164,7 @@ class Actions(object):
if self.repo_settings.sign_manifests:
self.sign_manifest(myupdates, myremoved, mymanifests)
- if self.vcs_settings.vcs == 'git':
- # It's not safe to use the git commit -a option since
there might
- # be some modified files elsewhere in the working tree
that the
- # user doesn't want to commit. Therefore, call git
update-index
- # in order to ensure that the index is updated with the
latest
- # versions of all new and modified files in the
relevant portion
- # of the working tree.
- myfiles = mymanifests + myupdates
- myfiles.sort()
- update_index_cmd = ["git", "update-index"]
- update_index_cmd.extend(f.lstrip("./") for f in myfiles)
- if self.options.pretend:
- print("(%s)" % (" ".join(update_index_cmd),))
- else:
- retval = spawn(update_index_cmd, env=os.environ)
- if retval != os.EX_OK:
- writemsg_level(
- "!!! Exiting on %s (shell) "
- "error code: %s\n" %
(self.vcs_settings.vcs, retval),
- level=logging.ERROR,
noiselevel=-1)
- sys.exit(retval)
+ self.vcs_settings.changes.update_index(mymanifests, myupdates)
self.add_manifest(mymanifests, myheaders, myupdates, myremoved,
commitmessage)
diff --git a/pym/repoman/modules/vcs/changes.py
b/pym/repoman/modules/vcs/changes.py
index 27b627f..948407c 100644
--- a/pym/repoman/modules/vcs/changes.py
+++ b/pym/repoman/modules/vcs/changes.py
@@ -85,3 +85,7 @@ class ChangesBase(object):
def clear_attic(myheaders):
'''Old CVS leftover'''
pass
+
+ def update_index(self, mymanifests, myupdates):
+ '''Update the vcs's modified index if it is needed'''
+ pass
diff --git a/pym/repoman/modules/vcs/git/changes.py
b/pym/repoman/modules/vcs/git/changes.py
index 1970b3a..018458c 100644
--- a/pym/repoman/modules/vcs/git/changes.py
+++ b/pym/repoman/modules/vcs/git/changes.py
@@ -2,11 +2,16 @@
Git module Changes class submodule
'''
+import logging
+import sys
+
from repoman.modules.vcs.changes import ChangesBase
from repoman._subprocess import repoman_popen
from repoman._portage import portage
from portage import os
from portage.package.ebuild.digestgen import digestgen
+from portage.process import spawn
+from portage.util import writemsg_level
class Changes(ChangesBase):
@@ -63,3 +68,25 @@ class Changes(ChangesBase):
for x in broken_changelog_manifests:
self.repoman_settings["O"] =
os.path.join(self.repo_settings.repodir, x)
digestgen(mysettings=self.repoman_settings,
myportdb=self.repo_settings.portdb)
+
+ def update_index(self, mymanifests, myupdates):
+ # It's not safe to use the git commit -a option since there
might
+ # be some modified files elsewhere in the working tree that the
+ # user doesn't want to commit. Therefore, call git update-index
+ # in order to ensure that the index is updated with the latest
+ # versions of all new and modified files in the relevant portion
+ # of the working tree.
+ myfiles = mymanifests + myupdates
+ myfiles.sort()
+ update_index_cmd = ["git", "update-index"]
+ update_index_cmd.extend(f.lstrip("./") for f in myfiles)
+ if self.options.pretend:
+ print("(%s)" % (" ".join(update_index_cmd),))
+ else:
+ retval = spawn(update_index_cmd, env=os.environ)
+ if retval != os.EX_OK:
+
