commit: fd99fc9fb0e348b7151f867da8b2dcb2b305f173
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 10 17:39:04 2017 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Jul 10 22:29:33 2017 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fd99fc9f
repoman: Create a new ModuleConfig class
This class will load the module configuration data from the target repo.
Move the Modules class handling code to this new class.
Update scanner.py to use the new ModuleConfig class for Modules class
calls.
Move the hard-coded loop lists to use the ModuleConfig instance
configure/dynamically determined lists.
TODO: add masters inheritance config stacking.
repoman/pym/repoman/modules/scan/module.py | 84 ++++++++++++++++++++++++++++++
repoman/pym/repoman/scanner.py | 83 ++++++++++++-----------------
2 files changed, 118 insertions(+), 49 deletions(-)
diff --git a/repoman/pym/repoman/modules/scan/module.py
b/repoman/pym/repoman/modules/scan/module.py
new file mode 100644
index 000000000..f7f135f73
--- /dev/null
+++ b/repoman/pym/repoman/modules/scan/module.py
@@ -0,0 +1,84 @@
+
+'''
+moudules/scan/module.py
+Module loading and run list generator
+'''
+
+import logging
+import os
+import yaml
+
+from portage.module import InvalidModuleName, Modules
+from portage.util import stack_lists
+
+MODULES_PATH = os.path.dirname(__file__)
+# initial development debug info
+logging.debug("module path: %s", MODULES_PATH)
+
+
+class ModuleConfig(object):
+ '''Holds the scan modules configuration information and
+ creates the ordered list of modulles to run'''
+
+ def __init__(self, configpaths):
+ '''Module init
+
+ @param configpaths: ordered list of filepaths to load
+ '''
+ self.configpaths = configpaths
+
+ self.controller = Modules(path=MODULES_PATH,
namepath="repoman.modules.scan")
+ logging.debug("module_names: %s", self.controller.module_names)
+
+ self._configs = None
+ self.enabled = []
+ self.pkgs_loop = []
+ self.ebuilds_loop = []
+ self.final_loop = []
+ self.modules_forced = ['ebuild', 'mtime']
+ self.load_configs()
+ for loop in ['pkgs', 'ebuilds', 'final']:
+ logging.debug("ModuleConfig; Processing loop %s", loop)
+ setattr(self, '%s_loop' % loop,
self._determine_list(loop))
+
+
+ def load_configs(self, configpaths=None):
+ '''load the config files in order'''
+ if configpaths:
+ self.configpaths = configpaths
+ elif not self.configpaths:
+ logging.error("ModuleConfig; Error: No repository.yml
files defined")
+ configs = []
+ for path in self.configpaths:
+ logging.debug("ModuleConfig; Processing: %s", path)
+ if os.path.exists(path):
+ try:
+ with open(path, 'r') as inputfile:
+
configs.append(yaml.safe_load(inputfile.read()))
+ except IOError as error:
+ logging,error("Failed to load file:
%s", inputfile)
+ logging.exception(error)
+ logging.debug("ModuleConfig; completed : %s", path)
+ logging.debug("ModuleConfig; new _configs: %s", configs)
+ self._configs = configs
+
+ def _determine_list(self, loop):
+ '''Determine the ordered list from the config data and
+ the moule_runsIn value in the module_spec
+
+ @returns: list of modules
+ '''
+ lists = [c['modules'] for c in self._configs]
+ stacked = self.modules_forced + stack_lists(lists)
+ mlist = []
+ try:
+ for mod in stacked:
+ logging.debug("ModuleConfig; checking loop %s,
module: %s, in: %s",
+ loop, mod,
self.controller.get_spec(mod, 'module_runsIn'))
+ if loop in self.controller.get_spec(mod,
'module_runsIn'):
+ mlist.append(mod)
+ except InvalidModuleName:
+ logging.error("ModuleConfig; unkown module: %s,
skipping", mod)
+
+ logging.debug("ModuleConfig; mlist: %s", mlist)
+ return mlist
diff --git a/repoman/pym/repoman/scanner.py b/repoman/pym/repoman/scanner.py
index e13d4f68b..ffce701be 100644
--- a/repoman/pym/repoman/scanner.py
+++ b/repoman/pym/repoman/scanner.py
@@ -15,20 +15,10 @@ from repoman.modules.commit import repochecks
from repoman.modules.commit import manifest
from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
from repoman.repos import repo_metadata
+from repoman.modules.scan.module import ModuleConfig
from repoman.modules.scan.scan import scan
from repoman.modules.vcs.vcs import vcs_files_to_cps
-from portage.module import Modules
-
-MODULES_PATH = os.path.join(os.path.dirname(__file__), "modules", "scan")
-# initial development debug info
-logging.debug("module path: %s", MODULES_PATH)
-
-MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
-
-MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
-# initial development debug info
-logging.debug("module_names: %s", MODULE_NAMES)
DATA_TYPES = {'dict': dict, 'Future': ExtendedFuture, 'list': list, 'set': set}
@@ -124,6 +114,12 @@ class Scanner(object):
if self.vcs_settings.vcs is None:
self.options.echangelog = 'n'
+ # Initialize the ModuleConfig class here
+ # TODO Add layout.conf masters repository.yml config to the
list to load/stack
+ self.moduleconfig = ModuleConfig([
+
'/home/bdolbec/git/gentoo/metadata/repoman/repository.yml',
+ ])
+
checks = {}
# The --echangelog option causes automatic ChangeLog generation,
# which invalidates changelog.ebuildadded and changelog.missing
@@ -212,7 +208,7 @@ class Scanner(object):
@returns: dictionary
'''
kwargs = {}
- for key in MODULE_CONTROLLER.modules[mod]['mod_kwargs']:
+ for key in
self.moduleconfig.controller.modules[mod]['mod_kwargs']:
kwargs[key] = self.kwargs[key]
return kwargs
@@ -224,7 +220,7 @@ class Scanner(object):
@param dynamic_data: dictionary structure
@returns: dictionary
'''
- func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs']
+ func_kwargs =
self.moduleconfig.controller.modules[mod]['func_kwargs']
# determine new keys
required = set(list(func_kwargs))
exist = set(list(dynamic_data))
@@ -324,19 +320,16 @@ class Scanner(object):
'validity_future',
]
# need to set it up for ==> self.modules or some other
ordered list
- for mod in [('manifests', 'Manifests'), ('ebuild',
'Ebuild'),
- ('keywords', 'KeywordChecks'),
('files', 'FileChecks'),
- ('fetches', 'FetchChecks'),
- ('pkgmetadata', 'PkgMetadata'),
- ]:
- mod_class = MODULE_CONTROLLER.get_class(mod[0])
+ logging.debug("***** starting pkgs_loop: %s",
self.moduleconfig.pkgs_loop)
+ for mod in self.moduleconfig.pkgs_loop:
+ mod_class =
self.moduleconfig.controller.get_class(mod)
logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod_class.__name__] =
mod_class(**self.set_kwargs(mod[0]))
- logging.debug("scan_pkgs; module: %s", mod[1])
- do_it, functions =
self.modules[mod[1]].runInPkgs
+ self.modules[mod_class.__name__] =
mod_class(**self.set_kwargs(mod))
+ logging.debug("scan_pkgs; module: %s",
mod_class.__name__)
+ do_it, functions =
self.modules[mod_class.__name__].runInPkgs
if do_it:
for func in functions:
- _continue =
func(**self.set_func_kwargs(mod[0], dynamic_data))
+ _continue =
func(**self.set_func_kwargs(mod, dynamic_data))
if _continue:
# If we can't access
all the metadata then it's totally unsafe to
# commit since there's
no way to generate a correct Manifest.
@@ -373,28 +366,19 @@ class Scanner(object):
# initialize per ebuild plugin checks here
# need to set it up for ==> self.modules_list or some
other ordered list
- for mod in [('ebuild', 'Ebuild'), ('live',
'LiveEclassChecks'),
- ('eapi', 'EAPIChecks'), ('ebuild_metadata',
'EbuildMetadata'),
- ('fetches', 'FetchChecks'),
- ('description', 'DescriptionChecks'),
- ('keywords', 'KeywordChecks'),
- ('pkgmetadata', 'PkgMetadata'), ('ruby',
'RubyEclassChecks'),
- ('restrict', 'RestrictChecks'),
- ('mtime', 'MtimeChecks'), ('multicheck',
'MultiCheck'),
- # Options.is_forced() is used to bypass further
checks
- ('options', 'Options'), ('profile',
'ProfileDependsChecks'),
- ]:
- if mod[0] and mod[1] not in self.modules:
- mod_class =
MODULE_CONTROLLER.get_class(mod[0])
- logging.debug("Initializing class name:
%s", mod_class.__name__)
- self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod[0]))
- logging.debug("scan_ebuilds: module: %s",
mod[1])
- do_it, functions =
self.modules[mod[1]].runInEbuilds
+ for mod in self.moduleconfig.ebuilds_loop:
+ if mod:
+ mod_class =
self.moduleconfig.controller.get_class(mod)
+ if mod_class.__name__ not in
self.modules:
+ logging.debug("Initializing
class name: %s", mod_class.__name__)
+
self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
+ logging.debug("scan_ebuilds: module: %s",
mod_class.__name__)
+ do_it, functions =
self.modules[mod_class.__name__].runInEbuilds
logging.debug("do_it: %s, functions: %s",
do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning
function: %s", func)
- _continue =
func(**self.set_func_kwargs(mod[0], dynamic_data))
+ _continue =
func(**self.set_func_kwargs(mod, dynamic_data))
if _continue:
# If we can't access
all the metadata then it's totally unsafe to
# commit since there's
no way to generate a correct Manifest.
@@ -414,18 +398,19 @@ class Scanner(object):
# initialize per pkg plugin final checks here
# need to set it up for ==> self.modules_list or some other
ordered list
xpkg_complete = False
- for mod in [('pkgmetadata', 'PkgMetadata'), ('keywords',
'KeywordChecks')]:
- if mod[0] and mod[1] not in self.modules:
- mod_class = MODULE_CONTROLLER.get_class(mod[0])
- logging.debug("Initializing class name: %s",
mod_class.__name__)
- self.modules[mod[1]] =
mod_class(**self.set_kwargs(mod[0]))
- logging.debug("scan_ebuilds final checks: module: %s",
mod[1])
- do_it, functions = self.modules[mod[1]].runInFinal
+ for mod in self.moduleconfig.final_loop:
+ if mod:
+ mod_class =
self.moduleconfig.controller.get_class(mod)
+ if mod_class.__name__ not in self.modules:
+ logging.debug("Initializing class name:
%s", mod_class.__name__)
+ self.modules[mod_class.__name__] =
mod_class(**self.set_kwargs(mod))
+ logging.debug("scan_ebuilds final checks: module: %s",
mod_class.__name__)
+ do_it, functions =
self.modules[mod_class.__name__].runInFinal
logging.debug("do_it: %s, functions: %s", do_it,
[x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s",
func)
- _continue =
func(**self.set_func_kwargs(mod[0], dynamic_data))
+ _continue =
func(**self.set_func_kwargs(mod, dynamic_data))
if _continue:
xpkg_complete = True
# logging.debug("\t>>>
Continuing")
