Read dependencies from layer.conf and try to create the LayerDependency entry by looking up the correct database object. Dependencies are found by layer name only - no collection support. layer.conf parsing is handled by the bitbake code.
Signed-off-by: Liam R. Howlett <liam.howl...@windriver.com> --- layerindex/layerconfparse.py | 39 ++++++++++++++ layerindex/models.py | 4 ++ layerindex/recipeparse.py | 31 ++--------- layerindex/tools/import_layer.py | 8 +++ layerindex/update.py | 20 +++++++- layerindex/utils.py | 108 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 29 deletions(-) create mode 100644 layerindex/layerconfparse.py diff --git a/layerindex/layerconfparse.py b/layerindex/layerconfparse.py new file mode 100644 index 0000000..19b390c --- /dev/null +++ b/layerindex/layerconfparse.py @@ -0,0 +1,39 @@ +# Utility functions for parsing layer.conf using bitbake within layerindex-web +# +# Copyright (C) 2016 Wind River Systems +# Author: Liam R. Howlett <liam.howl...@windriver.com> +# +# Licensed under the MIT license, see COPYING.MIT for details +# + +import sys +import os +import os.path +import utils +import tempfile +import re + +class LayerConfParse: + def __init__(self, enable_tracking=False, logger=None, bitbakepath=None): + import settings + self.logger = logger + + if not bitbakepath: + fetchdir = settings.LAYER_FETCH_DIR + bitbakepath = os.path.join(fetchdir, 'bitbake') + self.bbpath = bitbakepath + + # Set up BBPATH. + os.environ['BBPATH'] = str("%s" % self.bbpath) + self.tinfoil = utils.setup_tinfoil(self.bbpath, enable_tracking) + self.config_data_copy = bb.data.createCopy(self.tinfoil.config_data) + + + def parse_layer(self, layerdir): + utils.parse_layer_conf(layerdir, self.config_data_copy) + return self.config_data_copy + + def shutdown(self): + self.tinfoil.shutdown() + + diff --git a/layerindex/models.py b/layerindex/models.py index 6aec030..2db8818 100644 --- a/layerindex/models.py +++ b/layerindex/models.py @@ -209,6 +209,9 @@ class LayerBranch(models.Model): return "%s: %s" % (self.layer.name, self.branch.name) + def get_required(self): + return self.dependencies_set.filter(required=True) + class LayerMaintainer(models.Model): MAINTAINER_STATUS_CHOICES = ( ('A', 'Active'), @@ -230,6 +233,7 @@ class LayerMaintainer(models.Model): class LayerDependency(models.Model): layerbranch = models.ForeignKey(LayerBranch, related_name='dependencies_set') dependency = models.ForeignKey(LayerItem, related_name='dependents_set') + required = models.BooleanField(default=True) class Meta: verbose_name_plural = "Layer dependencies" diff --git a/layerindex/recipeparse.py b/layerindex/recipeparse.py index 61d6fd4..8a63117 100644 --- a/layerindex/recipeparse.py +++ b/layerindex/recipeparse.py @@ -20,31 +20,6 @@ class RecipeParseError(Exception): def __str__(self): return self.msg -def _setup_tinfoil(bitbakepath, enable_tracking): - sys.path.insert(0, bitbakepath + '/lib') - import bb.tinfoil - import bb.cooker - import bb.data - try: - tinfoil = bb.tinfoil.Tinfoil(tracking=enable_tracking) - except TypeError: - # old API - tinfoil = bb.tinfoil.Tinfoil() - if enable_tracking: - tinfoil.cooker.enableDataTracking() - tinfoil.prepare(config_only = True) - - return tinfoil - -def _parse_layer_conf(layerdir, data): - data.setVar('LAYERDIR', str(layerdir)) - if hasattr(bb, "cookerdata"): - # Newer BitBake - data = bb.cookerdata.parse_config_file(os.path.join(layerdir, "conf", "layer.conf"), data) - else: - # Older BitBake (1.18 and below) - data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data) - data.expandVarref('LAYERDIR') def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False, classic=False, logger=None): @@ -97,7 +72,7 @@ def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR) os.chdir(tempdir) - tinfoil = _setup_tinfoil(bitbakepath, enable_tracking) + tinfoil = utils.setup_tinfoil(bitbakepath, enable_tracking) # Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log) oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True) @@ -125,7 +100,7 @@ def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch): # or across layers, but also because custom variable values might be # set in layer.conf. config_data_copy = bb.data.createCopy(config_data) - _parse_layer_conf(layerdir, config_data_copy) + utils.parse_layer_conf(layerdir, config_data_copy) for dep in layerbranch.dependencies_set.all(): depurldir = dep.dependency.get_fetch_dir() deprepodir = os.path.join(fetchdir, depurldir) @@ -133,7 +108,7 @@ def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch): if not deplayerbranch: raise RecipeParseError('Dependency %s of layer %s does not have branch record for branch %s' % (dep.dependency.name, layer.name, layerbranch.branch.name)) deplayerdir = os.path.join(deprepodir, deplayerbranch.vcs_subdir) - _parse_layer_conf(deplayerdir, config_data_copy) + utils.parse_layer_conf(deplayerdir, config_data_copy) config_data_copy.delVar('LAYERDIR') return config_data_copy diff --git a/layerindex/tools/import_layer.py b/layerindex/tools/import_layer.py index f84fd85..0a13d21 100755 --- a/layerindex/tools/import_layer.py +++ b/layerindex/tools/import_layer.py @@ -19,6 +19,7 @@ import glob import utils import logging import subprocess +from layerconfparse import LayerConfParse class DryRunRollbackException(Exception): pass @@ -375,11 +376,18 @@ def main(): if layer.name != settings.CORE_LAYER_NAME: if not core_layer: core_layer = utils.get_layer(settings.CORE_LAYER_NAME) + if core_layer: + logger.debug('Adding dep %s to %s' % (core_layer.name, layer.name)) layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() + layerconfparser = LayerConfParse(logger=logger) + config_data = layerconfparser.parse_layer(layerdir) + layerconfparser.shutdown() + utils.add_dependencies(layerbranch, config_data, logger=logger) + # Get some extra meta-information readme_files = glob.glob(os.path.join(layerdir, 'README*')) diff --git a/layerindex/update.py b/layerindex/update.py index 423eb53..ecd2380 100755 --- a/layerindex/update.py +++ b/layerindex/update.py @@ -16,6 +16,8 @@ import subprocess import signal from distutils.version import LooseVersion import utils +from layerconfparse import LayerConfParse + import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -92,7 +94,7 @@ def main(): utils.setup_django() import settings - from layerindex.models import Branch, LayerItem + from layerindex.models import Branch, LayerItem, LayerDependency logger.setLevel(options.loglevel) @@ -201,6 +203,22 @@ def main(): # Interrupted by user, break out of loop break + # Once indexed, then conf/layer.conf dependencies should be reevaluated. + layerconfparser = LayerConfParse(logger=logger, bitbakepath=bitbakepath) + for branch in branches: + for layer in layerquery: + urldir = layer.get_fetch_dir() + repodir = os.path.join(fetchdir, urldir) + + layerbranch = layer.get_layerbranch(branch) + if layerbranch.vcs_subdir: + repodir = os.path.join(repodir, layerbranch.vcs_subdir) + config_data = layerconfparser.parse_layer(repodir) + utils.add_dependencies(layerbranch, config_data, logger=logger) + + layerconfparser.shutdown() + + finally: utils.unlock_file(lockfile) diff --git a/layerindex/utils.py b/layerindex/utils.py index 23b81f5..f82f8c7 100644 --- a/layerindex/utils.py +++ b/layerindex/utils.py @@ -27,6 +27,114 @@ def get_layer(layername): return res[0] return None +def get_dependency_layer(depname, version_str=None, logger=None): + from layerindex.models import LayerItem, LayerBranch + + # Get any LayerBranch with a layer that has a name that matches the depname + res = list(LayerBranch.objects.filter(layer__name=depname)) + + # Nothing found, return. + if len(res) == 0: + return None + + # If there is no version constraint, return the first one found. + if not version_str: + return res[0].layer + + (operator, dep_version) = version_str.split() + for layerbranch in res: + layer_ver = layerbranch.version + + # If there is no version in the found layer, then don't use this layer. + if not layer_ver: + continue + + try: + success = bb.utils.vercmp_string_op(layer_ver, version_str, operator) + except bb.utils.VersionStringException as vse: + raise vse + + if success: + return layerbranch.layer + + return None + +def add_dependencies(layerbranch, config_data, logger=None): + _add_dependency("LAYERDEPENDS", 'dependency', layerbranch, config_data, logger) + +def _add_dependency(var, name, layerbranch, config_data, logger=None): + from layerindex.models import LayerBranch, LayerDependency + + layer_name = layerbranch.layer.name + var_name = layer_name + + dep_list = config_data.getVar("%s_%s" % (var, var_name), True) + + if not dep_list: + return + + try: + dep_dict = bb.utils.explode_dep_versions2(dep_list) + except bb.utils.VersionStringException as vse: + logger.debug('Error parsing %s_%s for %s\n%s' % (var, var_name, layer_name, str(vse))) + return + + for dep, ver_list in list(dep_dict.items()): + ver_str = None + if ver_list: + ver_str = ver_list[0] + + try: + dep_layer = get_dependency_layer(dep, ver_str, logger) + except bb.utils.VersionStringException as vse: + if logger: + logger.error('Error getting %s %s for %s\n%s' %(name, dep. layer_name, str(vse))) + continue + + if not dep_layer: + if logger: + logger.error('Cannot resolve %s %s (version %s) for %s' % (name, dep, ver_str, layer_name)) + continue + + # Skip existing entries. + existing = list(LayerDependency.objects.filter(layerbranch=layerbranch).filter(dependency=dep_layer)) + if existing: + logger.debug('Skipping %s - already a dependency for %s' % (dep, layer_name)) + continue + + if logger: + logger.debug('Adding %s %s to %s' % (name, dep_layer.name, layer_name)) + layerdep = LayerDependency() + layerdep.layerbranch = layerbranch + layerdep.dependency = dep_layer + layerdep.save() + +def setup_tinfoil(bitbakepath, enable_tracking): + sys.path.insert(0, bitbakepath + '/lib') + import bb.tinfoil + import bb.cooker + import bb.data + try: + tinfoil = bb.tinfoil.Tinfoil(tracking=enable_tracking) + except TypeError: + # old API + tinfoil = bb.tinfoil.Tinfoil() + if enable_tracking: + tinfoil.cooker.enableDataTracking() + tinfoil.prepare(config_only = True) + + return tinfoil + +def parse_layer_conf(layerdir, data): + data.setVar('LAYERDIR', str(layerdir)) + if hasattr(bb, "cookerdata"): + # Newer BitBake + data = bb.cookerdata.parse_config_file(os.path.join(layerdir, "conf", "layer.conf"), data) + else: + # Older BitBake (1.18 and below) + data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data) + data.expandVarref('LAYERDIR') + def runcmd(cmd, destdir=None, printerr=True, logger=None): """ execute command, raise CalledProcessError if fail -- 1.9.1 -- _______________________________________________ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto