[OE-core] [PATCH] kernel.bbclass: Fix race condition
Race condition between do_compile_kernelmodules and do_shared_workdir
tasks occurs when do_compilemodules changes files in include/generated/*
while do_shared_workdir tries to copy them to shared working directory.
Functionality of do_shared_workdir has been moved to separate function
mk_shared_workdir and called from do_kernel_compile. do_shared_workdir
left in the code as it may be used in many recipes.
[YOCTO #7321]
Signed-off-by: Ed Bartosh
---
meta/classes/kernel.bbclass | 12 +++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 125ed88..aad430a 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -231,6 +231,8 @@ kernel_do_install() {
[ -e Module.symvers ] && install -m 0644 Module.symvers
${D}/boot/Module.symvers-${KERNEL_VERSION}
install -d ${D}${sysconfdir}/modules-load.d
install -d ${D}${sysconfdir}/modprobe.d
+
+ mk_shared_workdir
}
do_install[prefuncs] += "package_get_auto_pr"
@@ -245,7 +247,7 @@ emit_depmod_pkgdata() {
PACKAGEFUNCS += "emit_depmod_pkgdata"
-do_shared_workdir () {
+mk_shared_workdir () {
cd ${B}
kerneldir=${STAGING_KERNEL_BUILDDIR}
@@ -289,6 +291,14 @@ do_shared_workdir () {
fi
}
+# NOTE!!! Functionality of do_shared_workdir has been moved to
mk_shared_workdir
+# and called from kernel_do_compile.
+# It caused race condition with do_compile_kernelmodules when it runs
+# in parallel with do_compile_kernelmodules
+do_shared_workdir () {
+ :
+}
+
# We don't need to stage anything, not the modules/firmware since those would
clash with linux-firmware
sysroot_stage_all () {
:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: Consider PSEUDO_* environment variables
Used default values of PSEUDO_* environment variables only
if variables are not set.
This allows to set custom PSEUDO_PREFIX and other pseudo
variables in order to use pseudo database from non-standard
locations.
Change-Id: I0bc1af5e521121d1f96d590cb6edb23cf0cb0b83
Signed-off-by: Ed Bartosh
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/kickstart/custom_commands/partition.py | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/scripts/lib/wic/kickstart/custom_commands/partition.py
b/scripts/lib/wic/kickstart/custom_commands/partition.py
index 4f5a1e5..f3d553b 100644
--- a/scripts/lib/wic/kickstart/custom_commands/partition.py
+++ b/scripts/lib/wic/kickstart/custom_commands/partition.py
@@ -193,10 +193,15 @@ class Wic_PartData(Mic_PartData):
Currently handles ext2/3/4, btrfs and vfat.
"""
-pseudo = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot
-pseudo += "export PSEUDO_LOCALSTATEDIR=%s/../pseudo;" % rootfs_dir
-pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir
-pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
+p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot)
+p_localstatedir = os.environ.get("PSEUDO_LOCALSTATEDIR",
+ "%s/../pseudo" % rootfs_dir)
+p_passwd = os.environ.get("PSEUDO_PASSWD", rootfs_dir)
+p_nosymlinkexp = os.environ.get("PSEUDO_NOSYMLINKEXP", "1")
+pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
+pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % p_localstatedir
+pseudo += "export PSEUDO_PASSWD=%s;" % p_passwd
+pseudo += "export PSEUDO_NOSYMLINKEXP=%s;" % p_nosymlinkexp
pseudo += "%s/usr/bin/pseudo " % native_sysroot
if self.fstype.startswith("ext"):
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [bitbake][PATCH] bitbake: bin/bitbake: Create bitbake_main API
Moved most of functionality of bin/bitbake to lib/bb/main.py
to be able to call bitbake from python code.
Signed-off-by: Ed Bartosh
---
bitbake/bin/bitbake| 362 +
bitbake/lib/bb/main.py | 391 +
2 files changed, 396 insertions(+), 357 deletions(-)
create mode 100755 bitbake/lib/bb/main.py
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 49efdf1..cc21ce5 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -23,373 +23,21 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
-import sys, logging
+import sys
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),
'lib'))
-
-import optparse
-import warnings
-from traceback import format_exception
try:
import bb
except RuntimeError as exc:
sys.exit(str(exc))
-from bb import event
-import bb.msg
-from bb import cooker
-from bb import ui
-from bb import server
-from bb import cookerdata
-
-__version__ = "1.25.0"
-logger = logging.getLogger("BitBake")
-
-# Python multiprocessing requires /dev/shm on Linux
-if sys.platform.startswith('linux') and not os.access('/dev/shm', os.W_OK |
os.X_OK):
-sys.exit("FATAL: /dev/shm does not exist or is not writable")
-
-# Unbuffer stdout to avoid log truncation in the event
-# of an unorderly exit as well as to provide timely
-# updates to log files for use with tail
-try:
-if sys.stdout.name == '':
-sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
-except:
-pass
-
-
-def get_ui(config):
-if not config.ui:
-# modify 'ui' attribute because it is also read by cooker
-config.ui = os.environ.get('BITBAKE_UI', 'knotty')
-
-interface = config.ui
-
-try:
-# Dynamically load the UI based on the ui name. Although we
-# suggest a fixed set this allows you to have flexibility in which
-# ones are available.
-module = __import__("bb.ui", fromlist = [interface])
-return getattr(module, interface)
-except AttributeError:
-sys.exit("FATAL: Invalid user interface '%s' specified.\n"
- "Valid interfaces: depexp, goggle, ncurses, hob, knotty
[default]." % interface)
-
-
-# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring
others"""
-warnlog = logging.getLogger("BitBake.Warnings")
-_warnings_showwarning = warnings.showwarning
-def _showwarning(message, category, filename, lineno, file=None, line=None):
-if file is not None:
-if _warnings_showwarning is not None:
-_warnings_showwarning(message, category, filename, lineno, file,
line)
-else:
-s = warnings.formatwarning(message, category, filename, lineno)
-warnlog.warn(s)
-
-warnings.showwarning = _showwarning
-warnings.filterwarnings("ignore")
-warnings.filterwarnings("default", module="($|(oe|bb)\.)")
-warnings.filterwarnings("ignore", category=PendingDeprecationWarning)
-warnings.filterwarnings("ignore", category=ImportWarning)
-warnings.filterwarnings("ignore", category=DeprecationWarning,
module="$")
-warnings.filterwarnings("ignore", message="With-statements now directly
support multiple context managers")
-
-class BitBakeConfigParameters(cookerdata.ConfigParameters):
-
-def parseCommandLine(self):
-parser = optparse.OptionParser(
-version = "BitBake Build Tool Core version %s, %%prog version %s"
% (bb.__version__, __version__),
-usage = """%prog [options] [recipename/target ...]
-
-Executes the specified task (default is 'build') for a given set of target
recipes (.bb files).
-It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH
which
-will provide the layer, BBFILES and other configuration information.""")
-
-parser.add_option("-b", "--buildfile", help = "Execute tasks from a
specific .bb recipe directly. WARNING: Does not handle any dependencies from
other recipes.",
- action = "store", dest = "buildfile", default = None)
-
-parser.add_option("-k", "--continue", help = "Continue as much as
possible after an error. While the target that failed and anything depending on
it cannot be built, as much as possible will be built before stopping.",
- action = "store_false", dest = "abort", default = True)
-
-parser.add_option("-a", "--tryaltconfigs", help = "Continue with
builds by trying to use alternative providers where possible
[OE-core] [PATCH] lib/oe/lsb.py: Stop parsing /etc/SuSE-release
/etc/SuSE-release should not be parsed to obtain release info
Here is the quote from /etc/SuSE-release:
/etc/SuSE-release is deprecated and will be removed in the future,
use /etc/os-release instead
All recent OpenSUSE versions already have /etc/os-release and
oe.lsb/release_dict_file function already parses it.
Current implementation of release_dict_file returns different values
when parsing /etc/SuSE-release and /etc/os-release. This is one more
advantage of getting rid of parsing /etc/SuSE-release.
Signed-off-by: Ed Bartosh
---
meta/lib/oe/lsb.py | 8
1 file changed, 8 deletions(-)
diff --git a/meta/lib/oe/lsb.py b/meta/lib/oe/lsb.py
index 50c1d47..e25e282 100644
--- a/meta/lib/oe/lsb.py
+++ b/meta/lib/oe/lsb.py
@@ -37,14 +37,6 @@ def release_dict_file():
if match:
data['DISTRIB_ID'] = match.group(1)
data['DISTRIB_RELEASE'] = match.group(2)
-elif os.path.exists('/etc/SuSE-release'):
-data = {}
-data['DISTRIB_ID'] = 'SUSE LINUX'
-with open('/etc/SuSE-release') as f:
-for line in f:
-if line.startswith('VERSION = '):
-data['DISTRIB_RELEASE'] = line[10:].rstrip()
-break
elif os.path.exists('/etc/os-release'):
data = {}
with open('/etc/os-release') as f:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] lib/oe/lsb.py: Stop parsing /etc/SuSE-release
On Fri, Mar 20, 2015 at 11:38:25AM +, Paul Eggleton wrote: > Hi Ed, > > On Friday 20 March 2015 13:22:50 Ed Bartosh wrote: > > /etc/SuSE-release should not be parsed to obtain release info > > > > Here is the quote from /etc/SuSE-release: > > /etc/SuSE-release is deprecated and will be removed in the future, > > use /etc/os-release instead > > > > All recent OpenSUSE versions already have /etc/os-release and > > oe.lsb/release_dict_file function already parses it. > > > > Current implementation of release_dict_file returns different values > > when parsing /etc/SuSE-release and /etc/os-release. This is one more > > advantage of getting rid of parsing /etc/SuSE-release. > > How long ago did OpenSuSE start including /etc/os-release? My concern is if > someone did happen to run on an older OpenSuSE release and then submit a bug > report with an attached log, we'd rather see the actual version rather than > "Unknown". If that's a possible scenario, we might want to just swap the > blocks around so that /etc/os-release is looked for first. > I've checked opensuse versions starting from 12.3, which was released 2 years ago and became OOL in Jan 2015. If you think it's not enough I can check 12.2 and 12.1 or swap the blocks as you've suggested. Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] linux-yocto-tiny: Enable CONFIG_BINFMT_SCRIPT
This SRCREV update fixes the failure of running /init
for poky-tiny image.
Signed-off-by: Ed Bartosh
---
meta/recipes-kernel/linux/linux-yocto-tiny_3.14.bb | 2 +-
meta/recipes-kernel/linux/linux-yocto-tiny_3.19.bb | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-kernel/linux/linux-yocto-tiny_3.14.bb
b/meta/recipes-kernel/linux/linux-yocto-tiny_3.14.bb
index f6b95a2..fd0fa26 100644
--- a/meta/recipes-kernel/linux/linux-yocto-tiny_3.14.bb
+++ b/meta/recipes-kernel/linux/linux-yocto-tiny_3.14.bb
@@ -10,7 +10,7 @@ KMETA = "meta"
KCONF_BSP_AUDIT_LEVEL = "2"
SRCREV_machine ?= "3bab81113682d3a5ffc8ea60cf770beed4831492"
-SRCREV_meta ?= "6eddbf47875ef48ddc5864957a7b63363100782b"
+SRCREV_meta ?= "4c54da246369a9da73f8b4571ea7c9e61faa4d69"
PV = "${LINUX_VERSION}+git${SRCPV}"
diff --git a/meta/recipes-kernel/linux/linux-yocto-tiny_3.19.bb
b/meta/recipes-kernel/linux/linux-yocto-tiny_3.19.bb
index d10449a..c05af4a 100644
--- a/meta/recipes-kernel/linux/linux-yocto-tiny_3.19.bb
+++ b/meta/recipes-kernel/linux/linux-yocto-tiny_3.19.bb
@@ -10,7 +10,7 @@ KMETA = "meta"
KCONF_BSP_AUDIT_LEVEL = "2"
SRCREV_machine ?= "43b9eced9ba8a57add36af07736344dcc383f711"
-SRCREV_meta ?= "f7a81274b461d0c748964d75c24bea8574d350fb"
+SRCREV_meta ?= "76bc151242d7f97f7eebc7591f83801cf07dc9af"
PV = "${LINUX_VERSION}+git${SRCPV}"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] lib/oe/lsb.py: Prioritize parsing of /etc/os-release
Moved parsing of /etc/os-release before parsing of
/etc/SuSE-release as /etc/SuSE-release is deprecated in
recent releases of OpenSuSE.
Here is the quote from /etc/SuSE-release:
/etc/SuSE-release is deprecated and will be removed in the future,
use /etc/os-release instead
Signed-off-by: Ed Bartosh
---
meta/lib/oe/lsb.py | 17 +
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/meta/lib/oe/lsb.py b/meta/lib/oe/lsb.py
index 50c1d47..ddfe71b 100644
--- a/meta/lib/oe/lsb.py
+++ b/meta/lib/oe/lsb.py
@@ -37,14 +37,6 @@ def release_dict_file():
if match:
data['DISTRIB_ID'] = match.group(1)
data['DISTRIB_RELEASE'] = match.group(2)
-elif os.path.exists('/etc/SuSE-release'):
-data = {}
-data['DISTRIB_ID'] = 'SUSE LINUX'
-with open('/etc/SuSE-release') as f:
-for line in f:
-if line.startswith('VERSION = '):
-data['DISTRIB_RELEASE'] = line[10:].rstrip()
-break
elif os.path.exists('/etc/os-release'):
data = {}
with open('/etc/os-release') as f:
@@ -53,6 +45,15 @@ def release_dict_file():
data['DISTRIB_ID'] = line[5:].rstrip().strip('"')
if line.startswith('VERSION_ID='):
data['DISTRIB_RELEASE'] = line[11:].rstrip().strip('"')
+elif os.path.exists('/etc/SuSE-release'):
+data = {}
+data['DISTRIB_ID'] = 'SUSE LINUX'
+with open('/etc/SuSE-release') as f:
+for line in f:
+if line.startswith('VERSION = '):
+data['DISTRIB_RELEASE'] = line[10:].rstrip()
+break
+
except IOError:
return None
return data
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] rootfs.py: Remove rpm database from staging area
Rpm database in staging area is used only by createrepo.
createrepo fails with the error
"rpmdb: BDB0060 PANIC: fatal region error detected"
if rpm database is broken from the previous run of createrepo.
Removing the databae before running createrepo can hopefully
prevent this failure to happen.
[YOCTO #6571]
Signed-off-by: Ed Bartosh
---
meta/lib/oe/rootfs.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 4e4e6eb..9f7dc65 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -306,6 +306,9 @@ class RpmRootfs(Rootfs):
bb.utils.remove(self.image_rootfs, True)
else:
self.pm.recovery_packaging_data()
+dbpath = os.path.join(self.d.getVar('STAGING_DIR_NATIVE', True),
+ 'var/lib/rpm/*')
+bb.utils.remove(dbpath, recurse=True)
bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
self.pm.create_configs()
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] package_manager: Run createrepo sequentially
Running multiple createrepo in parallel can cause raice conditions
when creating/accessing rpm database. Createrepo fails with the
error "package_manager: rpmdb: BDB0060 PANIC: fatal region error detected"
in such cases.
Running createrepo sequentially should fix the race as rpm database
will not be accessed by multiple createrepo at the same time.
[YOCTO #6571]
Signed-off-by: Ed Bartosh
---
meta/lib/oe/package_manager.py | 11 ---
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index c9a8084..c956116 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -108,25 +108,22 @@ class RpmIndexer(Indexer):
archs = archs.union(set(sdk_pkg_archs))
rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
-index_cmds = []
rpm_dirs_found = False
for arch in archs:
arch_dir = os.path.join(self.deploy_dir, arch)
if not os.path.isdir(arch_dir):
continue
-index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir))
-
rpm_dirs_found = True
+result = create_index("%s --update -q %s" % (rpm_createrepo,
arch_dir))
+if result:
+bb.fatal(result)
+
if not rpm_dirs_found:
bb.note("There are no packages in %s" % self.deploy_dir)
return
-result = oe.utils.multiprocess_exec(index_cmds, create_index)
-if result:
-bb.fatal('%s' % ('\n'.join(result)))
-
class OpkgIndexer(Indexer):
def write_index(self):
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] package_manager: Run createrepo sequentially
On Mon, Mar 30, 2015 at 10:11:13AM +0100, Richard Purdie wrote:
> On Mon, 2015-03-30 at 12:00 +0300, Ed Bartosh wrote:
> > Running multiple createrepo in parallel can cause raice conditions
> > when creating/accessing rpm database. Createrepo fails with the
> > error "package_manager: rpmdb: BDB0060 PANIC: fatal region error detected"
> > in such cases.
> >
> > Running createrepo sequentially should fix the race as rpm database
> > will not be accessed by multiple createrepo at the same time.
> >
> > [YOCTO #6571]
> >
> > Signed-off-by: Ed Bartosh
> > ---
> > meta/lib/oe/package_manager.py | 11 ---
> > 1 file changed, 4 insertions(+), 7 deletions(-)
> >
> > diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> > index c9a8084..c956116 100644
> > --- a/meta/lib/oe/package_manager.py
> > +++ b/meta/lib/oe/package_manager.py
> > @@ -108,25 +108,22 @@ class RpmIndexer(Indexer):
> > archs = archs.union(set(sdk_pkg_archs))
> >
> > rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
> > -index_cmds = []
> > rpm_dirs_found = False
> > for arch in archs:
> > arch_dir = os.path.join(self.deploy_dir, arch)
> > if not os.path.isdir(arch_dir):
> > continue
> >
> > -index_cmds.append("%s --update -q %s" % (rpm_createrepo,
> > arch_dir))
> > -
> > rpm_dirs_found = True
> >
> > +result = create_index("%s --update -q %s" % (rpm_createrepo,
> > arch_dir))
> > +if result:
> > +bb.fatal(result)
> > +
> > if not rpm_dirs_found:
> > bb.note("There are no packages in %s" % self.deploy_dir)
> > return
> >
> > -result = oe.utils.multiprocess_exec(index_cmds, create_index)
> > -if result:
> > -bb.fatal('%s' % ('\n'.join(result)))
> > -
> >
> > class OpkgIndexer(Indexer):
> > def write_index(self):
>
> Running these in parallel gives us a significant speed improvement.
> Could we instead point them at separate directories for the RPM database
> (within WORKDIR to address the previous issue too)?
>
I didn't see any noticeable performance gain at least for minimal image. I
measured the time of
bitbake build before and after this change. It was practically the same.
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] rootfs.py: Remove rpm database from staging area
On Mon, Mar 30, 2015 at 10:14:36AM -0500, Mark Hatle wrote:
> On 3/30/15 3:53 AM, Richard Purdie wrote:
> > On Mon, 2015-03-30 at 11:30 +0300, Ed Bartosh wrote:
> >> Rpm database in staging area is used only by createrepo.
> >> createrepo fails with the error
> >> "rpmdb: BDB0060 PANIC: fatal region error detected"
> >> if rpm database is broken from the previous run of createrepo.
> >>
> >> Removing the databae before running createrepo can hopefully
> >> prevent this failure to happen.
> >>
> >> [YOCTO #6571]
> >>
> >> Signed-off-by: Ed Bartosh
> >> ---
> >> meta/lib/oe/rootfs.py | 3 +++
> >> 1 file changed, 3 insertions(+)
> >>
> >> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> >> index 4e4e6eb..9f7dc65 100644
> >> --- a/meta/lib/oe/rootfs.py
> >> +++ b/meta/lib/oe/rootfs.py
> >> @@ -306,6 +306,9 @@ class RpmRootfs(Rootfs):
> >> bb.utils.remove(self.image_rootfs, True)
> >> else:
> >> self.pm.recovery_packaging_data()
> >> +dbpath = os.path.join(self.d.getVar('STAGING_DIR_NATIVE', True),
> >> + 'var/lib/rpm/*')
> >> +bb.utils.remove(dbpath, recurse=True)
> >> bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
> >>
> >> self.pm.crea
> >
> > This patch helps me see the problem a lot. I'd never realised there was
> > an rpm database in the native sysroot that was getting corrupted, I'd
> > always assumed it was the target rootfs one.
> >
> > I'm a little worried about what happens if you have multiple images
> > generating at the same time as this change as above may delete something
> > being worked on by another process.
> >
> > I'm wondering why is it getting into the sysroot at all? Rather than
> > delete it here, could we either not generate it at all, or place it
> > somewhere in WORKDIR (so that other tasks can't see it or race against
> > it)?
>
> I did some quick looking. It appears (at least on first glance) that the
> call to:
>
> rpm.TransactionSet()
>
> is what is opening the DB. It appears to me that we can change the call
> slightly, and it should do what is being requested:
>
> { "TransactionSet", (PyCFunction) rpmts_Create,
> METH_VARARGS|METH_KEYWORDS,
> "rpm.TransactionSet([rootDir, [db]]) -> ts\n\
> - Create a transaction set.\n" },
>
> So transaction set takes two arguments, the rootDir of the thing we're working
> against, and a DB path. It -should- be as simply as setting a rootDir to the
> WORKDIR. If that doesn't work, then both arguments will be needed.
>
> I'd suggest just adding a "--root" option to genpkgmetadata.py in the
> createrepo, and maybe a "--dbpath" option as well (second argument). Then
> only
> call the function with their values IF they were defined. I.e.
>
> if root:
> if dbpath:
> self.ts = rpm.TransactionSet(root, dbpath)
> else:
> self.ts = rpm.TransactionSet(root)
> else:
> self.ts = rpm.TransactionSet()
>
> (If there is a better way to do that in python, fine.. but passing in "None"
> or
> "" won't result in the desired behavior.. since RPM is actually parsing
> arguments and appears like it will use the values if passed in, whatever they
> are.)
>
I was going to do something like this:
if dbpath:
rpm.addMacro("_dbpath", dbpath)
self.ts = rpm.TransactionSet()
Brief test showed that this approach works just fine.
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] rootfs.py: Remove rpm database from staging area
On Mon, Mar 30, 2015 at 10:14:36AM -0500, Mark Hatle wrote:
> On 3/30/15 3:53 AM, Richard Purdie wrote:
> > On Mon, 2015-03-30 at 11:30 +0300, Ed Bartosh wrote:
> >> Rpm database in staging area is used only by createrepo.
> >> createrepo fails with the error
> >> "rpmdb: BDB0060 PANIC: fatal region error detected"
> >> if rpm database is broken from the previous run of createrepo.
> >>
> >> Removing the databae before running createrepo can hopefully
> >> prevent this failure to happen.
> >>
> >> [YOCTO #6571]
> >>
> >> Signed-off-by: Ed Bartosh
> >> ---
> >> meta/lib/oe/rootfs.py | 3 +++
> >> 1 file changed, 3 insertions(+)
> >>
> >> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> >> index 4e4e6eb..9f7dc65 100644
> >> --- a/meta/lib/oe/rootfs.py
> >> +++ b/meta/lib/oe/rootfs.py
> >> @@ -306,6 +306,9 @@ class RpmRootfs(Rootfs):
> >> bb.utils.remove(self.image_rootfs, True)
> >> else:
> >> self.pm.recovery_packaging_data()
> >> +dbpath = os.path.join(self.d.getVar('STAGING_DIR_NATIVE', True),
> >> + 'var/lib/rpm/*')
> >> +bb.utils.remove(dbpath, recurse=True)
> >> bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
> >>
> >> self.pm.crea
> >
> > This patch helps me see the problem a lot. I'd never realised there was
> > an rpm database in the native sysroot that was getting corrupted, I'd
> > always assumed it was the target rootfs one.
> >
> > I'm a little worried about what happens if you have multiple images
> > generating at the same time as this change as above may delete something
> > being worked on by another process.
> >
> > I'm wondering why is it getting into the sysroot at all? Rather than
> > delete it here, could we either not generate it at all, or place it
> > somewhere in WORKDIR (so that other tasks can't see it or race against
> > it)?
>
> I did some quick looking. It appears (at least on first glance) that the
> call to:
>
> rpm.TransactionSet()
>
> is what is opening the DB. It appears to me that we can change the call
> slightly, and it should do what is being requested:
>
> { "TransactionSet", (PyCFunction) rpmts_Create,
> METH_VARARGS|METH_KEYWORDS,
> "rpm.TransactionSet([rootDir, [db]]) -> ts\n\
> - Create a transaction set.\n" },
>
> So transaction set takes two arguments, the rootDir of the thing we're working
> against, and a DB path. It -should- be as simply as setting a rootDir to the
> WORKDIR. If that doesn't work, then both arguments will be needed.
>
> I'd suggest just adding a "--root" option to genpkgmetadata.py in the
> createrepo, and maybe a "--dbpath" option as well (second argument). Then
> only
> call the function with their values IF they were defined. I.e.
>
> if root:
> if dbpath:
> self.ts = rpm.TransactionSet(root, dbpath)
> else:
> self.ts = rpm.TransactionSet(root)
> else:
> self.ts = rpm.TransactionSet()
>
> (If there is a better way to do that in python, fine.. but passing in "None"
> or
> "" won't result in the desired behavior.. since RPM is actually parsing
> arguments and appears like it will use the values if passed in, whatever they
> are.)
>
Second parameter is not a path. Integer is required there, so I guess it's a db
mode something like that.
Defining _dbpath macro looks better from my point of view as it allows to
specify full path to the db directory.
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] createrepo: Implement --dbpath command line option
--dbpath option allows to specify path to the directory
with rpm database. By default createrepo uses or creates
rpm database in /var/lib/rpm/.
Upstream-Status: Pending
Signed-off-by: Ed Bartosh
---
.../createrepo/createrepo/createrepo-dbpath.patch | 51 ++
.../createrepo/createrepo_0.4.11.bb| 1 +
2 files changed, 52 insertions(+)
create mode 100644
meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
diff --git a/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
new file mode 100644
index 000..2b113bf
--- /dev/null
+++ b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
@@ -0,0 +1,51 @@
+--- createrepo-0.4.11.orig/genpkgmetadata.py 2015-03-30 22:18:19.90400
+0300
createrepo-0.4.11.orig/genpkgmetadata.py 2015-03-30 22:28:49.20800
+0300
+@@ -65,6 +65,7 @@
+ -p, --pretty = output xml files in pretty format.
+ --update = update existing metadata (if present)
+ -d, --database = generate the sqlite databases.
++ --dbpath = specify path to rpm db directory.
+ """)
+
+ sys.exit(retval)
+@@ -72,10 +73,13 @@
+ class MetaDataGenerator:
+ def __init__(self, cmds):
+ self.cmds = cmds
+-self.ts = rpm.TransactionSet()
+ self.pkgcount = 0
+ self.files = []
+
++if self.cmds['dbpath']:
++rpm.addMacro("_dbpath", self.cmds['dbpath'])
++self.ts = rpm.TransactionSet()
++
+ def _os_path_walk(self, top, func, arg):
+ """Directory tree walk with callback function.
+ copy of os.path.walk, fixes the link/stating problem
+@@ -435,6 +439,7 @@
+ cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
+ cmds['skip-symlinks'] = False
+ cmds['pkglist'] = []
++cmds['dbpath'] = None
+
+ try:
+ gopts, argsleft = getopt.getopt(args, 'phqVvndg:s:x:u:c:o:CSi:',
['help', 'exclude=',
+@@ -442,7 +447,7 @@
+ 'baseurl=',
'groupfile=', 'checksum=',
+ 'version',
'pretty', 'split', 'outputdir=',
+ 'noepoch',
'checkts', 'database', 'update',
+-
'skip-symlinks', 'pkglist='])
++
'skip-symlinks', 'pkglist=', 'dbpath='])
+ except getopt.error, e:
+ errorprint(_('Options Error: %s.') % e)
+ usage()
+@@ -516,6 +521,8 @@
+ cmds['skip-symlinks'] = True
+ elif arg in ['-i', '--pkglist']:
+ cmds['pkglist'] = a
++elif arg == '--dbpath':
++cmds['dbpath'] = os.path.realpath(a)
+
+ except ValueError, e:
+ errorprint(_('Options Error: %s') % e)
diff --git a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
index 49b45fc..adc193e 100644
--- a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
+++ b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
@@ -14,6 +14,7 @@ SRC_URI= "http://createrepo.baseurl.org/download/${BP}.tar.gz
\
file://python-scripts-should-use-interpreter-from-env.patch \
file://createrepo-rpm549.patch \
file://recommends.patch \
+ file://createrepo-dbpath.patch \
file://rpm-createsolvedb.py \
"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] package_manager: call createrepo with --dbpath pointing inside WORKDIR
Rpm database in staging area is used only by createrepo.
createrepo fails with the error
"rpmdb: BDB0060 PANIC: fatal region error detected"
if rpm database is broken during previous run of createrepo.
Made createrepo to create rpm db in $WORKDIR/rpmdb/ from scratch
for every build. This should potentially fix the failure.
[YOCTO #6571]
Signed-off-by: Ed Bartosh
---
meta/lib/oe/package_manager.py | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 743c7cb..2461acd 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -110,12 +110,16 @@ class RpmIndexer(Indexer):
rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
index_cmds = []
rpm_dirs_found = False
+dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb')
+if os.path.exists(dbpath):
+bb.utils.remove(dbpath, True)
for arch in archs:
arch_dir = os.path.join(self.deploy_dir, arch)
if not os.path.isdir(arch_dir):
continue
-index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir))
+index_cmds.append("%s --dbpath %s --update -q %s" % \
+ (rpm_createrepo, dbpath, arch_dir))
rpm_dirs_found = True
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] package_manager: call createrepo with --dbpath pointing inside WORKDIR
Rpm database in staging area is used only by createrepo.
createrepo fails with the error
"rpmdb: BDB0060 PANIC: fatal region error detected"
if rpm database is broken during previous run of createrepo.
Made createrepo to create rpm db in $WORKDIR/rpmdb/ from scratch
for every build and architecture. This should potentially fix the
failure as every run of createrepo will be using separate db.
[YOCTO #6571]
Signed-off-by: Ed Bartosh
---
meta/lib/oe/package_manager.py | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 743c7cb..ed0c9a7 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -111,11 +111,15 @@ class RpmIndexer(Indexer):
index_cmds = []
rpm_dirs_found = False
for arch in archs:
+dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb',
arch)
+if os.path.exists(dbpath):
+bb.utils.remove(dbpath, True)
arch_dir = os.path.join(self.deploy_dir, arch)
if not os.path.isdir(arch_dir):
continue
-index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir))
+index_cmds.append("%s --dbpath %s --update -q %s" % \
+ (rpm_createrepo, dbpath, arch_dir))
rpm_dirs_found = True
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] createrepo: Implement --dbpath command line option
On Tue, Mar 31, 2015 at 10:25:11PM +0100, Burton, Ross wrote: > Hi Ed, > > On 31 March 2015 at 00:49, Ed Bartosh wrote: > > > Upstream-Status: Pending > > > > The commit log needs just a Signed-off-by; the patch needs an explanation, > Signed-off-by, and Upstream-Status. The goal is that the patch itself is > self-explanatory, so should explain what it does and why, whether it went > upstream and why (not), and who admits creating it in the first place. > Thanks for the explanations. I'll update the patch. Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] createrepo: Implement --dbpath command line option
--dbpath option can be used in cases where users don't want
createrepo to use system rpm db to avoid possible collisiouns
with other programs.
For bitbake builds it would be possible to specify different
databases even for every createrepo run. Considering that rootfs
builds can run multiple createrepo in parallel, it can help to avoid
race conditions caused by accessing or creating the same rpm database
by multiple createrepo instances at the same time.
Signed-off-by: Ed Bartosh
---
.../createrepo/createrepo/createrepo-dbpath.patch | 51 ++
.../createrepo/createrepo_0.4.11.bb| 1 +
2 files changed, 52 insertions(+)
create mode 100644
meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
diff --git a/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
new file mode 100644
index 000..2b113bf
--- /dev/null
+++ b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
@@ -0,0 +1,51 @@
+--- createrepo-0.4.11.orig/genpkgmetadata.py 2015-03-30 22:18:19.90400
+0300
createrepo-0.4.11.orig/genpkgmetadata.py 2015-03-30 22:28:49.20800
+0300
+@@ -65,6 +65,7 @@
+ -p, --pretty = output xml files in pretty format.
+ --update = update existing metadata (if present)
+ -d, --database = generate the sqlite databases.
++ --dbpath = specify path to rpm db directory.
+ """)
+
+ sys.exit(retval)
+@@ -72,10 +73,13 @@
+ class MetaDataGenerator:
+ def __init__(self, cmds):
+ self.cmds = cmds
+-self.ts = rpm.TransactionSet()
+ self.pkgcount = 0
+ self.files = []
+
++if self.cmds['dbpath']:
++rpm.addMacro("_dbpath", self.cmds['dbpath'])
++self.ts = rpm.TransactionSet()
++
+ def _os_path_walk(self, top, func, arg):
+ """Directory tree walk with callback function.
+ copy of os.path.walk, fixes the link/stating problem
+@@ -435,6 +439,7 @@
+ cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
+ cmds['skip-symlinks'] = False
+ cmds['pkglist'] = []
++cmds['dbpath'] = None
+
+ try:
+ gopts, argsleft = getopt.getopt(args, 'phqVvndg:s:x:u:c:o:CSi:',
['help', 'exclude=',
+@@ -442,7 +447,7 @@
+ 'baseurl=',
'groupfile=', 'checksum=',
+ 'version',
'pretty', 'split', 'outputdir=',
+ 'noepoch',
'checkts', 'database', 'update',
+-
'skip-symlinks', 'pkglist='])
++
'skip-symlinks', 'pkglist=', 'dbpath='])
+ except getopt.error, e:
+ errorprint(_('Options Error: %s.') % e)
+ usage()
+@@ -516,6 +521,8 @@
+ cmds['skip-symlinks'] = True
+ elif arg in ['-i', '--pkglist']:
+ cmds['pkglist'] = a
++elif arg == '--dbpath':
++cmds['dbpath'] = os.path.realpath(a)
+
+ except ValueError, e:
+ errorprint(_('Options Error: %s') % e)
diff --git a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
index 49b45fc..adc193e 100644
--- a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
+++ b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
@@ -14,6 +14,7 @@ SRC_URI= "http://createrepo.baseurl.org/download/${BP}.tar.gz
\
file://python-scripts-should-use-interpreter-from-env.patch \
file://createrepo-rpm549.patch \
file://recommends.patch \
+ file://createrepo-dbpath.patch \
file://rpm-createsolvedb.py \
"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] createrepo: Implement --dbpath command line option
--dbpath option can be used in cases where users don't want
createrepo to use system rpm db to avoid possible collisiouns
with other programs.
For bitbake builds it would be possible to specify different
databases even for every createrepo run. Considering that rootfs
builds can run multiple createrepo in parallel, it can help to avoid
race conditions caused by accessing or creating the same rpm database
by multiple createrepo instances at the same time.
Signed-off-by: Ed Bartosh
---
.../createrepo/createrepo/createrepo-dbpath.patch | 61 ++
.../createrepo/createrepo_0.4.11.bb| 1 +
2 files changed, 62 insertions(+)
create mode 100644
meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
diff --git a/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
new file mode 100644
index 000..7275598
--- /dev/null
+++ b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
@@ -0,0 +1,61 @@
+createrepo: Implement --dbpath command line option
+
+--dbpath option allows to specify path to the directory
+with rpm database. By default createrepo uses or creates
+rpm database in /var/lib/rpm/
+
+Upstream-Status: Pending
+
+Signed-off-by: Ed Bartosh
+
+--- createrepo-0.4.11.orig/genpkgmetadata.py 2015-03-30 22:18:19.90400
+0300
createrepo-0.4.11/genpkgmetadata.py 2015-03-30 22:28:49.20800 +0300
+@@ -65,6 +65,7 @@
+ -p, --pretty = output xml files in pretty format.
+ --update = update existing metadata (if present)
+ -d, --database = generate the sqlite databases.
++ --dbpath = specify path to rpm db directory.
+ """)
+
+ sys.exit(retval)
+@@ -72,10 +73,13 @@
+ class MetaDataGenerator:
+ def __init__(self, cmds):
+ self.cmds = cmds
+-self.ts = rpm.TransactionSet()
+ self.pkgcount = 0
+ self.files = []
+
++if self.cmds['dbpath']:
++rpm.addMacro("_dbpath", self.cmds['dbpath'])
++self.ts = rpm.TransactionSet()
++
+ def _os_path_walk(self, top, func, arg):
+ """Directory tree walk with callback function.
+ copy of os.path.walk, fixes the link/stating problem
+@@ -435,6 +439,7 @@
+ cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
+ cmds['skip-symlinks'] = False
+ cmds['pkglist'] = []
++cmds['dbpath'] = None
+
+ try:
+ gopts, argsleft = getopt.getopt(args, 'phqVvndg:s:x:u:c:o:CSi:',
['help', 'exclude=',
+@@ -442,7 +447,7 @@
+ 'baseurl=',
'groupfile=', 'checksum=',
+ 'version',
'pretty', 'split', 'outputdir=',
+ 'noepoch',
'checkts', 'database', 'update',
+-
'skip-symlinks', 'pkglist='])
++
'skip-symlinks', 'pkglist=', 'dbpath='])
+ except getopt.error, e:
+ errorprint(_('Options Error: %s.') % e)
+ usage()
+@@ -516,6 +521,8 @@
+ cmds['skip-symlinks'] = True
+ elif arg in ['-i', '--pkglist']:
+ cmds['pkglist'] = a
++elif arg == '--dbpath':
++cmds['dbpath'] = os.path.realpath(a)
+
+ except ValueError, e:
+ errorprint(_('Options Error: %s') % e)
diff --git a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
index 49b45fc..adc193e 100644
--- a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
+++ b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
@@ -14,6 +14,7 @@ SRC_URI= "http://createrepo.baseurl.org/download/${BP}.tar.gz
\
file://python-scripts-should-use-interpreter-from-env.patch \
file://createrepo-rpm549.patch \
file://recommends.patch \
+ file://createrepo-dbpath.patch \
file://rpm-createsolvedb.py \
"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] createrepo: Implement --dbpath command line option
On Wed, Apr 01, 2015 at 03:06:52PM +0300, Ed Bartosh wrote:
> --dbpath option can be used in cases where users don't want
> createrepo to use system rpm db to avoid possible collisiouns
> with other programs.
>
> For bitbake builds it would be possible to specify different
> databases even for every createrepo run. Considering that rootfs
> builds can run multiple createrepo in parallel, it can help to avoid
> race conditions caused by accessing or creating the same rpm database
> by multiple createrepo instances at the same time.
>
> Signed-off-by: Ed Bartosh
> ---
> .../createrepo/createrepo/createrepo-dbpath.patch | 51
> ++
> .../createrepo/createrepo_0.4.11.bb| 1 +
> 2 files changed, 52 insertions(+)
> create mode 100644
> meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
>
> diff --git
> a/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
> b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
> new file mode 100644
> index 000..2b113bf
> --- /dev/null
> +++ b/meta/recipes-support/createrepo/createrepo/createrepo-dbpath.patch
> @@ -0,0 +1,51 @@
> +--- createrepo-0.4.11.orig/genpkgmetadata.py 2015-03-30 22:18:19.90400
> +0300
> createrepo-0.4.11.orig/genpkgmetadata.py 2015-03-30 22:28:49.20800
> +0300
> +@@ -65,6 +65,7 @@
> + -p, --pretty = output xml files in pretty format.
> + --update = update existing metadata (if present)
> + -d, --database = generate the sqlite databases.
> ++ --dbpath = specify path to rpm db directory.
> + """)
> +
> + sys.exit(retval)
> +@@ -72,10 +73,13 @@
> + class MetaDataGenerator:
> + def __init__(self, cmds):
> + self.cmds = cmds
> +-self.ts = rpm.TransactionSet()
> + self.pkgcount = 0
> + self.files = []
> +
> ++if self.cmds['dbpath']:
> ++rpm.addMacro("_dbpath", self.cmds['dbpath'])
> ++self.ts = rpm.TransactionSet()
> ++
> + def _os_path_walk(self, top, func, arg):
> + """Directory tree walk with callback function.
> + copy of os.path.walk, fixes the link/stating problem
> +@@ -435,6 +439,7 @@
> + cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
> + cmds['skip-symlinks'] = False
> + cmds['pkglist'] = []
> ++cmds['dbpath'] = None
> +
> + try:
> + gopts, argsleft = getopt.getopt(args, 'phqVvndg:s:x:u:c:o:CSi:',
> ['help', 'exclude=',
> +@@ -442,7 +447,7 @@
> +
> 'baseurl=', 'groupfile=', 'checksum=',
> +
> 'version', 'pretty', 'split', 'outputdir=',
> +
> 'noepoch', 'checkts', 'database', 'update',
> +-
> 'skip-symlinks', 'pkglist='])
> ++
> 'skip-symlinks', 'pkglist=', 'dbpath='])
> + except getopt.error, e:
> + errorprint(_('Options Error: %s.') % e)
> + usage()
> +@@ -516,6 +521,8 @@
> + cmds['skip-symlinks'] = True
> + elif arg in ['-i', '--pkglist']:
> + cmds['pkglist'] = a
> ++elif arg == '--dbpath':
> ++cmds['dbpath'] = os.path.realpath(a)
> +
> + except ValueError, e:
> + errorprint(_('Options Error: %s') % e)
> diff --git a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
> b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
> index 49b45fc..adc193e 100644
> --- a/meta/recipes-support/createrepo/createrepo_0.4.11.bb
> +++ b/meta/recipes-support/createrepo/createrepo_0.4.11.bb
> @@ -14,6 +14,7 @@ SRC_URI=
> "http://createrepo.baseurl.org/download/${BP}.tar.gz \
>file://python-scripts-should-use-interpreter-from-env.patch \
> file://createrepo-rpm549.patch \
> file://recommends.patch \
> + file://createrepo-dbpath.patch \
> file://rpm-createsolvedb.py \
> "
>
Please, don't apply this patch. I've just sent updated one. Sorry for the
confusion.
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: extended list of paths in find_binary_path
wic requires tools that are not always possible to find in $PATH.
This causes wic to fail with confusing errors like this:
External command 'parted' not found, exiting.
(Please install 'parted' on your host system)
Adding ~/bin/, /usr/local/sbin, /usr/local/bin, /usr/sbin, /usr/bin,
/sbin and /bin to the list of paths makes find_binary_path to
produce more reliable results.
[YOCTO #7122]
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/utils/fs_related.py | 13 -
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/scripts/lib/wic/utils/fs_related.py
b/scripts/lib/wic/utils/fs_related.py
index ea9f85c..832a44a 100644
--- a/scripts/lib/wic/utils/fs_related.py
+++ b/scripts/lib/wic/utils/fs_related.py
@@ -32,13 +32,16 @@ from wic.utils.errors import *
from wic.utils.oe.misc import *
def find_binary_path(binary):
+paths = []
if os.environ.has_key("PATH"):
paths = os.environ["PATH"].split(":")
-else:
-paths = []
-if os.environ.has_key("HOME"):
-paths += [os.environ["HOME"] + "/bin"]
-paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin",
"/usr/bin", "/sbin", "/bin"]
+if os.environ.has_key("HOME"):
+path = os.path.join(os.environ["HOME"], "bin")
+if path not in paths:
+paths.append(path)
+for path in ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin",
"/sbin", "/bin"]:
+if path not in paths:
+paths.append(path)
for path in paths:
bin_path = "%s/%s" % (path, binary)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: Print bitbake error output
Added bitbake error output to wic error message to make it more descriptive. [YOCTO #6911] Signed-off-by: Ed Bartosh --- scripts/lib/wic/utils/oe/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py index ea9b6e8..4872bd0 100644 --- a/scripts/lib/wic/utils/oe/misc.py +++ b/scripts/lib/wic/utils/oe/misc.py @@ -144,6 +144,7 @@ def find_bitbake_env_lines(image_name): rc, bitbake_env_lines = __exec_cmd(bitbake_env_cmd) if rc != 0: print "Couldn't get '%s' output." % bitbake_env_cmd +print "Bitbake failed with error:\n%s\n" % bitbake_env_lines return None return bitbake_env_lines -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [wic][PATCH] wic: extended list of paths in find_binary_path
On Sat, Apr 04, 2015 at 03:34:59PM -0300, Otavio Salvador wrote: > Em 04/04/2015 14:21, "Ed Bartosh" escreveu: > > > > wic requires tools that are not always possible to find in $PATH. > > This causes wic to fail with confusing errors like this: > > External command 'parted' not found, exiting. > >(Please install 'parted' on your host system) > > > > Adding ~/bin/, /usr/local/sbin, /usr/local/bin, /usr/sbin, /usr/bin, > > /sbin and /bin to the list of paths makes find_binary_path to > > produce more reliable results. > > > > [YOCTO #7122] > > > > Signed-off-by: Ed Bartosh > > I think we shouldn't do that. If user wants something to be found the PATH > environment variable should have them included. > Some users think that this is confusing: https://bugzilla.yoctoproject.org/show_bug.cgi?id=7122 For current implementation of wic it practically means that it must be always run as "PATH=/usr/sbin:$PATH wic" on OpenSUSE systems. > This also brings one serious error off wic which is it relying on host > utilities. It should use parted from native sysroot instead otherwise we > cannot be sure if witch version we will be using. This makes sense to me. Not sure how easy is to do it as I suspect that not only parted is used this way. BTW, this approach conflicts with this feature request: https://bugzilla.yoctoproject.org/show_bug.cgi?id=6558 So, should wic rely on system utilities or not? If it should not then what would you suggest to do with #6558? -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [wic][PATCH] wic: extended list of paths in find_binary_path
On Mon, Apr 06, 2015 at 09:00:50AM -0300, Otavio Salvador wrote: > On Sun, Apr 5, 2015 at 4:16 PM, Philip Balister wrote: > > On 04/04/2015 10:20 AM, Ed Bartosh wrote: > >> wic requires tools that are not always possible to find in $PATH. > >> This causes wic to fail with confusing errors like this: > >> External command 'parted' not found, exiting. > >>(Please install 'parted' on your host system) > >> > >> Adding ~/bin/, /usr/local/sbin, /usr/local/bin, /usr/sbin, /usr/bin, > >> /sbin and /bin to the list of paths makes find_binary_path to > >> produce more reliable results. > > > > I'm with Otavio. Given wic is intimately dependent on internal build > > artifacts, it should use utilities from the sysroot and not depend on > > tools from the path. Let's make wic 100% before creating a tool to build > > images from packages. > > If we want to have it as a 'standalone' tool we will need to somehow > package the tool set for it to use, otherwise we have a maintenance > nightmare (does patch X has been applied to Parted?). > Can you elaborate a bit on how tool set can be packaged? Back to the original topic. wic never used native parted as it's not useable(there is no executable in systoor). Anybody knows the reason? Documentation also suggests to install parted on the host: http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#wic-requirements -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: use native parted
Used exec_native_cmd instead of find_binary_path to run parted.
Got rid of find_binary_path as it's not used anywhere else.
There are several tools wic is trying to find not only in sysroot,
but also in host root. Parted is a special as on some distros it's
installed in /usr/sbin, which is not in the user's PATH. This makes
wic to fail with error "External command 'parted' not found, exiting."
[YOCTO #7122]
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/imager/direct.py | 2 +-
scripts/lib/wic/utils/fs_related.py| 18 --
scripts/lib/wic/utils/partitionedfs.py | 11 ++-
3 files changed, 7 insertions(+), 24 deletions(-)
diff --git a/scripts/lib/wic/imager/direct.py b/scripts/lib/wic/imager/direct.py
index d368401..c3d0563 100644
--- a/scripts/lib/wic/imager/direct.py
+++ b/scripts/lib/wic/imager/direct.py
@@ -242,7 +242,7 @@ class DirectImageCreator(BaseImageCreator):
"""
parts = self._get_parts()
-self.__image = Image()
+self.__image = Image(self.native_sysroot)
for p in parts:
# as a convenience, set source to the boot partition source
diff --git a/scripts/lib/wic/utils/fs_related.py
b/scripts/lib/wic/utils/fs_related.py
index ea9f85c..033acc9 100644
--- a/scripts/lib/wic/utils/fs_related.py
+++ b/scripts/lib/wic/utils/fs_related.py
@@ -31,24 +31,6 @@ from wic.utils import runner
from wic.utils.errors import *
from wic.utils.oe.misc import *
-def find_binary_path(binary):
-if os.environ.has_key("PATH"):
-paths = os.environ["PATH"].split(":")
-else:
-paths = []
-if os.environ.has_key("HOME"):
-paths += [os.environ["HOME"] + "/bin"]
-paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin",
"/usr/bin", "/sbin", "/bin"]
-
-for path in paths:
-bin_path = "%s/%s" % (path, binary)
-if os.path.exists(bin_path):
-return bin_path
-
-print "External command '%s' not found, exiting." % binary
-print " (Please install '%s' on your host system)" % binary
-sys.exit(1)
-
def makedirs(dirname):
"""A version of os.makedirs() that doesn't throw an
exception if the leaf directory already exists.
diff --git a/scripts/lib/wic/utils/partitionedfs.py
b/scripts/lib/wic/utils/partitionedfs.py
index 162f8e1..1c10cb5 100644
--- a/scripts/lib/wic/utils/partitionedfs.py
+++ b/scripts/lib/wic/utils/partitionedfs.py
@@ -42,13 +42,13 @@ class Image:
An Image is a container for a set of DiskImages and associated
partitions.
"""
-def __init__(self):
+def __init__(self, native_sysroot=None):
self.disks = {}
self.partitions = []
-self.parted = find_binary_path("parted")
# Size of a sector used in calculations
self.sector_size = SECTOR_SIZE
self._partitions_layed_out = False
+self.native_sysroot = native_sysroot
def __add_disk(self, disk_name):
""" Add a disk 'disk_name' to the internal list of disks. Note,
@@ -227,11 +227,12 @@ class Image:
def __run_parted(self, args):
""" Run parted with arguments specified in the 'args' list. """
-args.insert(0, self.parted)
+args.insert(0, "parted")
+args = ' '.join(args)
msger.debug(args)
-rc, out = runner.runtool(args, catch = 3)
-out = out.strip()
+rc, out = exec_native_cmd(args, self.native_sysroot)
+
if out:
msger.debug('"parted" output: %s' % out)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: don't use host paths when looking for native tools
exec_native_cmd should not use host paths in $PATH to
avoid finding and using host tools.
[YOCTO: #6204]
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/utils/oe/misc.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py
index ea9b6e8..5facbe0 100644
--- a/scripts/lib/wic/utils/oe/misc.py
+++ b/scripts/lib/wic/utils/oe/misc.py
@@ -81,7 +81,7 @@ def exec_native_cmd(cmd_and_args, native_sysroot, catch = 3):
Always need to execute native commands as_shell
"""
native_paths = \
-"export PATH=%s/sbin:%s/usr/sbin:%s/usr/bin:$PATH" % \
+"export PATH=%s/sbin:%s/usr/sbin:%s/usr/bin" % \
(native_sysroot, native_sysroot, native_sysroot)
native_cmd_and_args = "%s;%s" % (native_paths, cmd_and_args)
msger.debug("exec_native_cmd: %s" % cmd_and_args)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: make error message more informative
Included name of the missing program into the error message produced
by exec_native_cmd when program is not found.
Removed mentioning of the host programs as this API is not running
them anymore.
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/utils/oe/misc.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py
index 5facbe0..7769f3f 100644
--- a/scripts/lib/wic/utils/oe/misc.py
+++ b/scripts/lib/wic/utils/oe/misc.py
@@ -92,9 +92,9 @@ def exec_native_cmd(cmd_and_args, native_sysroot, catch = 3):
rc, out = __exec_cmd(native_cmd_and_args, True, catch)
if rc == 127: # shell command-not-found
-msger.error("A native (host) program required to build the image "
+msger.error("A native program %s required to build the image "
"was not found (see details above). Please make sure "
-"it's installed and try again.")
+"it's installed and try again." % args[0])
return (rc, out)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: Implement --build-rootfs command line option
-f/--build-rootfs option makes wic to run bitbake to
produce rootfs. This option requires image name to be specified
with -e/--image-name.
Signed-off-by: Ed Bartosh
---
scripts/lib/image/help.py | 10 ++
scripts/wic | 25 ++---
2 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/scripts/lib/image/help.py b/scripts/lib/image/help.py
index e1eb265..e365a07 100644
--- a/scripts/lib/image/help.py
+++ b/scripts/lib/image/help.py
@@ -111,7 +111,7 @@ wic_create_usage = """
[-i | --infile ]
[-e | --image-name] [-s, --skip-build-check] [-D, --debug]
[-r, --rootfs-dir] [-b, --bootimg-dir]
-[-k, --kernel-dir] [-n, --native-sysroot]
+[-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs]
This command creates an OpenEmbedded image based on the 'OE kickstart
commands' found in the .
@@ -132,7 +132,7 @@ SYNOPSIS
[-i | --infile ]
[-e | --image-name] [-s, --skip-build-check] [-D, --debug]
[-r, --rootfs-dir] [-b, --bootimg-dir]
-[-k, --kernel-dir] [-n, --native-sysroot]
+[-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs]
DESCRIPTION
This command creates an OpenEmbedded image based on the 'OE
@@ -167,6 +167,8 @@ DESCRIPTION
The -n option is used to specify the path to the native sysroot
containing the tools to use to build the image.
+The -f option is used to build rootfs by running "bitbake "
+
The -s option is used to skip the build check. The build check is
a simple sanity check used to determine whether the user has
sourced the build environment so that the -e option can operate
@@ -528,8 +530,8 @@ DESCRIPTION
usage: wic create [-o | ...]
[-i | --infile ]
[-e | --image-name] [-s, --skip-build-check] [-D, --debug]
-[-r, --rootfs-dir] [-b, --bootimg-dir]
-[-k, --kernel-dir] [-n, --native-sysroot]
+[-r, --rootfs-dir] [-b, --bootimg-dir] [-k, --kernel-dir]
+[-n, --native-sysroot] [-f, --build-rootfs]
This command creates an OpenEmbedded image based on the 'OE
kickstart commands' found in the .
diff --git a/scripts/wic b/scripts/wic
index e7df60f..feff302 100755
--- a/scripts/wic
+++ b/scripts/wic
@@ -40,11 +40,15 @@ import logging
# External modules
scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
lib_path = scripts_path + '/lib'
-sys.path = sys.path + [lib_path]
+bitbake_path = os.path.join(scripts_path, '../bitbake/lib')
+sys.path = sys.path + [lib_path, bitbake_path]
from image.help import *
from image.engine import *
+from bb import cookerdata
+from bb.main import bitbake_main, BitBakeConfigParameters
+
def rootfs_dir_to_args(krootfs_dir):
"""
Get a rootfs_dir dict and serialize to string
@@ -94,6 +98,7 @@ def wic_create_subcommand(args, usage_str):
action = "store", help = "path to the native sysroot
containing the tools to use to build the image")
parser.add_option("-p", "--skip-build-check", dest = "build_check",
action = "store_false", default = True, help = "skip the
build check")
+parser.add_option("-f", "--build-rootfs", action="store_true", help =
"build rootfs")
parser.add_option("-D", "--debug", dest = "debug", action = "store_true",
default = False, help = "output debug information")
@@ -123,8 +128,6 @@ def wic_create_subcommand(args, usage_str):
else:
print "Done.\n"
-print "Creating image(s)...\n"
-
bitbake_env_lines = find_bitbake_env_lines(options.image_name)
if not bitbake_env_lines:
print "Couldn't get bitbake environment, exiting."
@@ -134,9 +137,24 @@ def wic_create_subcommand(args, usage_str):
bootimg_dir = ""
if options.image_name:
+if options.build_rootfs:
+argv = ["bitbake", options.image_name]
+if options.debug:
+argv.append("--debug")
+
+print "Building rootfs...\n"
+if bitbake_main(BitBakeConfigParameters(argv),
+cookerdata.CookerConfiguration()):
+sys.exit(1)
+
(rootfs_dir, kernel_dir, bootimg_dir, native_sysroot) \
= find_artifacts(options.image_name)
+else:
+if options.build_rootfs:
+print "Image name is not specified, exiting. (Use -e/--image-name
to specify it)\n"
+sys.exit(1)
+
wks_file = args[0]
if not wks_file.endswith(".wks"):
@@ -194,6 +2
Re: [OE-core] [wic][PATCH 1/2] wic: rawcopy: ensure comparison is done on int
On Tue, Apr 07, 2015 at 12:21:58PM +0200, Alexandre Belloni wrote: > The size comparison may fail because it is done on strings rather than > integers. > > Signed-off-by: Alexandre Belloni > --- > scripts/lib/wic/plugins/source/rawcopy.py | 2 +- > 1 file changed, 1 insertion(+), 1 deletion(-) > > diff --git a/scripts/lib/wic/plugins/source/rawcopy.py > b/scripts/lib/wic/plugins/source/rawcopy.py > index cf6236a04f91..ba7fda787ed7 100644 > --- a/scripts/lib/wic/plugins/source/rawcopy.py > +++ b/scripts/lib/wic/plugins/source/rawcopy.py > @@ -77,7 +77,7 @@ class RawCopyPlugin(SourcePlugin): > out = exec_cmd(du_cmd) > filesize = out.split()[0] > > -if filesize > part.size: > +if int(filesize) > int(part.size): > part.size = filesize > > part.source_file = dst Acked-by: Ed Bartosh -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [wic][PATCH 2/2] wic: support ontrackdm6aux3 partitions
On Tue, Apr 07, 2015 at 12:21:59PM +0200, Alexandre Belloni wrote: > ontrack dm6 aux3 is a type of partition required to be able to boot a > Freescale > mxs based platform. > > Signed-off-by: Alexandre Belloni > --- > scripts/lib/wic/utils/partitionedfs.py | 2 ++ > 1 file changed, 2 insertions(+) > > diff --git a/scripts/lib/wic/utils/partitionedfs.py > b/scripts/lib/wic/utils/partitionedfs.py > index 162f8e1b9c6a..f394abc764c7 100644 > --- a/scripts/lib/wic/utils/partitionedfs.py > +++ b/scripts/lib/wic/utils/partitionedfs.py > @@ -296,6 +296,8 @@ class Image: > parted_fs_type = "fat32" > elif p['fstype'] == "msdos": > parted_fs_type = "fat16" > +elif p['fstype'] == "ontrackdm6aux3": > +parted_fs_type = "ontrackdm6aux3" > else: > # Type for ext2/ext3/ext4/btrfs > parted_fs_type = "ext2" Acked-by: Ed Bartosh -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [wic][PATCH] wic: extended list of paths in find_binary_path
On Mon, Apr 06, 2015 at 11:28:24AM -0300, Otavio Salvador wrote: > >> > >> If we want to have it as a 'standalone' tool we will need to somehow > >> package the tool set for it to use, otherwise we have a maintenance > >> nightmare (does patch X has been applied to Parted?). > >> > > Can you elaborate a bit on how tool set can be packaged? > > We could do a tarball like the buildtools toolchain. I was thinking about providing wic and its dependencies as repository with rpm/deb packages. Providing them as a tarball doesn't differ much from what we already have from my point of view. -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: code cleanup: bad indentation
Fixed pylint warning 'Bad indentation'
Signed-off-by: Ed Bartosh
---
scripts/wic | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/scripts/wic b/scripts/wic
index feff302..524156d 100755
--- a/scripts/wic
+++ b/scripts/wic
@@ -69,8 +69,8 @@ def callback_rootfs_dir(option, opt, value, parser):
if '=' in value:
(key, rootfs_dir) = value.split('=')
else:
- key = 'ROOTFS_DIR'
- rootfs_dir = value
+key = 'ROOTFS_DIR'
+rootfs_dir = value
parser.values.rootfs_dir[key] = rootfs_dir
@@ -207,8 +207,8 @@ def wic_create_subcommand(args, usage_str):
krootfs_dir = options.rootfs_dir
if krootfs_dir is None:
- krootfs_dir = {}
- krootfs_dir['ROOTFS_DIR'] = rootfs_dir
+krootfs_dir = {}
+krootfs_dir['ROOTFS_DIR'] = rootfs_dir
rootfs_dir = rootfs_dir_to_args(krootfs_dir)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: code cleanup: unused imports
Fixed pylint warning 'Unused import' Signed-off-by: Ed Bartosh --- scripts/lib/wic/conf.py| 3 +-- scripts/lib/wic/creator.py | 2 +- scripts/lib/wic/imager/baseimager.py | 2 +- scripts/lib/wic/imager/direct.py | 1 - scripts/lib/wic/kickstart/custom_commands/partition.py | 1 - scripts/lib/wic/pluginbase.py | 2 -- scripts/lib/wic/plugins/imager/direct_plugin.py| 5 - scripts/lib/wic/plugins/source/bootimg-efi.py | 2 -- scripts/lib/wic/plugins/source/bootimg-pcbios.py | 3 --- scripts/lib/wic/plugins/source/fsimage.py | 1 - scripts/lib/wic/plugins/source/rawcopy.py | 1 - scripts/lib/wic/plugins/source/rootfs.py | 3 --- scripts/lib/wic/utils/fs_related.py| 6 -- scripts/lib/wic/utils/misc.py | 1 - scripts/lib/wic/utils/partitionedfs.py | 2 -- 15 files changed, 3 insertions(+), 32 deletions(-) diff --git a/scripts/lib/wic/conf.py b/scripts/lib/wic/conf.py index be34355..ff6e2ca 100644 --- a/scripts/lib/wic/conf.py +++ b/scripts/lib/wic/conf.py @@ -15,8 +15,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os, sys, re -import ConfigParser +import os from wic import msger from wic import kickstart diff --git a/scripts/lib/wic/creator.py b/scripts/lib/wic/creator.py index 2219377..760848f 100644 --- a/scripts/lib/wic/creator.py +++ b/scripts/lib/wic/creator.py @@ -15,7 +15,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os, sys, re +import os, sys from optparse import SUPPRESS_HELP from wic import msger diff --git a/scripts/lib/wic/imager/baseimager.py b/scripts/lib/wic/imager/baseimager.py index e830527..2075358 100644 --- a/scripts/lib/wic/imager/baseimager.py +++ b/scripts/lib/wic/imager/baseimager.py @@ -17,7 +17,7 @@ # Temple Place - Suite 330, Boston, MA 02111-1307, USA. from __future__ import with_statement -import os, sys +import os import tempfile import shutil diff --git a/scripts/lib/wic/imager/direct.py b/scripts/lib/wic/imager/direct.py index c3d0563..c7e1c6c 100644 --- a/scripts/lib/wic/imager/direct.py +++ b/scripts/lib/wic/imager/direct.py @@ -25,7 +25,6 @@ # import os -import stat import shutil from wic import kickstart, msger diff --git a/scripts/lib/wic/kickstart/custom_commands/partition.py b/scripts/lib/wic/kickstart/custom_commands/partition.py index f3d553b..ff90601 100644 --- a/scripts/lib/wic/kickstart/custom_commands/partition.py +++ b/scripts/lib/wic/kickstart/custom_commands/partition.py @@ -24,7 +24,6 @@ # Tom Zanussi # -import shutil import os import tempfile diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py index e3de9ba..72bcad2 100644 --- a/scripts/lib/wic/pluginbase.py +++ b/scripts/lib/wic/pluginbase.py @@ -15,8 +15,6 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os -import shutil from wic import msger from wic.utils import errors diff --git a/scripts/lib/wic/plugins/imager/direct_plugin.py b/scripts/lib/wic/plugins/imager/direct_plugin.py index 5601c3f..17888e9 100644 --- a/scripts/lib/wic/plugins/imager/direct_plugin.py +++ b/scripts/lib/wic/plugins/imager/direct_plugin.py @@ -24,11 +24,6 @@ # Tom Zanussi # -import os -import shutil -import re -import tempfile - from wic import msger from wic.utils import misc, fs_related, errors, runner, cmdln from wic.conf import configmgr diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py index ee57881..d6c11ed 100644 --- a/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -26,8 +26,6 @@ import os import shutil -import re -import tempfile from wic import kickstart, msger from wic.utils import misc, fs_related, errors, runner, cmdln diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py index c4786a6..abb32eb 100644 --- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -25,9 +25,6 @@ # import os -import shutil -import re -import tempfile from wic import kickstart, msger from wic.utils import misc, fs_related, errors, runner, cmdln diff --git a/scripts/lib/wic/plugins/source/fsimage.py b/scripts/lib/wic/plugins/source/fsimage.py index 0967883..4a090ba 100644 --- a/scripts/lib/wic/plugins/source/fsimage.py +++ b/scripts/lib/wic/plugins/source/fsimage.py @@ -16,7 +16,6 @@ # import os -import re from wic import msger from wic.pluginbase import SourceP
[OE-core] [wic][PATCH] wic: Use __file__ instead of sys.argv[0]
Using __file__ makes the code work independently of the way it's loaded. In some cases wic can be imported by another program without executing it. sys.argv[0] would not contain path to the wic in such a cases. This is an enabler for unit testing with nose framework. Signed-off-by: Ed Bartosh --- scripts/wic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wic b/scripts/wic index dcf1a55..fd4a678 100755 --- a/scripts/wic +++ b/scripts/wic @@ -38,7 +38,7 @@ import optparse import logging # External modules -scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +scripts_path = os.path.abspath(os.path.dirname(__file__)) lib_path = scripts_path + '/lib' bitbake_path = os.path.join(scripts_path, '../bitbake/lib') sys.path = sys.path + [lib_path, bitbake_path] -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic:code cleanup: No space allowed
Fixed pylint warning
'No space allowed around keyword argument assignment'
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/imager/baseimager.py | 8 ++---
scripts/lib/wic/imager/direct.py | 10 +++---
scripts/lib/wic/kickstart/__init__.py | 14
scripts/lib/wic/msger.py | 10 +++---
scripts/lib/wic/utils/fs_related.py| 2 +-
scripts/lib/wic/utils/misc.py | 2 +-
scripts/lib/wic/utils/oe/misc.py | 8 ++---
scripts/lib/wic/utils/partitionedfs.py | 8 ++---
scripts/wic| 58 +-
9 files changed, 60 insertions(+), 60 deletions(-)
diff --git a/scripts/lib/wic/imager/baseimager.py
b/scripts/lib/wic/imager/baseimager.py
index 2075358..0afbc2c 100644
--- a/scripts/lib/wic/imager/baseimager.py
+++ b/scripts/lib/wic/imager/baseimager.py
@@ -42,7 +42,7 @@ class BaseImageCreator(object):
def __del__(self):
self.cleanup()
-def __init__(self, createopts = None):
+def __init__(self, createopts=None):
"""Initialize an ImageCreator instance.
ks -- a pykickstart.KickstartParser instance; this instance will be
@@ -131,8 +131,8 @@ class BaseImageCreator(object):
self.workdir = os.path.join(self.tmpdir, "build")
if not os.path.exists(self.workdir):
os.makedirs(self.workdir)
-self.__builddir = tempfile.mkdtemp(dir = self.workdir,
- prefix = "imgcreate-")
+self.__builddir = tempfile.mkdtemp(dir=self.workdir,
+ prefix="imgcreate-")
except OSError, (err, msg):
raise CreatorError("Failed create build directory in %s: %s" %
(self.tmpdir, msg))
@@ -178,7 +178,7 @@ class BaseImageCreator(object):
self._cleanup()
-shutil.rmtree(self.__builddir, ignore_errors = True)
+shutil.rmtree(self.__builddir, ignore_errors=True)
self.__builddir = None
self.__clean_tmpdir()
diff --git a/scripts/lib/wic/imager/direct.py b/scripts/lib/wic/imager/direct.py
index c7e1c6c..0e687bd 100644
--- a/scripts/lib/wic/imager/direct.py
+++ b/scripts/lib/wic/imager/direct.py
@@ -269,11 +269,11 @@ class DirectImageCreator(BaseImageCreator):
p.source_file,
p.fstype,
p.label,
- fsopts = p.fsopts,
- boot = p.active,
- align = p.align,
- no_table = p.no_table,
- part_type = p.part_type)
+ fsopts=p.fsopts,
+ boot=p.active,
+ align=p.align,
+ no_table=p.no_table,
+ part_type=p.part_type)
self._restore_fstab(fstab)
diff --git a/scripts/lib/wic/kickstart/__init__.py
b/scripts/lib/wic/kickstart/__init__.py
index 1095921..111723b 100644
--- a/scripts/lib/wic/kickstart/__init__.py
+++ b/scripts/lib/wic/kickstart/__init__.py
@@ -68,7 +68,7 @@ def read_kickstart(path):
return ks
-def get_image_size(ks, default = None):
+def get_image_size(ks, default=None):
__size = 0
for p in ks.handler.partition.partitions:
if p.mountpoint == "/" and p.size:
@@ -78,40 +78,40 @@ def get_image_size(ks, default = None):
else:
return default
-def get_image_fstype(ks, default = None):
+def get_image_fstype(ks, default=None):
for p in ks.handler.partition.partitions:
if p.mountpoint == "/" and p.fstype:
return p.fstype
return default
-def get_image_fsopts(ks, default = None):
+def get_image_fsopts(ks, default=None):
for p in ks.handler.partition.partitions:
if p.mountpoint == "/" and p.fsopts:
return p.fsopts
return default
-def get_timeout(ks, default = None):
+def get_timeout(ks, default=None):
if not hasattr(ks.handler.bootloader, "timeout"):
return default
if ks.handler.bootloader.timeout is None:
return default
return int(ks.handler.bootloader.timeout)
-def get_kernel_args(ks, default = "ro rd.live.image"):
+def get_kernel_args(ks, default="ro rd.live.image"):
if not hasattr(ks.handler.bootloader, "appendLine"):
return default
if ks.handler.bootloader.appendLine is None:
return default
return "%s %s" %(default, ks.handler.bootloader.appendLine)
-def get_menu_args(ks, default = ""):
+def get_menu_args(ks, default=""):
Re: [OE-core] [wic][PATCH] wic: Implement --build-rootfs command line option
On Wed, Apr 08, 2015 at 07:18:40PM -0300, João Henrique Ferreira de Freitas wrote: > > Hi Ed, > > I liked. > > Two points: > > - How about if user sets INHERIT = "rm_work"? I mean the image > recipe will be rm, right? I always put my images in RM_WORK_EXCLUDE > so wic can use it. > rm_work removes artifacts required by wic. Wic will complain that rootfs can't be found. There is a bug about i https://bugzilla.yoctoproject.org/show_bug.cgi?id=7042 I closed it as not a bug as wic uses build artifacts by design. > - What will be the behavior if: wic create test-image-4Gb -e > test-image --rootfs rootfs1=test-image --rootfs > rootfs2=bringup-image -f. A believed that only 'test-image' will be > build. Right? > I don't fully understand command line options in your example, but generally you're right. wic will take image name from -e option and run "bitbake test-image" in this case. Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: Add argv argument to main
Make it possible to call wic as an API from tests passing command line parameters as arguments to main. This is yet another enabler for wic unit testing. Signed-off-by: Ed Bartosh --- scripts/wic | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/wic b/scripts/wic index fd4a678..5bedd90 100755 --- a/scripts/wic +++ b/scripts/wic @@ -279,13 +279,13 @@ def start_logging(loglevel): logging.basicConfig(filname='wic.log', filemode='w', level=loglevel) -def main(): +def main(argv): parser = optparse.OptionParser(version="wic version %s" % __version__, usage=wic_usage) parser.disable_interspersed_args() -(options, args) = parser.parse_args() +(options, args) = parser.parse_args(argv) if len(args): if args[0] == "help": @@ -298,7 +298,7 @@ def main(): if __name__ == "__main__": try: -ret = main() +ret = main(sys.argv[1:]) except Exception: ret = 1 import traceback -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] kernel.bbclass: Fix race condition
Race condition between do_compile_kernelmodules and do_shared_workdir
tasks occurs when do_compilemodules changes files in include/generated/*
while do_shared_workdir tries to copy them to shared working directory.
Functionality of do_shared_workdir has been moved to separate function
mk_shared_workdir and called from do_kernel_compile. do_shared_workdir
left in the code as it may be used in many recipes.
[YOCTO #7321]
Signed-off-by: Ed Bartosh
Signed-off-by: Bruce Ashfield
---
meta/classes/kernel.bbclass | 12 +++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 125ed88..aad430a 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -231,6 +231,8 @@ kernel_do_install() {
[ -e Module.symvers ] && install -m 0644 Module.symvers
${D}/boot/Module.symvers-${KERNEL_VERSION}
install -d ${D}${sysconfdir}/modules-load.d
install -d ${D}${sysconfdir}/modprobe.d
+
+ mk_shared_workdir
}
do_install[prefuncs] += "package_get_auto_pr"
@@ -245,7 +247,7 @@ emit_depmod_pkgdata() {
PACKAGEFUNCS += "emit_depmod_pkgdata"
-do_shared_workdir () {
+mk_shared_workdir () {
cd ${B}
kerneldir=${STAGING_KERNEL_BUILDDIR}
@@ -289,6 +291,14 @@ do_shared_workdir () {
fi
}
+# NOTE!!! Functionality of do_shared_workdir has been moved to
mk_shared_workdir
+# and called from kernel_do_compile.
+# It caused race condition with do_compile_kernelmodules when it runs
+# in parallel with do_compile_kernelmodules
+do_shared_workdir () {
+ :
+}
+
# We don't need to stage anything, not the modules/firmware since those would
clash with linux-firmware
sysroot_stage_all () {
:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: code cleanup: undefined variables
Fixed forgoten imports Wic_PartData and ImageError.
Removed unused and undefined variable syslinux_conf.
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/kickstart/custom_commands/__init__.py | 1 +
scripts/lib/wic/plugins/source/bootimg-efi.py | 3 +--
scripts/lib/wic/plugins/source/bootimg-pcbios.py | 1 +
3 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/scripts/lib/wic/kickstart/custom_commands/__init__.py
b/scripts/lib/wic/kickstart/custom_commands/__init__.py
index f84c6d9..b4e613d 100644
--- a/scripts/lib/wic/kickstart/custom_commands/__init__.py
+++ b/scripts/lib/wic/kickstart/custom_commands/__init__.py
@@ -1,6 +1,7 @@
from micpartition import Mic_Partition
from micpartition import Mic_PartData
from partition import Wic_Partition
+from partition import Wic_PartData
__all__ = (
"Mic_Partition",
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py
b/scripts/lib/wic/plugins/source/bootimg-efi.py
index d6c11ed..883378e 100644
--- a/scripts/lib/wic/plugins/source/bootimg-efi.py
+++ b/scripts/lib/wic/plugins/source/bootimg-efi.py
@@ -27,6 +27,7 @@
import os
import shutil
+from wic.utils.errors import ImageError
from wic import kickstart, msger
from wic.utils import misc, fs_related, errors, runner, cmdln
from wic.conf import configmgr
@@ -72,8 +73,6 @@ class BootimgEFIPlugin(SourcePlugin):
grubefi_conf += "linux %s root=%s rootwait %s\n" \
% (kernel, rootstr, options)
grubefi_conf += "}\n"
-if splashline:
-syslinux_conf += "%s\n" % splashline
msger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg" \
% cr_workdir)
diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py
b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
index abb32eb..2280867 100644
--- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py
+++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
@@ -26,6 +26,7 @@
import os
+from wic.utils.errors import ImageError
from wic import kickstart, msger
from wic.utils import misc, fs_related, errors, runner, cmdln
from wic.conf import configmgr
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: code cleanup: wildcard imports
Here is what PEP8(Style Guide for Python Code) says about this:
Wildcard imports (from import *) should be avoided, as they
make it unclear which names are present in the namespace, confusing
both readers and many automated tools.
Signed-off-by: Ed Bartosh
---
scripts/lib/image/engine.py| 18 +-
scripts/lib/wic/imager/direct.py | 1 -
scripts/lib/wic/plugin.py | 2 +-
scripts/lib/wic/plugins/source/bootimg-efi.py | 8 ++---
.../lib/wic/plugins/source/bootimg-partition.py| 2 +-
scripts/lib/wic/plugins/source/bootimg-pcbios.py | 9 ++---
scripts/lib/wic/plugins/source/fsimage.py | 2 +-
scripts/lib/wic/plugins/source/rawcopy.py | 2 +-
scripts/lib/wic/plugins/source/rootfs.py | 9 ++---
scripts/lib/wic/utils/fs_related.py| 5 +--
scripts/lib/wic/utils/partitionedfs.py | 4 +--
scripts/wic| 41 +++---
12 files changed, 43 insertions(+), 60 deletions(-)
diff --git a/scripts/lib/image/engine.py b/scripts/lib/image/engine.py
index 68d1ce2..05c2638 100644
--- a/scripts/lib/image/engine.py
+++ b/scripts/lib/image/engine.py
@@ -42,7 +42,7 @@ from wic.utils import cmdln, misc, errors
from wic.conf import configmgr
from wic.plugin import pluginmgr
from wic.__version__ import VERSION
-from wic.utils.oe.misc import *
+from wic.utils.oe import misc
def verify_build_env():
@@ -65,19 +65,19 @@ def find_artifacts(image_name):
Gather the build artifacts for the current image (the image_name
e.g. core-image-minimal) for the current MACHINE set in local.conf
"""
-bitbake_env_lines = get_bitbake_env_lines()
+bitbake_env_lines = misc.get_bitbake_env_lines()
rootfs_dir = kernel_dir = bootimg_dir = native_sysroot = ""
for line in bitbake_env_lines.split('\n'):
-if (get_line_val(line, "IMAGE_ROOTFS")):
-rootfs_dir = get_line_val(line, "IMAGE_ROOTFS")
+if (misc.get_line_val(line, "IMAGE_ROOTFS")):
+rootfs_dir = misc.get_line_val(line, "IMAGE_ROOTFS")
continue
-if (get_line_val(line, "DEPLOY_DIR_IMAGE")):
-kernel_dir = get_line_val(line, "DEPLOY_DIR_IMAGE")
+if (misc.get_line_val(line, "DEPLOY_DIR_IMAGE")):
+kernel_dir = misc.get_line_val(line, "DEPLOY_DIR_IMAGE")
continue
-if (get_line_val(line, "STAGING_DIR_NATIVE")):
-native_sysroot = get_line_val(line, "STAGING_DIR_NATIVE")
+if (misc.get_line_val(line, "STAGING_DIR_NATIVE")):
+native_sysroot = misc.get_line_val(line, "STAGING_DIR_NATIVE")
continue
return (rootfs_dir, kernel_dir, bootimg_dir, native_sysroot)
@@ -87,7 +87,7 @@ CANNED_IMAGE_DIR = "lib/image/canned-wks" # relative to
scripts
SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
def build_canned_image_list(dl):
-layers_path = get_bitbake_var("BBLAYERS")
+layers_path = misc.get_bitbake_var("BBLAYERS")
canned_wks_layer_dirs = []
if layers_path is not None:
diff --git a/scripts/lib/wic/imager/direct.py b/scripts/lib/wic/imager/direct.py
index 0e687bd..9a7d0f5 100644
--- a/scripts/lib/wic/imager/direct.py
+++ b/scripts/lib/wic/imager/direct.py
@@ -32,7 +32,6 @@ from wic.utils import fs_related, runner, misc
from wic.utils.partitionedfs import Image
from wic.utils.errors import CreatorError, ImageError
from wic.imager.baseimager import BaseImageCreator
-from wic.utils.oe.misc import *
from wic.plugin import pluginmgr
disk_methods = {
diff --git a/scripts/lib/wic/plugin.py b/scripts/lib/wic/plugin.py
index 41a8017..9872d20 100644
--- a/scripts/lib/wic/plugin.py
+++ b/scripts/lib/wic/plugin.py
@@ -20,7 +20,7 @@ import os, sys
from wic import msger
from wic import pluginbase
from wic.utils import errors
-from wic.utils.oe.misc import *
+from wic.utils.oe.misc import get_bitbake_var
__ALL__ = ['PluginMgr', 'pluginmgr']
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py
b/scripts/lib/wic/plugins/source/bootimg-efi.py
index 883378e..2fc0357 100644
--- a/scripts/lib/wic/plugins/source/bootimg-efi.py
+++ b/scripts/lib/wic/plugins/source/bootimg-efi.py
@@ -29,13 +29,9 @@ import shutil
from wic.utils.errors import ImageError
from wic import kickstart, msger
-from wic.utils import misc, fs_related, errors, runner, cmdln
-from wic.conf import configmgr
-from wic.plugin import pluginmgr
-import wic.imager.direct as direct
from wic.pluginbase import SourcePlugin
-from wic.utils.oe.misc import *
-from wic.imager.direct import DirectImageCreator
+from wic.utils.oe.misc import exec_cmd, exec_native_cmd, get_bitbake_var, \
+
[OE-core] [wic][PATCH] wic: try to find bitbake using find_executable API
From: Ed Bartosh
Current wic code was assuming that bitbake directory is on the
same level as scripts, which is not the case for oe classic.
Using find_executable bitbake location should be determined better
as this API uss $PATH to search for executables.
Fixes [YOCTO #7621]
Signed-off-by: Ed Bartosh
Signed-off-by: Ed Bartosh
---
scripts/wic | 20 +++-
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/scripts/wic b/scripts/wic
index 5bedd90..7d388c9 100755
--- a/scripts/wic
+++ b/scripts/wic
@@ -36,19 +36,25 @@ import os
import sys
import optparse
import logging
+from distutils import spawn
# External modules
scripts_path = os.path.abspath(os.path.dirname(__file__))
lib_path = scripts_path + '/lib'
-bitbake_path = os.path.join(scripts_path, '../bitbake/lib')
-sys.path = sys.path + [lib_path, bitbake_path]
+sys.path.append(lib_path)
+
+bitbake_exe = spawn.find_executable('bitbake')
+if bitbake_exe:
+bitbake_path = os.path.join(os.path.dirname(bitbake_exe), '../lib')
+sys.path.append(bitbake_path)
+from bb import cookerdata
+from bb.main import bitbake_main, BitBakeConfigParameters
+else:
+bitbake_main = None
from image.help import *
from image.engine import *
-from bb import cookerdata
-from bb.main import bitbake_main, BitBakeConfigParameters
-
def rootfs_dir_to_args(krootfs_dir):
"""
Get a rootfs_dir dict and serialize to string
@@ -109,6 +115,10 @@ def wic_create_subcommand(args, usage_str):
parser.print_help()
sys.exit(1)
+if options.build_rootfs and not bitbake_main:
+logging.error("Can't build roofs as bitbake is not in the $PATH")
+sys.exit(1)
+
if not options.image_name and not (options.rootfs_dir and
options.bootimg_dir and
options.kernel_dir and
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] Fwd: Re: [wic][PATCH] wic: try to find bitbake using find_executable API
On Mon, Apr 20, 2015 at 08:31:05AM -0400, Philip Balister wrote: > I forgot to cc the list. Unfortunately, this led to the commit going in > with the reference to oe-classic. > > Please people, get your terminology correct. The word soup in the wold > is insane. > My bad. I've received your e-mail, but didn't have time to send the fix during weekend. Today I've noticed that my patch was already merged. -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [master-next][PATCH] wic: Add plugin for single partition disk
Hi Adrian,
Thank you for the plugin! The implementation looks good to me.
See my comments below.
On Mon, Apr 20, 2015 at 04:54:23PM +0200, Adrian Freihofer wrote:
> The wic plugin creates a disk image containig one ext2/3/4 partition.
> No additional boot partition is required. Syslinux is installed into
> the image. The target device is a legacy BIOS PC.
>
> Purpose of this plugin:
> Other avaliable plugins create a fat partition for /boot and an ext
> partition for rootfs. Current linux-yocto kernel packages are not
> compatible with this disk layout. The boot partition is not mounted
> by default, hence the kernel is installed into rootfs and not into
> boot partition. A kernel update ends up in a bricked device. The old
> kernel which is still in boot likely does not even boot with updated
> kernel modules from /. Even if the boot partition is mounted during
> the kernel update the update will fail. The kernel package installs
> a symbolic link which is not supported by the fat partition.
> Creating just one ext partition for boot and rootfs solves all issues
> related to package based kernel updates on the device.
>
When do you think it would make sense to stop using far partition for /boot ?
> The plugin depends on syslinux-nomtools a user space installer for
> syslinux on ext filesystems.
> Thanks to Robert Yang who implemented syslinux-nomtools and supported
> the implementation of this plugin.
>
It's not related to this patch may be, but still. Is syslinux-nomtools
incompatible
with syslinux? Why not to have just one syslinux?
> Signed-off-by: Adrian Freihofer
> ---
> scripts/lib/wic/kickstart/__init__.py | 10 ++
> .../lib/wic/plugins/source/rootfs_pcbios_ext.py| 198
> +
> 2 files changed, 208 insertions(+)
> create mode 100644 scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
>
> diff --git a/scripts/lib/wic/kickstart/__init__.py
> b/scripts/lib/wic/kickstart/__init__.py
> index 111723b..eb9def9 100644
> --- a/scripts/lib/wic/kickstart/__init__.py
> +++ b/scripts/lib/wic/kickstart/__init__.py
> @@ -104,6 +104,16 @@ def get_kernel_args(ks, default="ro rd.live.image"):
> return default
> return "%s %s" %(default, ks.handler.bootloader.appendLine)
>
> +def get_kernel_args_console_serial(kargs):
> +consoles = []
> +for param in kargs.split():
> +param_match =
> re.match("console=(ttyS|ttyUSB)([0-9]+),?([0-9]*)([noe]?)([0-9]?)(r?)", param)
> +if param_match:
> +# console name without index, console index, baudrate, parity,
> number of bits, flow control
> +consoles.append((param_match.group(1), param_match.group(2),
> param_match.group(3),
> + param_match.group(4), param_match.group(5),
> param_match.group(6)))
> +return consoles
> +
> def get_menu_args(ks, default=""):
> if not hasattr(ks.handler.bootloader, "menus"):
> return default
> diff --git a/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
> b/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
> new file mode 100644
> index 000..a05ddcf
> --- /dev/null
> +++ b/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
> @@ -0,0 +1,198 @@
> +# ex:ts=4:sw=4:sts=4:et
> +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> +#
> +# This program is free software; you can distribute it and/or modify
> +# it under the terms of the GNU General Public License version 2 as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for mo details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc.,
> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +#
> +# DESCRIPTION
> +# This plugin creates a disk image containing a bootable root partition with
> +# syslinux installed. The filesystem is ext2/3/4, no extra boot partition is
> +# required.
> +#
> +# Example kickstart file:
> +# part / --source rootfs-pcbios-ext --ondisk sda --fstype=ext4 --label
> rootfs --align 1024
> +# bootloader --source rootfs-pcbios-ext --timeout=0 --append="rootwait
> rootfstype=ext4"
> +#
> +# The first line generates a root file system including a syslinux.cfg file
> +# The "--source rootfs-pcbios-ext" in the second line triggers the
> ldlinux.sys
> +# installation into the image.
> +#
> +# AUTHOR
> +# Adrian Freihofer
> +#
> +
> +import os
> +from wic.utils.errors import ImageError
> +from wic import kickstart
> +from wic import msger
> +from wic.utils import runner
> +from wic.pluginbase import SourcePlugin
> +from wic.utils.oe import misc
> +
> +
> +class RootfsPlugin(SourcePlugin):
> +name = 'rootfs-pcbios-ext'
> +
> +@staticmethod
Re: [OE-core] [master-next][PATCH] wic: Add plugin for single partition disk
On Mon, Apr 20, 2015 at 10:27:26PM +0200, Adrian Freihofer wrote: > Hi Ed, > > Thank you for the response. See my comments below. > > On Mon, 2015-04-20 at 21:21 +0300, Ed Bartosh wrote: > > Hi Adrian, > > > > Thank you for the plugin! The implementation looks good to me. > > See my comments below. > > > > On Mon, Apr 20, 2015 at 04:54:23PM +0200, Adrian Freihofer wrote: > > > The wic plugin creates a disk image containig one ext2/3/4 partition. > > > No additional boot partition is required. Syslinux is installed into > > > the image. The target device is a legacy BIOS PC. > > > > > > Purpose of this plugin: > > > Other avaliable plugins create a fat partition for /boot and an ext > > > partition for rootfs. Current linux-yocto kernel packages are not > > > compatible with this disk layout. The boot partition is not mounted > > > by default, hence the kernel is installed into rootfs and not into > > > boot partition. A kernel update ends up in a bricked device. The old > > > kernel which is still in boot likely does not even boot with updated > > > kernel modules from /. Even if the boot partition is mounted during > > > the kernel update the update will fail. The kernel package installs > > > a symbolic link which is not supported by the fat partition. > > > Creating just one ext partition for boot and rootfs solves all issues > > > related to package based kernel updates on the device. > > > > > When do you think it would make sense to stop using far partition for /boot > > ? > Maybe I've missed something. But as far as I can understand the current > implementation of poky image classes and wic, there is now solution > available which just works. If the user creates an image and later runs > a package based kernel update (e.g. opkg update on the device) the > device is "bricked". The kernel packages do not care about a separate > fat partition for boot. Further on they install a symbolic link which is > not possible on a fat boot partition! If I'm right, my plugin should be > applied immediately. It just solves problems for users of PCs with BIOS. > > You're absolutely right here. Solution with syslinux-nomtools looks much better than what we currently have. That's why I asked when we can switch to new syslinux. As far as i understood getting rid of using fat /boot partition and switching to syslinux-nomtools would be beneficial for everyone as it would make partition scheme simpler and kernel upgrades would not breake it. > > > > > The plugin depends on syslinux-nomtools a user space installer for > > > syslinux on ext filesystems. > > > Thanks to Robert Yang who implemented syslinux-nomtools and supported > > > the implementation of this plugin. > > > > > > > It's not related to this patch may be, but still. Is syslinux-nomtools > > incompatible > > with syslinux? Why not to have just one syslinux? > Regarding the bits installed on the device, there is just one syslinux > bootloader. The bootloader itself has been merged to support all kind of > file systems. > But there are different installers. The installers are just host tools. > The syslinux team distinguishes between installers depending on > different user space libraries and installers depending on kernel > features. Installers depending on the kernel need a mounted file system > (e.g. extlinux). They do not run without root permissions and are > therefore out of scope to be used in poky. The installers which do not > require root permissions depend on a user space implementation of the > corresponding file system. To keep the dependencies per installer > minimal the syslinux team decided to provide different installers for > different file systems. So far there is only a user space installer for > fat file systems available. This is may be the explanation for the dual > partition layout in current poky. There was simply no user space > solution available until now. > Robert Yang from Windriver provided a patch set for a user space > installer for ext file systems. It is called syslinux-nomtools (syslinux > installer which does not depend on msdos file system libraries). The > patches have been merged into poky a few days or weeks ago. One of the > patches is still on master-next which was the reason to commit my plugin > on this branch as well. > I'm aware of Robert's great work. Just wondering why we can't switch to syslinux-nomtools installer and forget about fat /boot partition. I can't think about any other reasons than legacy ones. We should keep supporting curent solution for so
Re: [OE-core] [wic][PATCH] wic: Implement --build-rootfs command line option
On Mon, Apr 20, 2015 at 06:13:20PM +0200, Adrian Freihofer wrote: > Let wic call bitbake seems to be a great idea. > > One question is open to me. What's the recommended approach to resolve > the dependencies between the image, the bootloader and native tools > which are additionally required to create the final disk image? This is open to me too. I was kinda surprised when I saw that no images from 'wic list images' output can be built for not that obvious reasons: $ for image in `wic list images | cut -f3 -d' '` ; do wic create $image -e core-image-minimal; done Checking basic build environment... Done. Creating image(s)... Error: Couldn't find HDDDIR, exiting Checking basic build environment... Done. Creating image(s)... Error: No boot files defined, IMAGE_BOOT_FILES unset Checking basic build environment... Done. Creating image(s)... Error: Couldn't find HDDDIR, exiting Checking basic build environment... Done. Creating image(s)... Error: Please build syslinux first Only last message makes sense to me. The rest two don't give any hints on how to solve the problem. > Basically the right place to resolve this dependency is the kickstart > file. Bitbake should not even know that syslinux or any other bootloader > will be added to the final disk image. In other words > IMAGE_EXTRA_DEPENDS or similar are not the right place to add the > bootloader dependency anymore. But if bitbake does not provide e.g. > syslinux-native wic will fail to use it. This is kind of a chicken egg > problem... > A nice solution would be if wic would maintain a simple dependency list > For example: There is a kickstart file assembling a disk image including > my-mini-image and syslinux-native. To create the partition layout, > parted-native is required as well. Hence bitbake should be called like > this: "bitbake my-mini-image parted-native syslinux-native" before wic > starts assembling the disk image. my-mini-image is provided by the -e > parameter of wic. The latter two tools should be provided by the wic > plugins creating the partitions and the disk image. My proposal is to > extend the internal API of wic by a dependency list where plugins can > append required tools e.g. during start up of wic. This would enable the > disk plugin to request "parted-native" and the boot partition plugin to > request "syslinux-native" before bitbake is called. I like the proposal. Just want to point out to a couple of things. First, parted-native seems to be essential tool for wic, so wic core should check for it from my point of view. Second, it depends on the environment if wic is able to find bootloader or not. For example, baking bootloader is not always enough for wic to find it. I'm still puzzled by this fact, but it depends on the MACHINE and may be other parameters. For example, wic is not able to find bootloader if core-image-minimal is built for default machine qemux86(see errors above), but it works just fine for intel-corei7-64. So, I would propose to start from making wic to work for any build. This way we can understand requirements better. > Further on one more problem could be solved easily. If wic takes care > about the final disk assembly bitbake --fetch-only might not be > sufficient to download everything needed to create the whole firmware. > This makes it hard to create a complete premirror. If wic would support > the dependency list as mentioned above and provide a command line > parameter --fetch-only everything would be consistent. Instead of > calling "bitbake my-mini-image --fetch-only" the user just calls the > corresponding wic command with --fetch-only appended. This would result > in "bitbake my-mini-image parted-native syslinux-native --fetch-only". What's fetch-only option? I don't see it among bitbake options. -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] split_and_strip_files: regroup hardlinks to make build deterministic
Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf
Created separate group of hardlinks for the files inside
the same package. This should prevent stripped files to be
populated outside of package directories.
[YOCTO #7586]
Signed-off-by: Ed Bartosh
---
meta/classes/package.bbclass | 15 ++-
meta/lib/oe/package.py | 3 +--
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f1b966d..e54c366 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -812,7 +812,7 @@ python fixup_perms () {
}
python split_and_strip_files () {
-import stat, errno
+import stat, errno, subprocess
dvar = d.getVar('PKGD', True)
pn = d.getVar('PN', True)
@@ -877,6 +877,7 @@ python split_and_strip_files () {
symlinks = {}
hardlinks = {}
kernmods = []
+inodes = {}
libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
@@ -914,6 +915,18 @@ python split_and_strip_files () {
#bb.note("Sym: %s (%d)" % (ltarget,
isELF(ltarget)))
symlinks[file] = target
continue
+
+if s.st_ino in inodes:
+os.unlink(file)
+os.link(inodes[s.st_ino], file)
+elif s.st_nlink > 1:
+inodes[s.st_ino] = file
+cmd = "cp -fa %s %s.tmp && mv -f %s.tmp %s" % (file,
file, file, file)
+bb.debug(1, "split_and_strip_files: breaking hardlink:
%s" % cmd)
+ret = subprocess.call(cmd, shell=True)
+if ret:
+bb.error("split_and_strip_files: '%s' command
failed" % cmd)
+
# It's a file (or hardlink), not a link
# ...but is it ELF, and is it already stripped?
elf_file = isELF(file)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 8bc56c6..ea6feaa 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -30,8 +30,7 @@ def runstrip(arg):
elif elftype & 8 or elftype & 4:
extraflags = "--remove-section=.comment --remove-section=.note"
-# Use mv to break hardlinks
-stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' &&
mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
bb.debug(1, "runstrip: %s" % stripcmd)
ret = subprocess.call(stripcmd, shell=True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [master-next][PATCHv2] wic: Add plugin for single partition disk
On Tue, Apr 21, 2015 at 11:48:04PM +0200, Adrian Freihofer wrote: > So far I did not recognize the MACHINE dependent behavior you mentioned > in your email. Thanks for the hint. May be wic has already a complexity > demanding for unit tests... > Be my guest :) http://lists.openembedded.org/pipermail/openembedded-core/2015-April/103590.html > Thanks for your support! I hope the updated patch fulfills the > requirements. I'll go and review it right away. Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] split_and_strip_files: regroup hardlinks to make build deterministic
Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf
Created separate group of hardlinks for the files inside
the same package. This should prevent stripped files to be
populated outside of package directories.
[YOCTO #7586]
Signed-off-by: Ed Bartosh
---
meta/classes/package.bbclass | 10 ++
meta/lib/oe/package.py | 3 +--
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f1b966d..b9748e7 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -877,6 +877,7 @@ python split_and_strip_files () {
symlinks = {}
hardlinks = {}
kernmods = []
+inodes = {}
libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
@@ -914,6 +915,15 @@ python split_and_strip_files () {
#bb.note("Sym: %s (%d)" % (ltarget,
isELF(ltarget)))
symlinks[file] = target
continue
+
+if s.st_ino in inodes:
+os.unlink(file)
+os.link(inodes[s.st_ino], file)
+elif s.st_nlink > 1:
+inodes[s.st_ino] = file
+# break hardlink
+bb.utils.copyfile(file, file)
+
# It's a file (or hardlink), not a link
# ...but is it ELF, and is it already stripped?
elf_file = isELF(file)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 8bc56c6..ea6feaa 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -30,8 +30,7 @@ def runstrip(arg):
elif elftype & 8 or elftype & 4:
extraflags = "--remove-section=.comment --remove-section=.note"
-# Use mv to break hardlinks
-stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' &&
mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
bb.debug(1, "runstrip: %s" % stripcmd)
ret = subprocess.call(stripcmd, shell=True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [master-next][PATCHv2] wic: Add plugin for single partition disk
On Tue, Apr 21, 2015 at 11:08:32PM +0200, Adrian Freihofer wrote:
> The wic plugin creates a disk image containig one ext2/3/4 partition.
> No additional boot partition is required. Syslinux is installed into
> the image. The target device is a legacy BIOS PC.
>
> Purpose of this plugin:
> Other avaliable plugins create a fat partition for /boot and an ext
> partition for rootfs. Current linux-yocto kernel packages are not
> compatible with this disk layout. The boot partition is not mounted
> by default, hence the kernel is installed into rootfs and not into
> boot partition. A kernel update ends up in a bricked device. The old
> kernel which is still in boot likely does not even boot with updated
> kernel modules from /. Even if the boot partition is mounted during
> the kernel update the update will fail. The kernel package installs
> a symbolic link which is not supported by the fat partition.
> Creating just one ext partition for boot and rootfs solves all issues
> related to package based kernel updates on the device.
>
> The plugin depends on syslinux-nomtools a user space installer for
> syslinux on ext filesystems.
> Thanks to Robert Yang who implemented syslinux-nomtools and supported
> the implementation of this plugin.
>
Thank you for the update.
See my comments below.
I'd suggest to fix at least this pylint warnings:
C: 78, 4: Class method do_configure_partition should have 'cls' as first
argument (bad-classmethod-argument)
C:119, 4: Class method do_prepare_partition should have 'cls' as first argument
(bad-classmethod-argument)
C:167, 4: Class method do_install_disk should have 'cls' as first argument
(bad-classmethod-argument)
C:183, 8: Invalid variable name "rc" (invalid-name)
> Signed-off-by: Adrian Freihofer
> ---
> scripts/lib/wic/kickstart/__init__.py | 13 ++
> .../lib/wic/plugins/source/rootfs_pcbios_ext.py| 185
> +
> scripts/lib/wic/utils/syslinux.py | 60 +++
> 3 files changed, 258 insertions(+)
> create mode 100644 scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
> create mode 100644 scripts/lib/wic/utils/syslinux.py
>
> diff --git a/scripts/lib/wic/kickstart/__init__.py
> b/scripts/lib/wic/kickstart/__init__.py
> index 111723b..1530c41 100644
> --- a/scripts/lib/wic/kickstart/__init__.py
> +++ b/scripts/lib/wic/kickstart/__init__.py
> @@ -104,6 +104,19 @@ def get_kernel_args(ks, default="ro rd.live.image"):
> return default
> return "%s %s" %(default, ks.handler.bootloader.appendLine)
>
> +def get_kernel_args_console_serial(kargs):
> +"""
> +Extract kernel parameters related to serial consoles
> +"""
> +consoles = []
> +for param in kargs.split():
> +param_match =
> re.match("console=(ttyS|ttyUSB)([0-9]+),?([0-9]*)([noe]?)([0-9]?)(r?)", param)
> +if param_match:
> +# console name without index, console index, baudrate, parity,
> number of bits, flow control
> +consoles.append((param_match.group(1), param_match.group(2),
> param_match.group(3),
> + param_match.group(4), param_match.group(5),
> param_match.group(6)))
> +return consoles
> +
> def get_menu_args(ks, default=""):
> if not hasattr(ks.handler.bootloader, "menus"):
> return default
> diff --git a/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
> b/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
> new file mode 100644
> index 000..b0ad299
> --- /dev/null
> +++ b/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py
> @@ -0,0 +1,185 @@
> +# ex:ts=4:sw=4:sts=4:et
> +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> +#
> +# This program is free software; you can distribute it and/or modify
> +# it under the terms of the GNU General Public License version 2 as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for mo details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc.,
> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +#
> +# AUTHOR
> +# Adrian Freihofer
> +#
> +
> +import os
> +from wic import kickstart
> +from wic import msger
> +from wic.utils import syslinux
> +from wic.utils import runner
> +from wic.utils.oe import misc
> +from wic.utils.errors import ImageError
> +from wic.pluginbase import SourcePlugin
> +
> +
> +# pylint: disable=no-init
> +class RootfsPlugin(SourcePlugin):
> +"""
> +Create root partition and install syslinux bootloader
> +
> +This plugin creates a disk image containing a bootable root partition
> with
> +syslinux installed. The filesystem is ext2/3/4, no extra boot partition
> is
> +re
[OE-core] [wic][PATCH] wic tests: Add functional tests for directdisk
Added 2 simple functional test cases to check if
wic is able to produce images using directdisk.wks
Signed-off-by: Ed Bartosh
---
scripts/setup.cfg | 2 +-
scripts/tests/02_wic_test_directdisk.py | 61 +
2 files changed, 62 insertions(+), 1 deletion(-)
create mode 100644 scripts/tests/02_wic_test_directdisk.py
diff --git a/scripts/setup.cfg b/scripts/setup.cfg
index 48a0bb2..f378efa 100644
--- a/scripts/setup.cfg
+++ b/scripts/setup.cfg
@@ -3,4 +3,4 @@ verbosity=2
exe=1
with-coverage=1
cover-erase=1
-cover-package=wic,image
+cover-package=wic,image,wic.imager
diff --git a/scripts/tests/02_wic_test_directdisk.py
b/scripts/tests/02_wic_test_directdisk.py
new file mode 100644
index 000..9108621
--- /dev/null
+++ b/scripts/tests/02_wic_test_directdisk.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2015, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# AUTHORS
+# Ed Bartosh
+
+"""Test directdisk builds."""
+
+import imp
+import unittest
+import os
+import shutil
+import glob
+
+class TestWicDirectdisk(unittest.TestCase):
+"""Test build of directdisk image"""
+def __init__(self, methodName='runTest'):
+unittest.TestCase.__init__(self, methodName)
+self.main = imp.load_source("wic", "wic").main
+self.resultdir = "/var/tmp/wic/build/"
+
+def setUp(self):
+"""This code executes before each test method."""
+shutil.rmtree(self.resultdir)
+os.chdir(os.getenv("BUILDDIR"))
+
+def _build(self, cmdline):
+"""Call self.main with provided commandline."""
+self.assertIsNone(self.main(cmdline))
+self.assertEqual(1, len(glob.glob(self.resultdir + \
+"directdisk-*.direct")))
+
+def testbuild_image_name(self):
+self._build(["create", "directdisk",
+ "--image-name", "core-image-minimal"])
+
+def testbuild_artifacts(self):
+self._build(["create", "directdisk",
+ "-b", "tmp/sysroots/qemux86/usr/share",
+ "-k", "tmp/deploy/images/qemux86",
+ "-n", "tmp/sysroots/x86_64-linux",
+ "-r", "tmp/work/qemux86-poky-linux/"
+ "core-image-minimal/1.0-r0/rootfs"])
+
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic tests: Add docstrings to test methods
Docstrings make nosetests output more clear:
Test wic create --help ... ok
Test wic --help ... ok
Test wic list --help ... ok
Test wic create directdisk providing all artifacts. ... ok
Test wic create directdisk --image-name core-image-minimal ... ok
Signed-off-by: Ed Bartosh
---
scripts/tests/01_wic_test_help.py | 4
scripts/tests/02_wic_test_directdisk.py | 2 ++
2 files changed, 6 insertions(+)
diff --git a/scripts/tests/01_wic_test_help.py
b/scripts/tests/01_wic_test_help.py
index a64cf2f..4aaa66f 100644
--- a/scripts/tests/01_wic_test_help.py
+++ b/scripts/tests/01_wic_test_help.py
@@ -28,13 +28,17 @@ class TestWicHelp(unittest.TestCase):
"""Test help output of wic (sub)commands"""
def setUp(self):
+"""This code is executed before each test method."""
self.main = imp.load_source("wic", "wic").main
def testhelp(self):
+"""Test wic --help"""
self.assertRaises(SystemExit, self.main, ['--help'])
def testcreatehelp(self):
+"""Test wic create --help"""
self.assertRaises(SystemExit, self.main, ['create', '--help'])
def testlisthelp(self):
+"""Test wic list --help"""
self.assertRaises(SystemExit, self.main, ['list', '--help'])
diff --git a/scripts/tests/02_wic_test_directdisk.py
b/scripts/tests/02_wic_test_directdisk.py
index 9108621..6beff71 100644
--- a/scripts/tests/02_wic_test_directdisk.py
+++ b/scripts/tests/02_wic_test_directdisk.py
@@ -48,10 +48,12 @@ class TestWicDirectdisk(unittest.TestCase):
"directdisk-*.direct")))
def testbuild_image_name(self):
+"""Test wic create directdisk --image-name core-image-minimal"""
self._build(["create", "directdisk",
"--image-name", "core-image-minimal"])
def testbuild_artifacts(self):
+"""Test wic create directdisk providing all artifacts."""
self._build(["create", "directdisk",
"-b", "tmp/sysroots/qemux86/usr/share",
"-k", "tmp/deploy/images/qemux86",
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] split_and_strip_files: regroup hardlinks to make build deterministic
Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf
Created separate group of hardlinks for the files inside
the same package. This should prevent stripped files to be
populated outside of package directories.
[YOCTO #7586]
Signed-off-by: Ed Bartosh
---
meta/classes/package.bbclass | 12
meta/lib/oe/package.py | 3 +--
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f1b966d..4019f12 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -877,6 +877,7 @@ python split_and_strip_files () {
symlinks = {}
hardlinks = {}
kernmods = []
+inodes = {}
libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
@@ -914,6 +915,17 @@ python split_and_strip_files () {
#bb.note("Sym: %s (%d)" % (ltarget,
isELF(ltarget)))
symlinks[file] = target
continue
+
+if s.st_ino in inodes:
+os.unlink(file)
+os.link(inodes[s.st_ino], file)
+cpath.updatecache(file)
+elif s.st_nlink > 1:
+inodes[s.st_ino] = file
+# break hardlink
+bb.utils.copyfile(file, file)
+cpath.updatecache(file)
+
# It's a file (or hardlink), not a link
# ...but is it ELF, and is it already stripped?
elf_file = isELF(file)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 8bc56c6..ea6feaa 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -30,8 +30,7 @@ def runstrip(arg):
elif elftype & 8 or elftype & 4:
extraflags = "--remove-section=.comment --remove-section=.note"
-# Use mv to break hardlinks
-stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' &&
mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
bb.debug(1, "runstrip: %s" % stripcmd)
ret = subprocess.call(stripcmd, shell=True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] split_and_strip_files: regroup hardlinks to make build deterministic
Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf
Created separate group of hardlinks for the files inside
the same package. This should prevent stripped files to be
populated outside of package directories.
[YOCTO #7586]
Signed-off-by: Ed Bartosh
---
meta/classes/package.bbclass | 12
meta/lib/oe/package.py | 3 +--
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f1b966d..dc87206 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -877,6 +877,7 @@ python split_and_strip_files () {
symlinks = {}
hardlinks = {}
kernmods = []
+inodes = {}
libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
@@ -914,6 +915,17 @@ python split_and_strip_files () {
#bb.note("Sym: %s (%d)" % (ltarget,
isELF(ltarget)))
symlinks[file] = target
continue
+if s.st_nlink > 1:
+if s.st_ino in inodes:
+os.unlink(file)
+os.link(inodes[s.st_ino], file)
+elif s.st_nlink > 1:
+inodes[s.st_ino] = file
+# break hardlink
+bb.utils.copyfile(file, file)
+cpath.updatecache(file)
+s = cpath.lstat(file)
+
# It's a file (or hardlink), not a link
# ...but is it ELF, and is it already stripped?
elf_file = isELF(file)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 8bc56c6..ea6feaa 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -30,8 +30,7 @@ def runstrip(arg):
elif elftype & 8 or elftype & 4:
extraflags = "--remove-section=.comment --remove-section=.note"
-# Use mv to break hardlinks
-stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' &&
mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
bb.debug(1, "runstrip: %s" % stripcmd)
ret = subprocess.call(stripcmd, shell=True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [master-next][PATCHv3] wic: Add plugin for single partition disk
l_dir, native_sysroot):
> +"""
> +Assemble partitions to disk image
> +
> +Called after all partitions have been prepared and assembled into a
> +disk image. In this case, we install the MBR.
> +"""
> +mbrfile = os.path.join(native_sysroot, "usr/share/syslinux/mbr.bin")
> +if not os.path.exists(mbrfile):
> +msger.error("Couldn't find %s. Has syslinux-native been baked?"
> % mbrfile)
> +
> +full_path = disk['disk'].device
> +msger.debug("Installing MBR on disk %s as %s with size %s bytes" \
> +% (disk_name, full_path, disk['min_size']))
> +
> +ret_code = runner.show(['dd', 'if=%s' % mbrfile, 'of=%s' %
> full_path, 'conv=notrunc'])
> +if ret_code != 0:
> +raise ImageError("Unable to set MBR to %s" % full_path)
> diff --git a/scripts/lib/wic/utils/syslinux.py
> b/scripts/lib/wic/utils/syslinux.py
> new file mode 100644
> index 000..aace286
> --- /dev/null
> +++ b/scripts/lib/wic/utils/syslinux.py
> @@ -0,0 +1,58 @@
> +# ex:ts=4:sw=4:sts=4:et
> +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> +#
> +# This program is free software; you can redistribute it and/or modify it
> +# under the terms of the GNU General Public License as published by the Free
> +# Software Foundation; version 2 of the License
> +#
> +# This program is distributed in the hope that it will be useful, but
> +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
> +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
> +# for more details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc., 59
> +# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
> +#
> +# AUTHOR
> +# Adrian Freihofer
> +
> +
> +import re
> +from wic import msger
> +
> +
> +def serial_console_form_kargs(kernel_args):
> +"""
> +Create SERIAL... line from kernel parameters
> +
> +syslinux needs a line SERIAL port [baudrate [flowcontrol]]
> +in the syslinux.cfg file. The config line is generated based
> +on kernel boot parameters. The the parameters of the first
> +ttyS console are considered for syslinux config.
> +@param kernel_args kernel command line
> +@return line for syslinux config file e.g. "SERIAL 0 115200"
> +"""
> +syslinux_conf = ""
> +for param in kernel_args.split():
> +param_match =
> re.match("console=ttyS([0-9]+),?([0-9]*)([noe]?)([0-9]?)(r?)", param)
> +if param_match:
> +syslinux_conf += "SERIAL " + param_match.group(1)
> +# baudrate
> +if param_match.group(2):
> +syslinux_conf += " " + param_match.group(2)
> +# parity
> +if param_match.group(3) and param_match.group(3) != 'n':
> +msger.warning("syslinux does not support parity for console.
> {} is ignored."
> + .format(param_match.group(3)))
> +# number of bits
> +if param_match.group(4) and param_match.group(4) != '8':
> +msger.warning("syslinux supports 8 bit console configuration
> only. {} is ignored."
> + .format(param_match.group(4)))
> +# flow control
> +if param_match.group(5) and param_match.group(5) != '':
> +msger.warning("syslinux console flowcontrol configuration.
> {} is ignored."
> + .format(param_match.group(5)))
> +break
> +
> +return syslinux_conf
Acked-by: Ed Bartosh
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic tests: Move wic tests to oe-selftest infra
Modified existing wic test modules to work with oe-selftest
infrastructure.
$ oe-selftest --run-tests wic
...
test01_help (oeqa.selftest.wic.Wic)
Test wic --help ... ok
test02_createhelp (oeqa.selftest.wic.Wic)
Test wic create --help ... ok
test03_listhelp (oeqa.selftest.wic.Wic)
Test wic list --help ... ok
test04_build_image_name (oeqa.selftest.wic.Wic)
Test wic create directdisk --image-name core-image-minimal ... ok
test05_build_artifacts (oeqa.selftest.wic.Wic)
Test wic create directdisk providing all artifacts. ... ok
--
Ran 5 tests in 6.434s
OK
Signed-off-by: Ed Bartosh
---
meta/lib/oeqa/selftest/wic.py | 69 +
scripts/setup.cfg | 6 ---
scripts/tests/01_wic_test_help.py | 44 -
scripts/tests/02_wic_test_directdisk.py | 63 --
4 files changed, 69 insertions(+), 113 deletions(-)
create mode 100644 meta/lib/oeqa/selftest/wic.py
delete mode 100644 scripts/setup.cfg
delete mode 100644 scripts/tests/01_wic_test_help.py
delete mode 100644 scripts/tests/02_wic_test_directdisk.py
diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py
new file mode 100644
index 000..c628b59
--- /dev/null
+++ b/meta/lib/oeqa/selftest/wic.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2015, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# AUTHORS
+# Ed Bartosh
+
+"""Test cases for wic."""
+
+import sys
+
+from glob import glob
+from shutil import rmtree
+
+from oeqa.selftest.base import oeSelfTest
+from oeqa.utils.commands import runCmd
+
+class Wic(oeSelfTest):
+"""Wic test class."""
+
+resultdir = "/var/tmp/wic/build/"
+
+def setUp(self):
+"""This code is executed before each test method."""
+rmtree(self.resultdir, ignore_errors=True)
+
+def test01_help(self):
+"""Test wic --help"""
+self.assertEqual(0, runCmd('wic --help').status)
+
+def test02_createhelp(self):
+"""Test wic create --help"""
+self.assertEqual(0, runCmd('wic creat --help').status)
+
+def test03_listhelp(self):
+"""Test wic list --help"""
+self.assertEqual(0, runCmd('wic list --help').status)
+
+def test04_build_image_name(self):
+"""Test wic create directdisk --image-name core-image-minimal"""
+self.assertEqual(0, runCmd("wic create directdisk "
+ "--image-name core-image-minimal").status)
+self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct")))
+
+def test05_build_artifacts(self):
+"""Test wic create directdisk providing all artifacts."""
+self.assertEqual(0, runCmd("wic create directdisk "
+ "-b tmp/sysroots/qemux86/usr/share "
+ "-k tmp/deploy/images/qemux86 "
+ "-n tmp/sysroots/x86_64-linux "
+ "-r tmp/work/qemux86-poky-linux/"
+ "core-image-minimal/1.0-r0/rootfs").status)
+self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct")))
diff --git a/scripts/setup.cfg b/scripts/setup.cfg
deleted file mode 100644
index f378efa..000
--- a/scripts/setup.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-[nosetests]
-verbosity=2
-exe=1
-with-coverage=1
-cover-erase=1
-cover-package=wic,image,wic.imager
diff --git a/scripts/tests/01_wic_test_help.py
b/scripts/tests/01_wic_test_help.py
deleted file mode 100644
index 4aaa66f..000
--- a/scripts/tests/01_wic_test_help.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (c) 2015, Intel Corporation.
-# Al
[OE-core] [PATCH] oeqa/utils: Allow ~ in bblayers
Bitbake can parse ~ in bblayer's paths.
Added this functionality to oeqa code.
Signed-off-by: Ed Bartosh
---
meta/lib/oeqa/utils/commands.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index e8a467f..bc1dbb1 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -155,6 +155,8 @@ def get_test_layer():
layers = get_bb_var("BBLAYERS").split()
testlayer = None
for l in layers:
+if '~' in l:
+l = os.path.expanduser(l)
if "/meta-selftest" in l and os.path.isdir(l):
testlayer = l
break
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] oeqa/utils: Allow ~ in bblayers
On Thu, Apr 30, 2015 at 01:08:09PM +, Mario Domenech Goulart wrote:
> Hi Ed,
>
> On Thu, 30 Apr 2015 14:04:50 +0300 Ed Bartosh
> wrote:
>
> > Bitbake can parse ~ in bblayer's paths.
> > Added this functionality to oeqa code.
> >
> > Signed-off-by: Ed Bartosh
> > ---
> > meta/lib/oeqa/utils/commands.py | 2 ++
> > 1 file changed, 2 insertions(+)
> >
> > diff --git a/meta/lib/oeqa/utils/commands.py
> > b/meta/lib/oeqa/utils/commands.py
> > index e8a467f..bc1dbb1 100644
> > --- a/meta/lib/oeqa/utils/commands.py
> > +++ b/meta/lib/oeqa/utils/commands.py
> > @@ -155,6 +155,8 @@ def get_test_layer():
> > layers = get_bb_var("BBLAYERS").split()
> > testlayer = None
> > for l in layers:
> > +if '~' in l:
> > +l = os.path.expanduser(l)
>
> Is the "if '~' in l" test necessary?
>
I did this to show why expanduser is called at all. This makes code
self-explanatory and easier to understand.
I was suggested to do it here:
http://lists.openembedded.org/pipermail/bitbake-devel/2015-April/005661.html
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] kernel.bbclass: Fix race condition
Race condition between do_compile_kernelmodules and do_shared_workdir
tasks occurs when do_compilemodules changes files in include/generated/*
while do_shared_workdir tries to copy them to shared working directory.
Functionality of do_shared_workdir has been moved to separate function
mk_shared_workdir and called from do_kernel_compile. do_shared_workdir
left in the code as it may be used in many recipes.
[YOCTO #7321]
Signed-off-by: Ed Bartosh
Signed-off-by: Bruce Ashfield
---
meta/classes/kernel.bbclass | 12 +++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 125ed88..13c0180 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -192,6 +192,8 @@ kernel_do_compile() {
if test "${KERNEL_IMAGETYPE_FOR_MAKE}.gz" = "${KERNEL_IMAGETYPE}"; then
gzip -9c < "${KERNEL_IMAGETYPE_FOR_MAKE}" > "${KERNEL_OUTPUT}"
fi
+
+ mk_shared_workdir
}
do_compile_kernelmodules() {
@@ -245,7 +247,7 @@ emit_depmod_pkgdata() {
PACKAGEFUNCS += "emit_depmod_pkgdata"
-do_shared_workdir () {
+mk_shared_workdir () {
cd ${B}
kerneldir=${STAGING_KERNEL_BUILDDIR}
@@ -289,6 +291,14 @@ do_shared_workdir () {
fi
}
+# NOTE!!! Functionality of do_shared_workdir has been moved to
mk_shared_workdir
+# and called from kernel_do_compile.
+# It caused race condition with do_compile_kernelmodules when it runs
+# in parallel with do_compile_kernelmodules
+do_shared_workdir () {
+ :
+}
+
# We don't need to stage anything, not the modules/firmware since those would
clash with linux-firmware
sysroot_stage_all () {
:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] bitbake: toastergui: show relative paths in configvars view
Stripped topdir from paths to config files in configvars view.
[YOCTO #7463]
Signed-off-by: Ed Bartosh
---
bitbake/lib/toaster/toastergui/templates/configvars.html | 4 ++--
bitbake/lib/toaster/toastergui/templatetags/projecttags.py | 6 ++
bitbake/lib/toaster/toastergui/views.py| 2 ++
3 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/bitbake/lib/toaster/toastergui/templates/configvars.html
b/bitbake/lib/toaster/toastergui/templates/configvars.html
index 3e4c7e8..99ffe8b 100644
--- a/bitbake/lib/toaster/toastergui/templates/configvars.html
+++ b/bitbake/lib/toaster/toastergui/templates/configvars.html
@@ -55,7 +55,7 @@
{{variable.variable_value|truncatechars:153}}
{% if variable.vhistory.all %} {% autoescape off %}
-{{variable.vhistory.all | filter_setin_files:file_filter |
cut_layer_path_prefix:layer_names}}
+{{variable.vhistory.all | filter_setin_files:file_filter |
cut_layer_path_prefix:layer_names | cut_topdir_path_prefix:topdir}}
{% endautoescape %} {% endif %}
@@ -115,7 +115,7 @@
{% for vh in variable.vhistory.all %}
-
{{forloop.counter}}{{vh.file_name|cut_layer_path_prefix:layer_names}}{{vh.operation}}{{vh.line_number}}
+
{{forloop.counter}}{{vh.file_name|cut_layer_path_prefix:layer_names|cut_topdir_path_prefix:topdir}}{{vh.operation}}{{vh.line_number}}
{%endfor%}
diff --git a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
index 54700e3..67762a5 100644
--- a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
+++ b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
@@ -20,6 +20,7 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
from datetime import datetime, timedelta
+from os.path import relpath
import re
from django import template
from django.utils import timezone
@@ -326,3 +327,8 @@ def cut_layer_path_prefix(fullpath,layer_names):
parts = re.split(lname, fullpath, 1)
return lname + parts[1]
return fullpath
+
+@register.filter
+def cut_topdir_path_prefix(fullpath, topdir):
+"""Cut topdir from the fullpath."""
+return relpath(fullpath, topdir) if fullpath.startswith(topdir) else
fullpath
diff --git a/bitbake/lib/toaster/toastergui/views.py
b/bitbake/lib/toaster/toastergui/views.py
index 7849b50..d2cd367 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -39,6 +39,7 @@ from datetime import timedelta, datetime, date
from django.utils import formats
from toastergui.templatetags.projecttags import json as jsonfilter
import json
+from os.path import dirname
# all new sessions should come through the landing page;
# determine in which mode we are running in, and redirect appropriately
@@ -1326,6 +1327,7 @@ def configvars(request, build_id):
'default_orderby' : 'variable_name:+',
'search_term':search_term,
'layer_names' : layer_names,
+'topdir': dirname(build_dir),
# Specifies the display of columns for the table, appearance in
"Edit columns" box, toggling default show/hide, and specifying filters for
columns
'tablecols' : [
{'name': 'Variable',
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] bitbake: toastergui: show relative paths in configvars view
On Tue, May 05, 2015 at 03:57:13PM +0300, Ed Bartosh wrote:
Sorry, wrong list. Please ignore.
> Stripped topdir from paths to config files in configvars view.
>
> [YOCTO #7463]
>
> Signed-off-by: Ed Bartosh
> ---
> bitbake/lib/toaster/toastergui/templates/configvars.html | 4 ++--
> bitbake/lib/toaster/toastergui/templatetags/projecttags.py | 6 ++
> bitbake/lib/toaster/toastergui/views.py| 2 ++
> 3 files changed, 10 insertions(+), 2 deletions(-)
>
> diff --git a/bitbake/lib/toaster/toastergui/templates/configvars.html
> b/bitbake/lib/toaster/toastergui/templates/configvars.html
> index 3e4c7e8..99ffe8b 100644
> --- a/bitbake/lib/toaster/toastergui/templates/configvars.html
> +++ b/bitbake/lib/toaster/toastergui/templates/configvars.html
> @@ -55,7 +55,7 @@
> href="#variable-{{variable.pk}}">{{variable.variable_value|truncatechars:153}}
> href="#variable-{{variable.pk}}">
> {% if variable.vhistory.all %} {% autoescape off %}
> -{{variable.vhistory.all | filter_setin_files:file_filter |
> cut_layer_path_prefix:layer_names}}
> +{{variable.vhistory.all | filter_setin_files:file_filter |
> cut_layer_path_prefix:layer_names | cut_topdir_path_prefix:topdir}}
> {% endautoescape %} {% endif %}
>
>
> @@ -115,7 +115,7 @@
>
> {% for vh in variable.vhistory.all %}
>
> -
> {{forloop.counter}}{{vh.file_name|cut_layer_path_prefix:layer_names}}{{vh.operation}}{{vh.line_number}}
> +
> {{forloop.counter}}{{vh.file_name|cut_layer_path_prefix:layer_names|cut_topdir_path_prefix:topdir}}{{vh.operation}}{{vh.line_number}}
>
> {%endfor%}
>
> diff --git a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
> b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
> index 54700e3..67762a5 100644
> --- a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
> +++ b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
> @@ -20,6 +20,7 @@
> # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
>
> from datetime import datetime, timedelta
> +from os.path import relpath
> import re
> from django import template
> from django.utils import timezone
> @@ -326,3 +327,8 @@ def cut_layer_path_prefix(fullpath,layer_names):
> parts = re.split(lname, fullpath, 1)
> return lname + parts[1]
> return fullpath
> +
> +@register.filter
> +def cut_topdir_path_prefix(fullpath, topdir):
> +"""Cut topdir from the fullpath."""
> +return relpath(fullpath, topdir) if fullpath.startswith(topdir) else
> fullpath
> diff --git a/bitbake/lib/toaster/toastergui/views.py
> b/bitbake/lib/toaster/toastergui/views.py
> index 7849b50..d2cd367 100755
> --- a/bitbake/lib/toaster/toastergui/views.py
> +++ b/bitbake/lib/toaster/toastergui/views.py
> @@ -39,6 +39,7 @@ from datetime import timedelta, datetime, date
> from django.utils import formats
> from toastergui.templatetags.projecttags import json as jsonfilter
> import json
> +from os.path import dirname
>
> # all new sessions should come through the landing page;
> # determine in which mode we are running in, and redirect appropriately
> @@ -1326,6 +1327,7 @@ def configvars(request, build_id):
> 'default_orderby' : 'variable_name:+',
> 'search_term':search_term,
> 'layer_names' : layer_names,
> +'topdir': dirname(build_dir),
> # Specifies the display of columns for the table, appearance in
> "Edit columns" box, toggling default show/hide, and specifying filters for
> columns
> 'tablecols' : [
> {'name': 'Variable',
> --
> 2.1.4
>
--
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] kernel.bbclass: Fix race condition
Race condition between do_compile_kernelmodules and do_shared_workdir
tasks occurs when do_compile_kernelmodules changes files in
include/generated/* while do_shared_workdir tries to copy them to
shared working directory.
Functionality of do_shared_workdir has been moved to separate function
mk_shared_workdir and called from do_compile_kernelmodules.
do_shared_workdir is left in the code as it may be used in many recipes.
[YOCTO #7321]
Signed-off-by: Ed Bartosh
Signed-off-by: Bruce Ashfield
---
meta/classes/kernel.bbclass | 12 +++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 125ed88..7878a73 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -201,6 +201,8 @@ do_compile_kernelmodules() {
else
bbnote "no modules to compile"
fi
+
+ mk_shared_workdir
}
addtask compile_kernelmodules after do_compile before do_strip
@@ -245,7 +247,7 @@ emit_depmod_pkgdata() {
PACKAGEFUNCS += "emit_depmod_pkgdata"
-do_shared_workdir () {
+mk_shared_workdir () {
cd ${B}
kerneldir=${STAGING_KERNEL_BUILDDIR}
@@ -289,6 +291,14 @@ do_shared_workdir () {
fi
}
+# NOTE!!! Functionality of do_shared_workdir has been moved to
mk_shared_workdir
+# and called from kernel_do_compile.
+# It caused race condition with do_compile_kernelmodules when it runs
+# in parallel with do_compile_kernelmodules
+do_shared_workdir () {
+ :
+}
+
# We don't need to stage anything, not the modules/firmware since those would
clash with linux-firmware
sysroot_stage_all () {
:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v.4] kernel.bbclass: Fix race condition
Race condition between do_compile_kernelmodules and do_shared_workdir
tasks occurs when do_compile_kernelmodules changes files in
include/generated/* while do_shared_workdir tries to copy them to
shared working directory.
Fixed race by moving do_shared_workdir after do_compile_kernelmodules.
[YOCTO #7321]
Signed-off-by: Ed Bartosh
---
meta/classes/kernel.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 125ed88..9207ae4 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -234,7 +234,7 @@ kernel_do_install() {
}
do_install[prefuncs] += "package_get_auto_pr"
-addtask shared_workdir after do_compile before do_install
+addtask shared_workdir after do_compile_kernelmodules before do_install
emit_depmod_pkgdata() {
# Stash data for depmod
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v.5] kernel.bbclass: Fix race condition
Race condition between do_compile_kernelmodules and do_shared_workdir
tasks occurs when do_compile_kernelmodules changes files in
include/generated/* while do_shared_workdir tries to copy them to
shared working directory.
Fixed race by moving do_shared_workdir after do_compile but before
do_compile_kernelmodules.
[YOCTO #7321]
Signed-off-by: Ed Bartosh
---
meta/classes/kernel.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 125ed88..74092e9 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -234,7 +234,7 @@ kernel_do_install() {
}
do_install[prefuncs] += "package_get_auto_pr"
-addtask shared_workdir after do_compile before do_install
+addtask shared_workdir after do_compile before do_compile_kernelmodules
emit_depmod_pkgdata() {
# Stash data for depmod
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v2] wic: remove intermediate partitions
Hi Alexandre, Thank you for the patch! Sorry for delayed review. I was on vacation. Acked-by: Ed Bartosh On Fri, May 08, 2015 at 09:17:14PM +0200, Alexandre Belloni wrote: > Remove intermediate partitions that may have been created by a previous > wic invocation. Those partitions are causing issues on some systems. In > particular vfat partition creation is hanging on mcopy execution on > Fedora. > > Signed-off-by: Alexandre Belloni > --- > > Changes in v2: > - use os.remove instead of executing rm directly. > > scripts/lib/wic/kickstart/custom_commands/partition.py | 8 > 1 file changed, 8 insertions(+) > > diff --git a/scripts/lib/wic/kickstart/custom_commands/partition.py > b/scripts/lib/wic/kickstart/custom_commands/partition.py > index ff906010b8bf..e864076c992d 100644 > --- a/scripts/lib/wic/kickstart/custom_commands/partition.py > +++ b/scripts/lib/wic/kickstart/custom_commands/partition.py > @@ -230,6 +230,7 @@ class Wic_PartData(Mic_PartData): > image_rootfs = rootfs_dir > rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label ,self.fstype) > > +os.remove(rootfs) > du_cmd = "du -ks %s" % image_rootfs > out = exec_cmd(du_cmd) > actual_rootfs_size = int(out.split()[0]) > @@ -281,6 +282,7 @@ class Wic_PartData(Mic_PartData): > image_rootfs = rootfs_dir > rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label, self.fstype) > > +os.remove(rootfs) > du_cmd = "du -ks %s" % image_rootfs > out = exec_cmd(du_cmd) > actual_rootfs_size = int(out.split()[0]) > @@ -325,6 +327,7 @@ class Wic_PartData(Mic_PartData): > image_rootfs = rootfs_dir > rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label, self.fstype) > > +os.remove(rootfs) > du_cmd = "du -bks %s" % image_rootfs > out = exec_cmd(du_cmd) > blocks = int(out.split()[0]) > @@ -377,6 +380,7 @@ class Wic_PartData(Mic_PartData): > image_rootfs = rootfs_dir > rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label ,self.fstype) > > +os.remove(rootfs) > squashfs_cmd = "mksquashfs %s %s -noappend" % \ > (image_rootfs, rootfs) > exec_native_cmd(pseudo + squashfs_cmd, native_sysroot) > @@ -415,6 +419,7 @@ class Wic_PartData(Mic_PartData): > """ > fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) > > +os.remove(fs) > dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \ > (fs, self.size) > exec_cmd(dd_cmd) > @@ -442,6 +447,7 @@ class Wic_PartData(Mic_PartData): > """ > fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) > > +os.remove(fs) > dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \ > (fs, self.size) > exec_cmd(dd_cmd) > @@ -466,6 +472,7 @@ class Wic_PartData(Mic_PartData): > Prepare an empty vfat partition. > """ > fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) > +os.remove(fs) > > blocks = self.size > > @@ -492,6 +499,7 @@ class Wic_PartData(Mic_PartData): >"Proceeding as requested." % self.mountpoint) > > fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) > +os.remove(fs) > > # it is not possible to create a squashfs without source data, > # thus prepare an empty temp dir that is used as source > -- > 2.1.4 > -- -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] rootfs.py: Improve rpm log_check_regex
Current regex can cause false negatives if paths in bitbake log files contain "ERR" or "Fail". do_rootfs fails with return code 1 in this case. Improved regexp is based on error messages produced by rpm. Those are found by analyzing rpm source code. [YOCTO #7789] Signed-off-by: Ed Bartosh --- meta/lib/oe/rootfs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py index 6fb749f..ce23b23 100644 --- a/meta/lib/oe/rootfs.py +++ b/meta/lib/oe/rootfs.py @@ -295,7 +295,9 @@ class Rootfs(object): class RpmRootfs(Rootfs): def __init__(self, d, manifest_dir): super(RpmRootfs, self).__init__(d) -self.log_check_regex = '(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)' +self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ + '|exit 1|ERROR: |Error: |Error |ERROR '\ + '|Failed |Failed: |Failed$|Failed\(\d+\):)' self.manifest = RpmManifest(d, manifest_dir) self.pm = RpmPM(d, -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: Make sure file exists before removing it
Bunch of os.remove calls were added to the partition.py lately. They're causing wic to fail with OSError: [Errno 2] No such file or directory if file doesn't exist. Added check for file existence to all recently added calls of os.remove. That should fix this regression. Signed-off-by: Ed Bartosh --- scripts/lib/wic/kickstart/custom_commands/partition.py | 16 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/lib/wic/kickstart/custom_commands/partition.py b/scripts/lib/wic/kickstart/custom_commands/partition.py index e864076..d9f77d9 100644 --- a/scripts/lib/wic/kickstart/custom_commands/partition.py +++ b/scripts/lib/wic/kickstart/custom_commands/partition.py @@ -230,7 +230,7 @@ class Wic_PartData(Mic_PartData): image_rootfs = rootfs_dir rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label ,self.fstype) -os.remove(rootfs) +os.path.isfile(rootfs) and os.remove(rootfs) du_cmd = "du -ks %s" % image_rootfs out = exec_cmd(du_cmd) actual_rootfs_size = int(out.split()[0]) @@ -282,7 +282,7 @@ class Wic_PartData(Mic_PartData): image_rootfs = rootfs_dir rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label, self.fstype) -os.remove(rootfs) +os.path.isfile(rootfs) and os.remove(rootfs) du_cmd = "du -ks %s" % image_rootfs out = exec_cmd(du_cmd) actual_rootfs_size = int(out.split()[0]) @@ -327,7 +327,7 @@ class Wic_PartData(Mic_PartData): image_rootfs = rootfs_dir rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label, self.fstype) -os.remove(rootfs) +os.path.isfile(rootfs) and os.remove(rootfs) du_cmd = "du -bks %s" % image_rootfs out = exec_cmd(du_cmd) blocks = int(out.split()[0]) @@ -380,7 +380,7 @@ class Wic_PartData(Mic_PartData): image_rootfs = rootfs_dir rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label ,self.fstype) -os.remove(rootfs) +os.path.isfile(rootfs) and os.remove(rootfs) squashfs_cmd = "mksquashfs %s %s -noappend" % \ (image_rootfs, rootfs) exec_native_cmd(pseudo + squashfs_cmd, native_sysroot) @@ -419,7 +419,7 @@ class Wic_PartData(Mic_PartData): """ fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) -os.remove(fs) +os.path.isfile(fs) and os.remove(fs) dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \ (fs, self.size) exec_cmd(dd_cmd) @@ -447,7 +447,7 @@ class Wic_PartData(Mic_PartData): """ fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) -os.remove(fs) +os.path.isfile(fs) and os.remove(fs) dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \ (fs, self.size) exec_cmd(dd_cmd) @@ -472,7 +472,7 @@ class Wic_PartData(Mic_PartData): Prepare an empty vfat partition. """ fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) -os.remove(fs) +os.path.isfile(fs) and os.remove(fs) blocks = self.size @@ -499,7 +499,7 @@ class Wic_PartData(Mic_PartData): "Proceeding as requested." % self.mountpoint) fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) -os.remove(fs) +os.path.isfile(fs) and os.remove(fs) # it is not possible to create a squashfs without source data, # thus prepare an empty temp dir that is used as source -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] oe-selftest: Build wic runtime requirements and images before testing
Some native tools (syslinux, parted, mtools, etc) are required
by wic to produce images. Unit tests fail if the tools are
not available.
Baked tools and image-core-minimal used by wic before running tests.
[YOCTO #7730]
Signed-off-by: Ed Bartosh
---
meta/lib/oeqa/selftest/wic.py | 6 ++
1 file changed, 6 insertions(+)
diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py
index c628b59..817685d 100644
--- a/meta/lib/oeqa/selftest/wic.py
+++ b/meta/lib/oeqa/selftest/wic.py
@@ -36,6 +36,12 @@ class Wic(oeSelfTest):
resultdir = "/var/tmp/wic/build/"
+@classmethod
+def setUpClass(cls):
+"""Build wic runtime dependencies and images used in the tests."""
+runCmd('bitbake syslinux syslinux-native parted-native '
+ 'dosfstools-native mtools-native core-image-minimal')
+
def setUp(self):
"""This code is executed before each test method."""
rmtree(self.resultdir, ignore_errors=True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v2] oe-selftest: Build wic runtime requirements and images before testing
Some native tools (syslinux, parted, mtools, etc) are required
by wic to produce images. Unit tests fail if the tools are
not available.
Baked tools and image-core-minimal used by wic before running tests.
[YOCTO #7730]
Signed-off-by: Ed Bartosh
---
meta/lib/oeqa/selftest/wic.py | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py
index c628b59..a2ce2cb 100644
--- a/meta/lib/oeqa/selftest/wic.py
+++ b/meta/lib/oeqa/selftest/wic.py
@@ -29,13 +29,19 @@ from glob import glob
from shutil import rmtree
from oeqa.selftest.base import oeSelfTest
-from oeqa.utils.commands import runCmd
+from oeqa.utils.commands import runCmd, bitbake
class Wic(oeSelfTest):
"""Wic test class."""
resultdir = "/var/tmp/wic/build/"
+@classmethod
+def setUpClass(cls):
+"""Build wic runtime dependencies and images used in the tests."""
+bitbake('syslinux syslinux-native parted-native '
+'dosfstools-native mtools-native core-image-minimal')
+
def setUp(self):
"""This code is executed before each test method."""
rmtree(self.resultdir, ignore_errors=True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] populate_sdk_ext: get NATIVELSBSTRING variable
Setting fixedlsbstring to 'universal' can break populate_sdk_ext
task as NATIVELSBSTRING can be set to universal- in
some cases.
Getting NATIVELSBSTRING value using getVar should fix this.
Signed-off-by: Ed Bartosh
---
meta/classes/populate_sdk_ext.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/classes/populate_sdk_ext.bbclass
b/meta/classes/populate_sdk_ext.bbclass
index 3c3a73c..6ef48c0 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -374,8 +374,8 @@ python copy_buildsystem () {
sstate_out = baseoutpath + '/sstate-cache'
bb.utils.remove(sstate_out, True)
-# uninative.bbclass sets NATIVELSBSTRING to 'universal'
-fixedlsbstring = 'universal'
+
+fixedlsbstring = d.getVar('NATIVELSBSTRING', True)
sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN', True) == '1')
sdk_ext_type = d.getVar('SDK_EXT_TYPE', True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v5 0/6] wic: bugfixes & --fixed-size support, tests, oe-selftest: minor fixes
On Tue, Dec 13, 2016 at 09:07:30AM +0100, Maciej Borzęcki wrote: > On Thu, Nov 24, 2016 at 8:08 AM, Maciej Borzecki > wrote: > > v5 of a patch series previously posted here [1]. > > > > Changes since v4: > > > > * dropped `wic: selftest: do not repeat core-image-minimal` & rebased > > dependant patches > > > > * minor formatting fix in `wic: selftest: add tests for --fixed-size > > partition flags` > > > > [1]. > > http://lists.openembedded.org/pipermail/openembedded-core/2016-November/129103.html > > > > Maciej Borzecki (6): > > oe-selftest: enforce en_US.UTF-8 locale > > oeqa/utils/commands.py: allow use of binaries from native sysroot > > wic: add --fixed-size wks option > > wic: selftest: avoid COMPATIBLE_HOST issues > > wic: selftest: do not assume bzImage kernel image > > wic: selftest: add tests for --fixed-size partition flags > > > > meta/lib/oeqa/selftest/wic.py | 123 > > +++-- > > meta/lib/oeqa/utils/commands.py| 9 ++- > > scripts/lib/wic/help.py| 14 +++- > > scripts/lib/wic/imager/direct.py | 2 +- > > scripts/lib/wic/ksparser.py| 41 +-- > > scripts/lib/wic/partition.py | 88 ++- > > scripts/lib/wic/utils/partitionedfs.py | 2 +- > > scripts/oe-selftest| 3 + > > 8 files changed, 234 insertions(+), 48 deletions(-) > > Is there any additional action needed from me at this point? A rebase > or a resend perhaps? > Rebase&resend would be nice as at lest one patch from this patchset is already accepted. -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] selftest/wic: extending test coverage for WIC script options
Hi Jair,
Thank you for the patch! My comments are below.
On Tue, Dec 13, 2016 at 09:53:27AM -0600, Jair Gonzalez wrote:
> The previous WIC script selftest didn't cover all of its command
> line options. The following test cases were added to complete
> covering them:
>
> 1552 Test wic --version
> 1553 Test wic help create
> 1554 Test wic help list
> 1555 Test wic list images
> 1556 Test wic list source-plugins
> 1557 Test wic listed images help
> 1558 Test wic debug, skip-build-check and build_rootfs
> 1559 Test image vars directory selection
> 1562 Test alternate output directory
> In addition, the following test cases were assigned an ID number on
> Testopia:
>
> 1560 Test creation of systemd-bootdisk image
> 1561 Test creation of sdimage-bootpart image
>
> Finally, part of the test methods were rearranged to group them by
> functionality, and some cleanup was made to improve the code's
> compliance with PEP8 style guide.
I'd suggest to split this patch to at least 3 patches:
- new testcases (fix for YOCTO 10594)
- assigning id numbers
- removing WKS_FILE = "wic-image-minimal" from config
- code cleanup
> Fixes [YOCTO 10594]
>
> Signed-off-by: Jair Gonzalez
>
> ---
> meta/lib/oeqa/selftest/wic.py | 246
> +-
> 1 file changed, 174 insertions(+), 72 deletions(-)
>
> diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py
> index e652fad..46bfb94 100644
> --- a/meta/lib/oeqa/selftest/wic.py
> +++ b/meta/lib/oeqa/selftest/wic.py
> @@ -37,13 +37,13 @@ class Wic(oeSelfTest):
> """Wic test class."""
>
> resultdir = "/var/tmp/wic/build/"
> +alternate_resultdir = "/var/tmp/wic/build/alt/"
> image_is_ready = False
>
> def setUpLocal(self):
> """This code is executed before each test method."""
> self.write_config('IMAGE_FSTYPES += " hddimg"\n'
> - 'MACHINE_FEATURES_append = " efi"\n'
> - 'WKS_FILE = "wic-image-minimal"\n')
I like the change, but it should be also in a separate patch.
> + 'MACHINE_FEATURES_append = " efi"\n')
>
> # Do this here instead of in setUpClass as the base setUp does some
> # clean up which can result in the native tools built earlier in
> @@ -56,10 +56,16 @@ class Wic(oeSelfTest):
>
> rmtree(self.resultdir, ignore_errors=True)
>
> +@testcase(1552)
> +def test_version(self):
> +"""Test wic --version"""
> +self.assertEqual(0, runCmd('wic --version').status)
> +
> @testcase(1208)
> def test_help(self):
> -"""Test wic --help"""
> +"""Test wic --help and wic -h"""
> self.assertEqual(0, runCmd('wic --help').status)
> +self.assertEqual(0, runCmd('wic -h').status)
> @testcase(1209)
> def test_createhelp(self):
> @@ -71,19 +77,74 @@ class Wic(oeSelfTest):
> """Test wic list --help"""
> self.assertEqual(0, runCmd('wic list --help').status)
>
> +@testcase(1553)
> +def test_help_create(self):
> +"""Test wic help create"""
> +self.assertEqual(0, runCmd('wic help create').status)
> +
> +@testcase(1554)
> +def test_help_list(self):
> +"""Test wic help list"""
> +self.assertEqual(0, runCmd('wic help list').status)
> +
> +@testcase(1215)
> +def test_help_overview(self):
> +"""Test wic help overview"""
> +self.assertEqual(0, runCmd('wic help overview').status)
> +
> +@testcase(1216)
> +def test_help_plugins(self):
> +"""Test wic help plugins"""
> +self.assertEqual(0, runCmd('wic help plugins').status)
> +
> +@testcase(1217)
> +def test_help_kickstart(self):
> +"""Test wic help kickstart"""
> +self.assertEqual(0, runCmd('wic help kickstart').status)
> +
> +@testcase(1555)
> +def test_list_images(self):
> +"""Test wic list images"""
> +self.assertEqual(0, runCmd('wic list images').status)
> +
> +@testcase(1556)
> +def test_list_source_plugins(self):
> +"""Test wic list source-plugins"""
> +self.assertEqual(0, runCmd('wic list source-plugins').status)
> +
> +@testcase(1557)
> +def test_listed_images_help(self):
> +"""Test wic listed images help"""
> +output = runCmd('wic list images').output
> +imageDetails = [line.split() for line in output.split('\n')]
> +imageList = [row[0] for row in imageDetails]
How about replacing two last lines with this?
imagelist = [line.split()[0] for line in output.split('\n')]
> +for image in imageList:
> +self.assertEqual(0, runCmd('wic list %s help' % image).status)
> +
> +@testcase(1213)
> +def test_unsupported_subcommand(self):
> +"""Test unsupported subcommand"""
> +self.assertEqual(1, runCmd('wic unsupported',
> + ignore_status=True).status)
> +
> +@testcase(1214)
> +
[OE-core] [PATCH v3] uninative: rebuild uninative for gcc 4.8 and 4.9
Some c++ libraries fail to build if uninative is built
with gcc 5.x and host gcc version is either 4.8 or 4.9.
The issue should be solved by making separate uninative sstate
directory structure sstate-cache/universal- for host gcc
versions 4.8 and 4.9. This causes rebuilds of uninative if host gcc
is either 4.8 or 4.9 and it doesn't match gcc version used to build
uninative.
[YOCTO #10441]
Signed-off-by: Ed Bartosh
---
meta/classes/populate_sdk_ext.bbclass | 5 +++--
meta/classes/uninative.bbclass| 2 +-
meta/lib/oe/utils.py | 14 ++
3 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/meta/classes/populate_sdk_ext.bbclass
b/meta/classes/populate_sdk_ext.bbclass
index 3c3a73c..1affa9d 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -374,8 +374,9 @@ python copy_buildsystem () {
sstate_out = baseoutpath + '/sstate-cache'
bb.utils.remove(sstate_out, True)
-# uninative.bbclass sets NATIVELSBSTRING to 'universal'
-fixedlsbstring = 'universal'
+
+# uninative.bbclass sets NATIVELSBSTRING to 'universal%s' %
oe.utils.host_gcc_version(d)
+fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d)
sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN', True) == '1')
sdk_ext_type = d.getVar('SDK_EXT_TYPE', True)
diff --git a/meta/classes/uninative.bbclass b/meta/classes/uninative.bbclass
index 9242320..11cbf9b 100644
--- a/meta/classes/uninative.bbclass
+++ b/meta/classes/uninative.bbclass
@@ -88,7 +88,7 @@ def enable_uninative(d):
loader = d.getVar("UNINATIVE_LOADER", True)
if os.path.exists(loader):
bb.debug(2, "Enabling uninative")
-d.setVar("NATIVELSBSTRING", "universal")
+d.setVar("NATIVELSBSTRING", "universal%s" %
oe.utils.host_gcc_version(d))
d.appendVar("SSTATEPOSTUNPACKFUNCS", " uninative_changeinterp")
d.prependVar("PATH",
"${STAGING_DIR}-uninative/${BUILD_ARCH}-linux${bindir_native}:")
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index d6545b1..2b095f1 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -230,6 +230,20 @@ def format_pkg_list(pkg_dict, ret_format=None):
return '\n'.join(output)
+def host_gcc_version(d):
+compiler = d.getVar("BUILD_CC", True)
+retval, output = getstatusoutput("%s --version" % compiler)
+if retval:
+bb.fatal("Error running %s --version: %s" % (compiler, output))
+
+import re
+match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0])
+if not match:
+bb.fatal("Can't get compiler version from %s --version output" %
compiler)
+
+version = match.group(1)
+return "-%s" % version if version in ("4.8", "4.9") else ""
+
#
# Python 2.7 doesn't have threaded pools (just multiprocessing)
# so implement a version here
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] selftest/wic: extending test coverage for WIC script options
On Tue, Dec 13, 2016 at 06:24:57PM -0600, Jair Gonzalez wrote:
> Hi Ed,
>
> Thank you for your response and suggestions. Below are my comments.
>
> > > +@testcase(1557)
> > > +def test_listed_images_help(self):
> > > +"""Test wic listed images help"""
> > > +output = runCmd('wic list images').output
> > > +imageDetails = [line.split() for line in output.split('\n')]
> > > +imageList = [row[0] for row in imageDetails]
> > How about replacing two last lines with this?
> > imagelist = [line.split()[0] for line in output.split('\n')]
> I agree. What about this?
> imagelist = [line.split()[0] for line in output.splitlines()]
This is event better, thanks!
> > > +
> > > + "--image-name=core-image-minimal").status)
> > Is '=' mandatory here?
> On wic's help it appears as mandatory, but on practice, it can be used both
> ways. I decided to use both ways along the module to test both usages and
> increase coverage, but not to dedicate specific test cases to each
> combination.
Makes sense to me.
> > > def test_compress_gzip(self):
> > > """Test compressing an image with gzip"""
> > > self.assertEqual(0, runCmd("wic create directdisk "
> > > - "--image-name core-image-minimal "
> > > + "-e core-image-minimal "
> > --image-name is more readable than -e from my point of view.
> Similarly to the '=' to define long option names' arguments, I used both
> forms of each option along the module to increase coverage.
OK
> > > +def test_debug_skip_build_check_and_build_rootfs(self):
> > > +"""Test wic debug, skip-build-check and build_rootfs"""
> > > +self.assertEqual(0, runCmd("wic create directdisk "
> > > + "--image-name=core-image-minimal "
> > > + "-D -s -f").status)
> > > +self.assertEqual(1, len(glob(self.resultdir +
> "directdisk-*.direct")))
> > > +self.assertEqual(0, runCmd("wic create directdisk "
> > > + "--image-name=core-image-minimal "
> > > + "--debug "
> > > + "--skip-build-check "
> > > + "--build-rootfs").status)
> > > +self.assertEqual(1, len(glob(self.resultdir +
> > > + "directdisk-*.direct")))
> > > +
> > I'd split this to two test cases as they're testing two different options.
> Actually, those are three different options (with their short and long
> versions). I did this to not add too many test cases, but as you mention,
> probably it's better to separate them by option to make it clearer.
Agreed.
> core-image-minimal").status)
> > > -self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-
> > *.direct")))
> > > -self.assertEqual(1, len(glob(self.resultdir +
> "HYBRID_ISO_IMG-*.iso")))
> > > + "--image-name core-image-minimal"
> > > + ).status)
> > This is less readable. Is this only to fit the line into 80 chars?
> > If so, let's not do it. Lines up to 100 chars long are more readable than
> this I
> > believe.
> I changed it to conform to PEP8 and increase readability on editors adjusted
> to 80 chars. However, if you consider it's better to leave it on 100 chars,
> I could work within that.
Let's agree on max 100 chars/line if it improves readability. Otherwise
80 is ok.
> > > +self.assertEqual(0, runCmd("wic create directdisk "
> > > + "--image-name=%s "
> > > + "--vars %s"
> > > + % (image, imgenvdir)).status)
> > > +
> > Do we really want to test short and long variant of options?
> > If so, we should do it for all options.
> Within the module, all short and long variant of options are tested. Not all
> combinations of long variants with '=' and without it are tested, though.
OK, makes sense to me.
> > > +@testcase(1562)
> > > +def test_alternate_output_dir(self):
> > > +"""Test alternate output directory"""
> > > +self.assertEqual(0, runCmd("wic create directdisk "
> > > + "-e core-image-minimal "
> > > + "-o %s"
> > > + % self.alternate_resultdir).status)
> > > +self.assertEqual(1, len(glob(self.alternate_resultdir +
> > > + "build/directdisk-*.direct")))
> > > +self.assertEqual(0, runCmd("wic create mkefidisk -e "
> > > + "core-image-minimal "
> > > + "--outdir %s"
> > > + % self.alternate_resultdir).status)
> > > +self.assertEqual(1, len(glob(self.alternate_resultdir +
> > > + "build/mkefidi
Re: [OE-core] [PATCH] wic: obey the rootfs size from the metadata
Hi Christopher,
Thank you for the patch!
+1
On Thu, Dec 15, 2016 at 12:42:39PM -0700, Christopher Larson wrote:
> From: Christopher Larson
>
> When no --size is specified for the rootfs in the .wks, we want to obey the
> rootfs size from the metadata, otherwise the defined IMAGE_ROOTFS_EXTRA_SPACE
> and IMAGE_OVERHEAD_FACTOR will not be obeyed. In some cases, this can result
> in image construction failure, if the size determined by du was insufficient
> to hold the files without the aforementioned extra space.
>
> This fallback from --size to ROOTFS_SIZE was already implemented when
> --rootfs-dir is specified in the .wks, but it did not occur otherwise, neither
> when --rootfs-dir= was passed to `wic create` nor when IMAGE_ROOTFS was used.
> This made a certain amount of sense, as this fallback logic happened at such
> a level that it wasn't able to identify which partitions were rootfs
> partitions otherwise. Rather than doing it at that level, we can do it in
> prepare_rootfs(), which is run by the rootfs source plugins.
>
> Note that IMAGE_OVERHEAD_FACTOR and a --overhead-factor in the .wks will now
> both be applied when --size isn't specified in the .wks. A warning is added
> about this, though a user won't see it unless wic fails or they examine the
> do_image_wic log.
>
> Fixes [YOCTO #10815]
>
> Signed-off-by: Christopher Larson
> ---
> scripts/lib/wic/partition.py | 13 -
> 1 file changed, 12 insertions(+), 1 deletion(-)
>
> diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
> index ac4c836..b191cde 100644
> --- a/scripts/lib/wic/partition.py
> +++ b/scripts/lib/wic/partition.py
> @@ -28,7 +28,7 @@ import os
> import tempfile
>
> from wic.utils.oe.misc import msger, parse_sourceparams
> -from wic.utils.oe.misc import exec_cmd, exec_native_cmd
> +from wic.utils.oe.misc import exec_cmd, exec_native_cmd, get_bitbake_var
> from wic.plugin import pluginmgr
>
> partition_methods = {
> @@ -194,6 +194,17 @@ class Partition():
> msger.error("File system for partition %s not specified in
> kickstart, " \
> "use --fstype option" % (self.mountpoint))
>
> +# Get rootfs size from bitbake variable if it's not set in .ks file
> +if not self.size:
> +# Bitbake variable ROOTFS_SIZE is calculated in
> +# Image._get_rootfs_size method from meta/lib/oe/image.py
> +# using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
> +# IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
> +rsize_bb = get_bitbake_var('ROOTFS_SIZE')
> +if rsize_bb:
> +msger.warning('overhead-factor was specified, but size was
> not, so bitbake variables will be used for the size. In this case both
> IMAGE_OVERHEAD_FACTOR and --overhead-factor will be applied')
> +self.size = int(round(float(rsize_bb)))
> +
> for prefix in ("ext", "btrfs", "vfat", "squashfs"):
> if self.fstype.startswith(prefix):
> method = getattr(self, "prepare_rootfs_" + prefix)
> --
> 2.8.0
>
--
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v2 3/3] selftest/wic: Add tests for --exclude-dir option.
On Wed, Dec 14, 2016 at 05:28:54PM +0100, Kristian Amlie wrote:
> Based partially on an earlier patch by Maciej Borzecki.
>
> Signed-off-by: Kristian Amlie
> ---
> meta/lib/oeqa/selftest/wic.py | 106
> +-
> 1 file changed, 105 insertions(+), 1 deletion(-)
>
> diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py
> index faac11e..b2b0fe8 100644
> --- a/meta/lib/oeqa/selftest/wic.py
> +++ b/meta/lib/oeqa/selftest/wic.py
> @@ -49,7 +49,8 @@ class Wic(oeSelfTest):
> # setUpClass being unavailable.
> if not Wic.image_is_ready:
> bitbake('syslinux syslinux-native parted-native gptfdisk-native '
> -'dosfstools-native mtools-native bmap-tools-native')
> +'dosfstools-native mtools-native bmap-tools-native '
> +'e2tools-native')
> bitbake('core-image-minimal')
> Wic.image_is_ready = True
>
> @@ -299,3 +300,106 @@ class Wic(oeSelfTest):
> self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \
> % image).status)
> self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image)))
> +
> +def test_exclude_path(self):
> +"""Test --exclude-path wks option."""
> +
> +# For using 'e2ls'.
> +old_path = os.environ['PATH']
> +os.environ['PATH'] = get_bb_var('PATH', 'core-image-minimal')
> +
> +wks_file = 'temp.wks'
> +ks = open(wks_file, 'w')
I'd use more pythonic 'with open(wks_file, 'w') as wks:' here.
> +rootfs_dir = get_bb_var('IMAGE_ROOTFS', 'core-image-minimal')
> +ks.write("""part / --source rootfs --ondisk mmcblk0 --fstype=ext4
> --exclude-path usr
> +part /usr --source rootfs --ondisk mmcblk0 --fstype=ext4 --rootfs-dir %s/usr
> +part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/
> --rootfs-dir %s/usr"""
> + % (rootfs_dir, rootfs_dir))
> +ks.close()
> +self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \
> + % wks_file).status)
> +
> +os.remove(wks_file)
> +wicout = glob(self.resultdir + "%s-*direct" % 'temp')
> +self.assertEqual(1, len(wicout))
> +
> +wicimg = wicout[0]
> +
> +# verify partition size with wic
> +res = runCmd("parted -m %s unit b p 2>/dev/null" % wicimg)
> +self.assertEqual(0, res.status)
> +
> +# parse parted output which looks like this:
> +# BYT;\n
> +#
> /var/tmp/wic/build/tmpfwvjjkf_-201611101222-hda.direct:200MiB:file:512:512:msdos::;\n
> +# 1:0.00MiB:200MiB:200MiB:ext4::;\n
> +partlns = res.output.splitlines()[2:]
> +
> +self.assertEqual(3, len(partlns))
> +
> +for part in [1, 2, 3]:
> +part_file = os.path.join(self.resultdir, "selftest_img.part%d" %
> part)
> +partln = partlns[part-1].split(":")
> +self.assertEqual(7, len(partln))
> +start = int(partln[1].rstrip("B")) / 512
> +length = int(partln[3].rstrip("B")) / 512
> +self.assertEqual(0, runCmd("dd if=%s of=%s skip=%d count=%d" %
> + (wicimg, part_file, start,
> length)).status)
> +
> +# Test partition 1, should contain the normal root directories,
> except
> +# /usr.
> +res = runCmd("e2ls %s" % os.path.join(self.resultdir,
> "selftest_img.part1"))
> +self.assertEqual(0, res.status)
> +files = res.output.split()
> +self.assertIn("etc", files)
> +self.assertNotIn("usr", files)
> +
> +# Partition 2, should contain common directories for /usr, not root
> +# directories.
> +res = runCmd("e2ls %s" % os.path.join(self.resultdir,
> "selftest_img.part2"))
> +self.assertEqual(0, res.status)
> +files = res.output.split()
> +self.assertNotIn("etc", files)
> +self.assertNotIn("usr", files)
> +self.assertIn("share", files)
> +
> +# Partition 3, should contain the same as partition 2, including the
> bin
> +# directory, but not the files inside it.
> +res = runCmd("e2ls %s" % os.path.join(self.resultdir,
> "selftest_img.part3"))
> +self.assertEqual(0, res.status)
> +files = res.output.split()
> +self.assertNotIn("etc", files)
> +self.assertNotIn("usr", files)
> +self.assertIn("share", files)
> +self.assertIn("bin", files)
> +res = runCmd("e2ls %s:bin" % os.path.join(self.resultdir,
> "selftest_img.part3"))
> +self.assertEqual(0, res.status)
> +self.assertEqual("No files found!", res.output.strip())
> +
> +for part in [1, 2, 3]:
> +part_file = os.path.join(self.resultdir, "selftest_img.part%d" %
> part)
> +os.remove(part_file)
> +
> +os.environ['PATH'] =
[OE-core] [wic][PATCH] meta-yocto-bsp: add kickstart file for MPC8315
Added kickstart file to produce partitioned image for MPC8315 reference hardware. [YOCTO #8719] Signed-off-by: Ed Bartosh --- scripts/lib/wic/canned-wks/mpc8315e-rdb.wks | 6 ++ 1 file changed, 6 insertions(+) create mode 100644 scripts/lib/wic/canned-wks/mpc8315e-rdb.wks diff --git a/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks b/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks new file mode 100644 index 000..af587b2 --- /dev/null +++ b/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks @@ -0,0 +1,6 @@ +# short-description: Create SD card image with a boot partition +# long-description: Creates a partitioned SD card image. Boot files +# are located in the first partition. + +part /boot --source bootimg-partition --ondisk sdb --fstype=ext3 --label boot +part / --source rootfs --ondisk sdb --fstype=ext3 --label root -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH v2] wic: add kickstart file for MPC8315
Added kickstart file to produce partitioned image for MPC8315 reference hardware. [YOCTO #8719] Signed-off-by: Ed Bartosh --- scripts/lib/wic/canned-wks/mpc8315e-rdb.wks | 6 ++ 1 file changed, 6 insertions(+) create mode 100644 scripts/lib/wic/canned-wks/mpc8315e-rdb.wks diff --git a/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks b/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks new file mode 100644 index 000..af587b2 --- /dev/null +++ b/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks @@ -0,0 +1,6 @@ +# short-description: Create SD card image with a boot partition +# long-description: Creates a partitioned SD card image. Boot files +# are located in the first partition. + +part /boot --source bootimg-partition --ondisk sdb --fstype=ext3 --label boot +part / --source rootfs --ondisk sdb --fstype=ext3 --label root -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [wic][PATCH] meta-yocto-bsp: add kickstart file for MPC8315
Sorry, wrong commit subject line. Please, ignore. On Fri, Dec 16, 2016 at 07:05:14PM +0200, Ed Bartosh wrote: > Added kickstart file to produce partitioned image for > MPC8315 reference hardware. > > [YOCTO #8719] > > Signed-off-by: Ed Bartosh > --- > scripts/lib/wic/canned-wks/mpc8315e-rdb.wks | 6 ++ > 1 file changed, 6 insertions(+) > create mode 100644 scripts/lib/wic/canned-wks/mpc8315e-rdb.wks > > diff --git a/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks > b/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks > new file mode 100644 > index 000..af587b2 > --- /dev/null > +++ b/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks > @@ -0,0 +1,6 @@ > +# short-description: Create SD card image with a boot partition > +# long-description: Creates a partitioned SD card image. Boot files > +# are located in the first partition. > + > +part /boot --source bootimg-partition --ondisk sdb --fstype=ext3 --label boot > +part / --source rootfs --ondisk sdb --fstype=ext3 --label root > -- > 2.1.4 > -- -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: look for wks files in /recipes-images
Currently wic looks for wks files in
/scripts/lib/wic/canned-wks/ directories.
This path is too nested and doesn't look consistent with the
naming scheme of layer directories.
Added 'recipes-images' directory to the list of paths
to look for wks files.
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/engine.py | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index 5b10463..8353edb 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -52,6 +52,7 @@ def verify_build_env():
CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
+RECIPES_IMAGES_DIR = "recipes-images"
def build_canned_image_list(path):
layers_path = misc.get_bitbake_var("BBLAYERS")
@@ -59,8 +60,10 @@ def build_canned_image_list(path):
if layers_path is not None:
for layer_path in layers_path.split():
-cpath = os.path.join(layer_path, SCRIPTS_CANNED_IMAGE_DIR)
-canned_wks_layer_dirs.append(cpath)
+for wks_path in (RECIPES_IMAGES_DIR, SCRIPTS_CANNED_IMAGE_DIR):
+cpath = os.path.join(layer_path, wks_path)
+if os.path.isdir(cpath):
+canned_wks_layer_dirs.append(cpath)
cpath = os.path.join(path, CANNED_IMAGE_DIR)
canned_wks_layer_dirs.append(cpath)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v2 3/3] selftest/wic: Add tests for --exclude-dir option.
On Mon, Dec 19, 2016 at 10:09:22AM +0100, Kristian Amlie wrote:
>
> >> +def test_exclude_path(self):
> >> +"""Test --exclude-path wks option."""
> >> +
> >> +# For using 'e2ls'.
> >> +old_path = os.environ['PATH']
> >> +os.environ['PATH'] = get_bb_var('PATH', 'core-image-minimal')
> >> +
> >> +wks_file = 'temp.wks'
> >> +ks = open(wks_file, 'w')
> >
> > I'd use more pythonic 'with open(wks_file, 'w') as wks:' here.
>
> Done for all three blocks!
>
Thank you!
The patchset looks good to me.
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] wic: obey the rootfs size from the metadata
On Fri, Dec 16, 2016 at 04:47:28PM +0200, Ed Bartosh wrote:
> Hi Christopher,
>
> Thank you for the patch!
>
> +1
>
Would you be willing to add test case(s) for this functionality to wic
selftest module?
> On Thu, Dec 15, 2016 at 12:42:39PM -0700, Christopher Larson wrote:
> > From: Christopher Larson
> >
> > When no --size is specified for the rootfs in the .wks, we want to obey the
> > rootfs size from the metadata, otherwise the defined
> > IMAGE_ROOTFS_EXTRA_SPACE
> > and IMAGE_OVERHEAD_FACTOR will not be obeyed. In some cases, this can result
> > in image construction failure, if the size determined by du was insufficient
> > to hold the files without the aforementioned extra space.
> >
> > This fallback from --size to ROOTFS_SIZE was already implemented when
> > --rootfs-dir is specified in the .wks, but it did not occur otherwise,
> > neither
> > when --rootfs-dir= was passed to `wic create` nor when IMAGE_ROOTFS was
> > used.
> > This made a certain amount of sense, as this fallback logic happened at such
> > a level that it wasn't able to identify which partitions were rootfs
> > partitions otherwise. Rather than doing it at that level, we can do it in
> > prepare_rootfs(), which is run by the rootfs source plugins.
> >
> > Note that IMAGE_OVERHEAD_FACTOR and a --overhead-factor in the .wks will now
> > both be applied when --size isn't specified in the .wks. A warning is added
> > about this, though a user won't see it unless wic fails or they examine the
> > do_image_wic log.
> >
> > Fixes [YOCTO #10815]
> >
> > Signed-off-by: Christopher Larson
> > ---
> > scripts/lib/wic/partition.py | 13 -
> > 1 file changed, 12 insertions(+), 1 deletion(-)
> >
> > diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
> > index ac4c836..b191cde 100644
> > --- a/scripts/lib/wic/partition.py
> > +++ b/scripts/lib/wic/partition.py
> > @@ -28,7 +28,7 @@ import os
> > import tempfile
> >
> > from wic.utils.oe.misc import msger, parse_sourceparams
> > -from wic.utils.oe.misc import exec_cmd, exec_native_cmd
> > +from wic.utils.oe.misc import exec_cmd, exec_native_cmd, get_bitbake_var
> > from wic.plugin import pluginmgr
> >
> > partition_methods = {
> > @@ -194,6 +194,17 @@ class Partition():
> > msger.error("File system for partition %s not specified in
> > kickstart, " \
> > "use --fstype option" % (self.mountpoint))
> >
> > +# Get rootfs size from bitbake variable if it's not set in .ks file
> > +if not self.size:
> > +# Bitbake variable ROOTFS_SIZE is calculated in
> > +# Image._get_rootfs_size method from meta/lib/oe/image.py
> > +# using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
> > +# IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
> > +rsize_bb = get_bitbake_var('ROOTFS_SIZE')
> > +if rsize_bb:
> > +msger.warning('overhead-factor was specified, but size was
> > not, so bitbake variables will be used for the size. In this case both
> > IMAGE_OVERHEAD_FACTOR and --overhead-factor will be applied')
> > +self.size = int(round(float(rsize_bb)))
> > +
> > for prefix in ("ext", "btrfs", "vfat", "squashfs"):
> > if self.fstype.startswith(prefix):
> > method = getattr(self, "prepare_rootfs_" + prefix)
> > --
> > 2.8.0
> >
>
> --
> --
> Regards,
> Ed
> --
> ___
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
--
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v2] wic: look for wks files in /wic
Currently wic looks for wks files in
/scripts/lib/wic/canned-wks/ directories.
This path is too nested and doesn't look consistent with the
naming scheme of layer directories.
Added /wic directory to the list of paths
to look for wks files.
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/engine.py | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index 5b10463..2adef2f 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -52,6 +52,7 @@ def verify_build_env():
CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
+WIC_DIR = "wic"
def build_canned_image_list(path):
layers_path = misc.get_bitbake_var("BBLAYERS")
@@ -59,8 +60,10 @@ def build_canned_image_list(path):
if layers_path is not None:
for layer_path in layers_path.split():
-cpath = os.path.join(layer_path, SCRIPTS_CANNED_IMAGE_DIR)
-canned_wks_layer_dirs.append(cpath)
+for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR):
+cpath = os.path.join(layer_path, wks_path)
+if os.path.isdir(cpath):
+canned_wks_layer_dirs.append(cpath)
cpath = os.path.join(path, CANNED_IMAGE_DIR)
canned_wks_layer_dirs.append(cpath)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] oe-selftest: import git module only when needed
git module is not included into standard Python
library and therefore causes import errors on the systems
where PythonGit is not installed.
As git module only used in the code implementing --repository
functionality it's better to import git only in the scope
that requires it.
[YOCTO #10821]
Signed-off-by: Ed Bartosh
---
meta/lib/oeqa/utils/metadata.py | 2 +-
scripts/oe-selftest | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oeqa/utils/metadata.py b/meta/lib/oeqa/utils/metadata.py
index ecbe763..5d8bf84 100644
--- a/meta/lib/oeqa/utils/metadata.py
+++ b/meta/lib/oeqa/utils/metadata.py
@@ -5,7 +5,6 @@
# Functions to get metadata from the testing host used
# for analytics of test results.
-from git import Repo, InvalidGitRepositoryError, NoSuchPathError
from collections import OrderedDict
from collections.abc import MutableMapping
from xml.dom.minidom import parseString
@@ -46,6 +45,7 @@ def metadata_from_data_store(d):
def get_layers(layers):
""" Returns layer name, branch, and revision as OrderedDict. """
+from git import Repo, InvalidGitRepositoryError, NoSuchPathError
layer_dict = OrderedDict()
for layer in layers.split():
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index f4b861f..bfcea66 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -36,7 +36,6 @@ import re
import fnmatch
import collections
import imp
-import git
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
import scriptpath
@@ -577,6 +576,7 @@ def main():
log.info("Finished")
if args.repository:
+import git
# Commit tests results to repository
metadata = metadata_from_bb()
git_dir = os.path.join(os.getcwd(), 'selftest')
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v2] wic: look for wks files in /wic
On Mon, Dec 19, 2016 at 03:31:29PM -0800, Khem Raj wrote:
> On Mon, Dec 19, 2016 at 4:41 AM, Ed Bartosh
> wrote:
> > Currently wic looks for wks files in
> > /scripts/lib/wic/canned-wks/ directories.
> > This path is too nested and doesn't look consistent with the
> > naming scheme of layer directories.
> >
> > Added /wic directory to the list of paths
> > to look for wks files.
>
> Does it also affect documentation ?
>
Yes, in a way. There is only indirect mentioning of
scripts/lib/wic/canned-wks/ in the documentation. I'm going to add new
path there or even directly mention both paths in dev manual.
> >
> > Signed-off-by: Ed Bartosh
> > ---
> > scripts/lib/wic/engine.py | 7 +--
> > 1 file changed, 5 insertions(+), 2 deletions(-)
> >
> > diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
> > index 5b10463..2adef2f 100644
> > --- a/scripts/lib/wic/engine.py
> > +++ b/scripts/lib/wic/engine.py
> > @@ -52,6 +52,7 @@ def verify_build_env():
> >
> > CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
> > SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
> > +WIC_DIR = "wic"
> >
> > def build_canned_image_list(path):
> > layers_path = misc.get_bitbake_var("BBLAYERS")
> > @@ -59,8 +60,10 @@ def build_canned_image_list(path):
> >
> > if layers_path is not None:
> > for layer_path in layers_path.split():
> > -cpath = os.path.join(layer_path, SCRIPTS_CANNED_IMAGE_DIR)
> > -canned_wks_layer_dirs.append(cpath)
> > +for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR):
> > +cpath = os.path.join(layer_path, wks_path)
> > +if os.path.isdir(cpath):
> > +canned_wks_layer_dirs.append(cpath)
> >
> > cpath = os.path.join(path, CANNED_IMAGE_DIR)
> > canned_wks_layer_dirs.append(cpath)
> > --
> > 2.1.4
> >
> > --
> > ___
> > Openembedded-core mailing list
> > Openembedded-core@lists.openembedded.org
> > http://lists.openembedded.org/mailman/listinfo/openembedded-core
--
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v2 0/4] selftest/wic: extending test coverage for WIC script options
On Mon, Dec 19, 2016 at 03:07:30PM -0600, Jair Gonzalez wrote: > Changed in V2: > > The original patch was splitted and updated according to the discussion on: > http://lists.openembedded.org/pipermail/openembedded-core/2016-December/130131.html > NOTE: The WKS_FILE entry deletion was ommitted from this patch set as it > was introduced on the first patch by mistake. > > The following changes since commit 573c646d4cc62dcd0c230381df4940bdf314d495: > > bitbake: BBHandler: use with instead of open/close (2016-12-16 10:23:24 > +) > > are available in the git repository at: > > git://git.yoctoproject.org/poky-contrib jairglez/wictest > http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=jairglez/wictest > > Jair Gonzalez (4): > selftest/wic: adding Testopia ID numbers to test cases missing it > selftest/wic: code cleanup > selftest/wic: reorganizing test methods by functionality > selftest/wic: extending test coverage for WIC script options > > meta/lib/oeqa/selftest/wic.py | 370 > +++--- > 1 file changed, 238 insertions(+), 132 deletions(-) > Thank you for the updated patchset! It looks good to me. BTW, did you measure its impact on the test run time? -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v6 0/5] wic: bugfixes & --fixed-size support, tests, oeqa: enhancements
On Mon, Dec 19, 2016 at 12:20:56PM +0100, Maciej Borzecki wrote: > v6 of a patch series previously posted here [1]. > > Changes since v5: > > * `oe-selftest: enforce en_US.UTF-8 locale` was merged to master, hence it's > no > longer part of this series > > * rebased on top of master (d0d260b0b85790ceb136dd6b0445e8e33d038f5e) > The patchset looks good to me. Thank you. Couple of questions: - did you test if Lars' change https://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=db08ffee0ad1451f3bf710f4d1b623938ba9aefb works with your patchset? - Regarding oe-selftest fixes, wich machine did you run oe-selftest on? It looks to me that a lot of test cases can be run on your machine if we use another wks. > [1]. > http://lists.openembedded.org/pipermail/openembedded-core/2016-November/129187.html > > Maciej Borzecki (5): > oeqa/utils/commands.py: allow use of binaries from native sysroot > wic: add --fixed-size wks option > wic: selftest: avoid COMPATIBLE_HOST issues > wic: selftest: do not assume bzImage kernel image > wic: selftest: add tests for --fixed-size partition flags > > meta/lib/oeqa/selftest/wic.py | 117 > +++-- > meta/lib/oeqa/utils/commands.py| 9 ++- > scripts/lib/wic/help.py| 14 +++- > scripts/lib/wic/imager/direct.py | 2 +- > scripts/lib/wic/ksparser.py| 41 ++-- > scripts/lib/wic/partition.py | 88 +++-- > scripts/lib/wic/utils/partitionedfs.py | 2 +- > 7 files changed, 224 insertions(+), 49 deletions(-) -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v6 0/5] wic: bugfixes & --fixed-size support, tests, oeqa: enhancements
On Tue, Dec 20, 2016 at 07:27:04PM +0100, Maciej Borzęcki wrote:
> On Tue, Dec 20, 2016 at 3:04 PM, Ed Bartosh
> wrote:
> > On Mon, Dec 19, 2016 at 12:20:56PM +0100, Maciej Borzecki wrote:
> >> v6 of a patch series previously posted here [1].
> >>
> >> Changes since v5:
> >>
> >> * `oe-selftest: enforce en_US.UTF-8 locale` was merged to master, hence
> >> it's no
> >> longer part of this series
> >>
> >> * rebased on top of master (d0d260b0b85790ceb136dd6b0445e8e33d038f5e)
> >>
> >
> > The patchset looks good to me. Thank you.
> >
> > Couple of questions:
> > - did you test if Lars' change
> >
> > https://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=db08ffee0ad1451f3bf710f4d1b623938ba9aefb
> > works with your patchset?
>
> Yes. The series was rebased on top of
> d0d260b0b85790ceb136dd6b0445e8e33d038f5e, which includes Lars' patch.
The series causes conflicts on the latest master. Can you check if it's
really the case or it's just something wrong with my setup?
git-pw apply 3904
Applying series: wic: bugfixes & --fixed-size support, tests,
oe-selftest: minor fixes (rev 3)
Applying: oe-selftest: enforce en_US.UTF-8 locale
Using index info to reconstruct a base tree...
M scripts/oe-selftest
Falling back to patching base and 3-way merge...
No changes -- Patch already applied.
Applying: oeqa/utils/commands.py: allow use of binaries from native
sysroot
Applying: wic: add --fixed-size wks option
Applying: wic: selftest: avoid COMPATIBLE_HOST issues
Using index info to reconstruct a base tree...
M meta/lib/oeqa/selftest/wic.py
Falling back to patching base and 3-way merge...
Auto-merging meta/lib/oeqa/selftest/wic.py
CONFLICT (content): Merge conflict in meta/lib/oeqa/selftest/wic.py
Failed to merge in the changes.
Patch failed at 0004 wic: selftest: avoid COMPATIBLE_HOST issues
The copy of the patch that failed is found in:
/home/ed/git/yocto/patchtest/openembedded-core/.git/rebase-apply/patch
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".
> The tests passed on all MACHINES I've used (though I'm not sure if
> scenario addressed in that patch is covered by relevant test cases).
>
I've asked Lars to add test case for that functionality.
> I believe I have also hit a weird bug in BitbakeVars._parse_line() while
> testing with this patch and meta-mender. I Hope you can confirm it.
>
Yes, I confirm this. Thanks for poining out to it. I'll fix it today.
> From a quick investigation I have determined that _parse_line() may
> interpret arbitrary lines that are of 'key=value' format as legitimate
> bitbake variables. In my case, IMAGE_CMD_sdimg defined in
> mender-sdimg.bbclass includes the following piece:
>
> IMAGE_CMD_sdimg() {
> ...
> REMAINING_SIZE=$(expr ${MENDER_STORAGE_TOTAL_SIZE_MB} - \
> ${MENDER_BOOT_PART_SIZE_MB} - \
> ${MENDER_DATA_PART_SIZE_MB} - \
> ${MENDER_PARTITIONING_OVERHEAD_MB} - \
> $boot_env_size_mb)
> ROOTFS_SIZE=$(expr $REMAINING_SIZE / 2)
> ...
> }
>
> This will obviously be seen in `bitbake -e output`. Looking at code
> here:
> https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/scripts/lib/wic/utils/oe/misc.py?id=db08ffee0ad1451f3bf710f4d1b623938ba9aefb#n151
> the line will be split at '=', then whitespace will be stripped from
> both key and value. So a line (taken from bitbake -e):
>
> ROOTFS_SIZE=\$(expr \$REMAINING_SIZE / 2)
>
> Will define a variable ROOTFS_SIZE with value '\\$(expr \\$REMAINING_SIZE /
> 2)'.
>
> It's a pure coincidence that Lars' patch queries ROOTFS_SIZE which also
> appears in bitbake -e output under exactly the same name.
>
> IMO, when parsing env output, we should expect variable name to start
> right at the beginning of a line, IOW skip the line if key[0].isspace() ==
> True
>
> >
> > - Regarding oe-selftest fixes, wich machine did you run oe-selftest on?
> > It looks to me that a lot of test cases can be run on your machine
> > if we use another wks.
>
> I've run it on beaglebone, vexpress-qemu, qemux86 and qemuarm. Agree
> with your observation though. I think that at least these TCs could be
> made machine agnostic:
>
> - test_build_artifacts
> - test_build_image_name
> - test_compress_*
> - test_gpt_image
> - test_wic_image_type
> - test_wrong_compressor
> - test_rootfs_*
>
>
Yes, that's exactly my point. We should
Re: [OE-core] [PATCH v2 0/4] selftest/wic: extending test coverage for WIC script options
On Tue, Dec 20, 2016 at 04:04:06PM -0600, Jair Gonzalez wrote: > > -Original Message- > > From: Ed Bartosh [mailto:ed.bart...@linux.intel.com] > > Sent: Tuesday, December 20, 2016 7:07 AM > > To: Jair Gonzalez > > Cc: openembedded-core@lists.openembedded.org > > Subject: Re: [OE-core] [PATCH v2 0/4] selftest/wic: extending test > coverage > > for WIC script options > > > > On Mon, Dec 19, 2016 at 03:07:30PM -0600, Jair Gonzalez wrote: > > > Changed in V2: > > > > > > The original patch was splitted and updated according to the > discussion on: > > > http://lists.openembedded.org/pipermail/openembedded-core/2016- > > Decembe > > > r/130131.html > > > NOTE: The WKS_FILE entry deletion was ommitted from this patch set as > > > it was introduced on the first patch by mistake. > > > > > > The following changes since commit > > 573c646d4cc62dcd0c230381df4940bdf314d495: > > > > > > bitbake: BBHandler: use with instead of open/close (2016-12-16 > > > 10:23:24 +) > > > > > > are available in the git repository at: > > > > > > git://git.yoctoproject.org/poky-contrib jairglez/wictest > > > > > > http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=jairglez/wict > > > est > > > > > > Jair Gonzalez (4): > > > selftest/wic: adding Testopia ID numbers to test cases missing it > > > selftest/wic: code cleanup > > > selftest/wic: reorganizing test methods by functionality > > > selftest/wic: extending test coverage for WIC script options > > > > > > meta/lib/oeqa/selftest/wic.py | 370 > > > +++--- > > > 1 file changed, 238 insertions(+), 132 deletions(-) > > > > > > > Thank you for the updated patchset! > > It looks good to me. > > > > BTW, did you measure its impact on the test run time? > > > > -- > > Regards, > > Ed > > Hi Ed, > > Thanks. Regarding the impact, I tested it on my local machine after cleaning > the cache, sstate-cache, downloads and tmp directories, and the difference > between them was about 130 seconds after I applied my commits: > Before: > Ran 28 tests in 4446.939s > After: > Ran 39 tests in 4578.604s > Difference: > 131.665s > > I also tried with a source repository with existing cache and the difference > was about 70 seconds: > Before: > Ran 28 tests in 272.170s > After: > Ran 39 tests in 339.637s > Difference: > 67.467s > Hi Jair, Thanks. The difference is acceptable from my point of view. -- Regards, Ed -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH] wic: fix parsing of 'bitbake -e' output
Current parsing code can wrongly interpret arbitrary lines
that are of 'key=value' format as legitimate bitbake variables.
Implemented more strict parsing of key=value pairs using
regular expressions.
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/utils/oe/misc.py | 12 +---
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py
index fe188c9..1dbbe92 100644
--- a/scripts/lib/wic/utils/oe/misc.py
+++ b/scripts/lib/wic/utils/oe/misc.py
@@ -27,6 +27,7 @@
"""Miscellaneous functions."""
import os
+import re
from collections import defaultdict
from distutils import spawn
@@ -155,14 +156,11 @@ class BitbakeVars(defaultdict):
"""
if "=" not in line:
return
-try:
-key, val = line.split("=")
-except ValueError:
+match = re.match("^(\w+)=(.+)", line)
+if not match:
return
-key = key.strip()
-val = val.strip()
-if key.replace('_', '').isalnum():
-self[image][key] = val.strip('"')
+key, val = match.groups()
+self[image][key] = val.strip('"')
def get_var(self, var, image=None):
"""
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [wic][PATCH v2] wic: fix parsing of 'bitbake -e' output
Current parsing code can wrongly interpret arbitrary lines
that are of 'key=value' format as legitimate bitbake variables.
Implemented more strict parsing of key=value pairs using
regular expressions.
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/utils/oe/misc.py | 14 ++
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py
index fe188c9..2a2fcc9 100644
--- a/scripts/lib/wic/utils/oe/misc.py
+++ b/scripts/lib/wic/utils/oe/misc.py
@@ -27,6 +27,7 @@
"""Miscellaneous functions."""
import os
+import re
from collections import defaultdict
from distutils import spawn
@@ -148,21 +149,18 @@ class BitbakeVars(defaultdict):
self.default_image = None
self.vars_dir = None
-def _parse_line(self, line, image):
+def _parse_line(self, line, image, matcher=re.compile(r"^(\w+)=(.+)")):
"""
Parse one line from bitbake -e output or from .env file.
Put result key-value pair into the storage.
"""
if "=" not in line:
return
-try:
-key, val = line.split("=")
-except ValueError:
+match = matcher.match(line)
+if not match:
return
-key = key.strip()
-val = val.strip()
-if key.replace('_', '').isalnum():
-self[image][key] = val.strip('"')
+key, val = match.groups()
+self[image][key] = val.strip('"')
def get_var(self, var, image=None):
"""
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] image_types.bbclass: look for wks files in /wic
Added /wic directory to the list of paths to look
for wks files. This makes wic behaviour consistent when
invoked manually and by bitbake.
Signed-off-by: Ed Bartosh
---
meta/classes/image_types.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index 8f04849..b526cfa 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -186,7 +186,7 @@ IMAGE_CMD_ubifs = "mkfs.ubifs -r ${IMAGE_ROOTFS} -o
${IMGDEPLOYDIR}/${IMAGE_NAME
WKS_FILE ??= "${IMAGE_BASENAME}.${MACHINE}.wks"
WKS_FILES ?= "${WKS_FILE} ${IMAGE_BASENAME}.wks"
-WKS_SEARCH_PATH ?= "${THISDIR}:${@':'.join('%s/scripts/lib/wic/canned-wks' % l
for l in '${BBPATH}:${COREBASE}'.split(':'))}"
+WKS_SEARCH_PATH ?= "${THISDIR}:${@':'.join('%s/wic' % p for p in
'${BBPATH}'.split(':'))}:${@':'.join('%s/scripts/lib/wic/canned-wks' % l for l
in '${BBPATH}:${COREBASE}'.split(':'))}"
WKS_FULL_PATH = "${@wks_search('${WKS_FILES}'.split(), '${WKS_SEARCH_PATH}')
or ''}"
def wks_search(files, search_path):
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/2] populate_sdk_ext: fix working with uninative sstate
Mapped uninative sstate directories to make ext SDK installer to
use them when it's run on systems with gcc version different from
gcc version used to build installer.
[YOCTO #10832]
Signed-off-by: Ed Bartosh
diff --git a/meta/classes/populate_sdk_ext.bbclass
b/meta/classes/populate_sdk_ext.bbclass
index 0812eea..fc9cc22 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -308,7 +308,10 @@ python copy_buildsystem () {
f.write('BB_SETSCENE_ENFORCE_WHITELIST = "%:* *:do_shared_workdir
*:do_rm_work"\n\n')
# Hide the config information from bitbake output (since it's
fixed within the SDK)
-f.write('BUILDCFG_HEADER = ""\n')
+f.write('BUILDCFG_HEADER = ""\n\n')
+
+# Map gcc-dependent uninative sstate cache for installer usage
+f.write('SSTATE_MIRRORS = "file://universal/(.*)
file://universal-4.9/\\1\\nfile://universal-4.9/(.*)
file://universal-4.8/\\1"\n\n')
# Allow additional config through sdk-extra.conf
fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/2] populate_sdk_ext: whitelist do_package tasks
With enabled SSTATE_MIRRORS sstate code expects mirrors to
contain entries for all tasks, which is not the case for ext
installer as it uses reduced sstate cache.
Added do_package tasks to BB_SETSCENE_ENFORCE_WHITELIST to prevent
installer failing with ERROR: Sstate artifact unavailable
[YOCTO #10832]
Signed-off-by: Ed Bartosh
diff --git a/meta/classes/populate_sdk_ext.bbclass
b/meta/classes/populate_sdk_ext.bbclass
index fc9cc22..d1b3fb4 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -305,7 +305,7 @@ python copy_buildsystem () {
f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')
# Set up whitelist for run on install
-f.write('BB_SETSCENE_ENFORCE_WHITELIST = "%:* *:do_shared_workdir
*:do_rm_work"\n\n')
+f.write('BB_SETSCENE_ENFORCE_WHITELIST = "%:* *:do_shared_workdir
*:do_rm_work *:do_package"\n\n')
# Hide the config information from bitbake output (since it's
fixed within the SDK)
f.write('BUILDCFG_HEADER = ""\n\n')
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v2] wic/isoimage-isohybrid: check for grub-mkimage
On Wed, Jan 04, 2017 at 09:04:43PM +0200, Ioan-Adrian Ratiu wrote:
> The isohybrid plugin uses grub-mkimage but doesn't make sure it gets built.
> Add a check to avoid the following error:
>
> Error: A native program grub-mkimage required to build the image was not
> found (see details above).
>
> Wic failed to find a recipe to build native grub-mkimage. Please file a bug
> against wic.
>
> Signed-off-by: Ioan-Adrian Ratiu
> ---
> scripts/lib/wic/plugins/source/isoimage-isohybrid.py | 5 +
> 1 file changed, 5 insertions(+)
>
> diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> index 849fd8bea3..d616316d61 100644
> --- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> +++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> @@ -242,6 +242,11 @@ class IsoImagePlugin(SourcePlugin):
> msger.info("Building mtools-native...\n")
> exec_cmd("bitbake mtools-native")
>
> +# Make sure grub-mkimage is available in native sysroot
> +if not os.path.isfile("%s/usr/bin/grub-mkimage" % native_sysroot):
> +msger.info("Building grub-native...\n")
> +exec_cmd("bitbake grub-native")
> +
> @classmethod
> def do_configure_partition(cls, part, source_params, creator, cr_workdir,
> oe_builddir, bootimg_dir, kernel_dir,
Unfortunately this approach will not work if wic is run from bitbake.
I'd suggest to simply add grub-native to NATIVE_RECIPES dictionary in
scripts/lib/wic/utils/oe/misc.py
It will make error message more informative and useful.
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v2] wic/isoimage-isohybrid: check for grub-mkimage
On Wed, Jan 04, 2017 at 10:12:55PM +0200, Ed Bartosh wrote:
> On Wed, Jan 04, 2017 at 09:04:43PM +0200, Ioan-Adrian Ratiu wrote:
> > The isohybrid plugin uses grub-mkimage but doesn't make sure it gets built.
> > Add a check to avoid the following error:
> >
> > Error: A native program grub-mkimage required to build the image was not
> > found (see details above).
> >
> > Wic failed to find a recipe to build native grub-mkimage. Please file a bug
> > against wic.
> >
> > Signed-off-by: Ioan-Adrian Ratiu
> > ---
> > scripts/lib/wic/plugins/source/isoimage-isohybrid.py | 5 +
> > 1 file changed, 5 insertions(+)
> >
> > diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> > b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> > index 849fd8bea3..d616316d61 100644
> > --- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> > +++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> > @@ -242,6 +242,11 @@ class IsoImagePlugin(SourcePlugin):
> > msger.info("Building mtools-native...\n")
> > exec_cmd("bitbake mtools-native")
> >
> > +# Make sure grub-mkimage is available in native sysroot
> > +if not os.path.isfile("%s/usr/bin/grub-mkimage" % native_sysroot):
> > +msger.info("Building grub-native...\n")
> > +exec_cmd("bitbake grub-native")
> > +
> > @classmethod
> > def do_configure_partition(cls, part, source_params, creator,
> > cr_workdir,
> > oe_builddir, bootimg_dir, kernel_dir,
>
> Unfortunately this approach will not work if wic is run from bitbake.
>
> I'd suggest to simply add grub-native to NATIVE_RECIPES dictionary in
> scripts/lib/wic/utils/oe/misc.py
> It will make error message more informative and useful.
>
The same should probably be done with the rest of native tools called from
this module. It's better to avoid building them implicitly.
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] oe-selftest: Improve BUILDDIR environment handling
On Wed, Jan 04, 2017 at 11:48:53PM +, Richard Purdie wrote:
> Its possible something (like bitbake/tinfoil2) may mess around with the
> environment and using the enviroment as a global variable store isn't
> particularly nice anyway.
>
> This patch changes the BUILDDIR usages so that the environment isn't used
> as a global store and a global variable is used instead. Whilst that
> is still not perfect, it does avoid the current double and triple backtraces
> we're seeing where tinfoil2/bitbake has trampled the enviroment leading
> to failures of failures making debugging even harder.
>
> Signed-off-by: Richard Purdie
> ---
> scripts/oe-selftest | 19 +--
> 1 file changed, 13 insertions(+), 6 deletions(-)
>
> diff --git a/scripts/oe-selftest b/scripts/oe-selftest
> index bfcea66..e166521 100755
> --- a/scripts/oe-selftest
> +++ b/scripts/oe-selftest
> @@ -111,9 +111,13 @@ def get_args_parser():
> help='Submit test results to a repository')
> return parser
>
> +builddir = None
> +
>
> def preflight_check():
>
> +global builddir
> +
> log.info("Checking that everything is in order before running the tests")
>
> if not os.environ.get("BUILDDIR"):
> @@ -135,7 +139,7 @@ def preflight_check():
> return True
>
> def add_include():
> -builddir = os.environ.get("BUILDDIR")
> +global builddir
You don't need to use 'global' here. It's only mandatory if
you change variable value:
https://docs.python.org/3/reference/simple_stmts.html#the-global-statement
Would it be more readable to use name in upper case: BUILDDIR?
> if "#include added by oe-selftest.py" \
> not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
> log.info("Adding: \"include selftest.inc\" in local.conf")
> @@ -149,7 +153,7 @@ def add_include():
> "\n#include added by oe-selftest.py\ninclude
> bblayers.inc")
>
> def remove_include():
> -builddir = os.environ.get("BUILDDIR")
> +global builddir
> if builddir is None:
> return
> if "#include added by oe-selftest.py" \
> @@ -165,18 +169,21 @@ def remove_include():
> "\n#include added by oe-selftest.py\ninclude
> bblayers.inc")
>
> def remove_inc_files():
> +global builddir
> +if builddir is None:
> +return
> try:
> -os.remove(os.path.join(os.environ.get("BUILDDIR"),
> "conf/selftest.inc"))
> +os.remove(os.path.join(builddir, "conf/selftest.inc"))
> for root, _, files in os.walk(get_test_layer()):
> for f in files:
> if f == 'test_recipe.inc':
> os.remove(os.path.join(root, f))
> -except (AttributeError, OSError,) as e:# AttributeError may happen
> if BUILDDIR is not set
> +except OSError as e:
> pass
>
> for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']:
> try:
> -os.remove(os.path.join(os.environ.get("BUILDDIR"), incl_file))
> +os.remove(os.path.join(builddir, incl_file))
> except:
> pass
>
> @@ -394,7 +401,7 @@ def coverage_setup(coverage_source, coverage_include,
> coverage_omit):
> """ Set up the coverage measurement for the testcases to be run """
> import datetime
> import subprocess
> -builddir = os.environ.get("BUILDDIR")
> +global builddir
> pokydir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
> curcommit= subprocess.check_output(["git", "--git-dir",
> os.path.join(pokydir, ".git"), "rev-parse", "HEAD"]).decode('utf-8')
> coveragerc = "%s/.coveragerc" % builddir
> --
> 2.7.4
>
> --
> ___
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
--
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] oe-selftest: Improve BUILDDIR environment handling
On Thu, Jan 05, 2017 at 10:10:50AM +, Richard Purdie wrote:
> On Thu, 2017-01-05 at 11:40 +0200, Ed Bartosh wrote:
> > On Wed, Jan 04, 2017 at 11:48:53PM +, Richard Purdie wrote:
> > >
> > > Its possible something (like bitbake/tinfoil2) may mess around with
> > > the
> > > environment and using the enviroment as a global variable store
> > > isn't
> > > particularly nice anyway.
> > >
> > > This patch changes the BUILDDIR usages so that the environment
> > > isn't used
> > > as a global store and a global variable is used instead. Whilst
> > > that
> > > is still not perfect, it does avoid the current double and triple
> > > backtraces
> > > we're seeing where tinfoil2/bitbake has trampled the enviroment
> > > leading
> > > to failures of failures making debugging even harder.
> > >
> > > Signed-off-by: Richard Purdie
> > > ---
> > > scripts/oe-selftest | 19 +--
> > > 1 file changed, 13 insertions(+), 6 deletions(-)
> > >
> > > diff --git a/scripts/oe-selftest b/scripts/oe-selftest
> > > index bfcea66..e166521 100755
> > > --- a/scripts/oe-selftest
> > > +++ b/scripts/oe-selftest
> > > @@ -111,9 +111,13 @@ def get_args_parser():
> > > help='Submit test results to a
> > > repository')
> > > return parser
> > >
> > > +builddir = None
> > > +
> > >
> > > def preflight_check():
> > >
> > > +global builddir
> > > +
> > > log.info("Checking that everything is in order before running
> > > the tests")
> > >
> > > if not os.environ.get("BUILDDIR"):
> > > @@ -135,7 +139,7 @@ def preflight_check():
> > > return True
> > >
> > > def add_include():
> > > -builddir = os.environ.get("BUILDDIR")
> > > +global builddir
> > You don't need to use 'global' here. It's only mandatory if
> > you change variable value:
> > https://docs.python.org/3/reference/simple_stmts.html#the-global-stat
> > ement
>
> In this case I did it purely for readability to make it clear how we
> were expecting it to work and to match the other uses. It doesn't hurt
> anything afaik.
>
> > Would it be more readable to use name in upper case: BUILDDIR?
>
> Not sure, that isn't something we've used as a style anywhere else so
> it doesn't really match any other style. Is that a python convention?
No, it's not. I thought it would be more readable as uppercase name
is the same as environment variable name. And it would be different
from local variable names, so you wouldn't need to use 'global'.
--
Regards,
Ed
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v4] wic/isoimage-isohybrid: remove do_stage_partition()
Hi Ioan-Adrian,
Thank you for the patch!
It looks good to me.
+1
On Sat, Jan 07, 2017 at 08:03:01PM +0200, Ioan-Adrian Ratiu wrote:
> The purpouse of this function was to check dependencies for building a
> hybrid iso and build them using bitbake if not found. Calling bitbake in
> this context means this wic plugin itself cannot be instrumented inside
> bitbake recipes which is undesirable, the benefits of this are clear:
> there is no need to maintain outside scripts to generate an iso using wic
> and the isohybrid building logic can be further abstracted away into an
> isohybrid.bbclass in the future which can be easily inherited or something
> similar.
>
> So remove the function and add all dependencies to NATIVE_RECIPES so that
> wic can print useful errors when they're not built.
>
> To automate building the isohybrid image dependencies, add the following
> somewhere in your image build inheritence hierarcy (or maybe create a
> bbclass in the future to do these sort of things automatically):
>
> DEPENDS += "syslinux syslinux-native cdrtools-native e2fsprogs-native \
> parted-native dosfstools-native mtools-native grub-efi-native"
>
> Signed-off-by: Ioan-Adrian Ratiu
> ---
> .../lib/wic/plugins/source/isoimage-isohybrid.py | 49
> --
> scripts/lib/wic/utils/oe/misc.py | 4 ++
> 2 files changed, 4 insertions(+), 49 deletions(-)
>
> diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> index 3858fd439b..15fd858d22 100644
> --- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> +++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
> @@ -194,55 +194,6 @@ class IsoImagePlugin(SourcePlugin):
> return initrd
>
> @classmethod
> -def do_stage_partition(cls, part, source_params, creator, cr_workdir,
> - oe_builddir, bootimg_dir, kernel_dir,
> - native_sysroot):
> -"""
> -Special content staging called before do_prepare_partition().
> -It cheks if all necessary tools are available, if not
> -tries to instal them.
> -"""
> -# Make sure parted is available in native sysroot
> -if not os.path.isfile("%s/usr/sbin/parted" % native_sysroot):
> -msger.info("Building parted-native...\n")
> -exec_cmd("bitbake parted-native")
> -
> -# Make sure mkfs.ext2/3/4 is available in native sysroot
> -if not os.path.isfile("%s/sbin/mkfs.ext2" % native_sysroot):
> -msger.info("Building e2fsprogs-native...\n")
> -exec_cmd("bitbake e2fsprogs-native")
> -
> -# Make sure syslinux is available in sysroot and in native sysroot
> -syslinux_dir = get_bitbake_var("STAGING_DATADIR")
> -if not syslinux_dir:
> -msger.error("Couldn't find STAGING_DATADIR, exiting.\n")
> -if not os.path.exists("%s/syslinux" % syslinux_dir):
> -msger.info("Building syslinux...\n")
> -exec_cmd("bitbake syslinux")
> -if not os.path.exists("%s/syslinux" % syslinux_dir):
> -msger.error("Please build syslinux first\n")
> -
> -# Make sure syslinux is available in native sysroot
> -if not os.path.exists("%s/usr/bin/syslinux" % native_sysroot):
> -msger.info("Building syslinux-native...\n")
> -exec_cmd("bitbake syslinux-native")
> -
> -#Make sure mkisofs is available in native sysroot
> -if not os.path.isfile("%s/usr/bin/mkisofs" % native_sysroot):
> -msger.info("Building cdrtools-native...\n")
> -exec_cmd("bitbake cdrtools-native")
> -
> -# Make sure mkfs.vfat is available in native sysroot
> -if not os.path.isfile("%s/sbin/mkfs.vfat" % native_sysroot):
> -msger.info("Building dosfstools-native...\n")
> -exec_cmd("bitbake dosfstools-native")
> -
> -# Make sure mtools is available in native sysroot
> -if not os.path.isfile("%s/usr/bin/mcopy" % native_sysroot):
> -msger.info("Building mtools-native...\n")
> -exec_cmd("bitbake mtools-native")
> -
> -@classmethod
> def do_configure_partition(cls, part, source_params, creator, cr_workdir,
> oe_builddir, bootimg_dir, kernel_dir,
> native_sysroot):
> diff --git a/scripts/lib/wic/utils/oe/misc.py
> b/scripts/lib/wic/utils/oe/misc.py
> index fe188c9d26..489393a24e 100644
> --- a/scripts/lib/wic/utils/oe/misc.py
> +++ b/scripts/lib/wic/utils/oe/misc.py
> @@ -35,8 +35,11 @@ from wic.utils import runner
>
> # executable -> recipe pairs for exec_native_cmd
> NATIVE_RECIPES = {"bmaptool": "bmap-tools",
> + "grub-mkimage": "grub-efi",
> + "isohybrid": "syslinux",
>"mcopy": "mtools",
>"mkdosfs": "dosfstools",
> +
[OE-core] [PATCH] canned-wks: remove mpc8315e-rdb.wks
This file has been moved to meta-yocto-bsp/wic/ Signed-off-by: Ed Bartosh --- scripts/lib/wic/canned-wks/mpc8315e-rdb.wks | 6 -- 1 file changed, 6 deletions(-) delete mode 100644 scripts/lib/wic/canned-wks/mpc8315e-rdb.wks diff --git a/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks b/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks deleted file mode 100644 index af587b2..000 --- a/scripts/lib/wic/canned-wks/mpc8315e-rdb.wks +++ /dev/null @@ -1,6 +0,0 @@ -# short-description: Create SD card image with a boot partition -# long-description: Creates a partitioned SD card image. Boot files -# are located in the first partition. - -part /boot --source bootimg-partition --ondisk sdb --fstype=ext3 --label boot -part / --source rootfs --ondisk sdb --fstype=ext3 --label root -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/2] wic: _exec_cmd: produce error if exit code is not 0
Current code doesn't always show error output of the
external command and even ignores non-zero exit code.
Moved checking of exit code value to the lowest level
possible: to _exec_cmd. This should make wic to always
check exit code of the external command and issue
an error if it's not 0.
[YOCTO #10816]
Signed-off-by: Ed Bartosh
---
scripts/lib/wic/utils/oe/misc.py | 14 --
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py
index fe188c9..fd71935 100644
--- a/scripts/lib/wic/utils/oe/misc.py
+++ b/scripts/lib/wic/utils/oe/misc.py
@@ -65,6 +65,10 @@ def _exec_cmd(cmd_and_args, as_shell=False, catch=3):
else:
ret, out = runner.runtool(args, catch)
out = out.strip()
+if ret != 0:
+msger.error("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \
+(cmd_and_args, ret, out))
+
msger.debug("_exec_cmd: output for %s (rc = %d): %s" % \
(cmd_and_args, ret, out))
@@ -79,10 +83,6 @@ def exec_cmd(cmd_and_args, as_shell=False, catch=3):
"""
ret, out = _exec_cmd(cmd_and_args, as_shell, catch)
-if ret != 0:
-msger.error("exec_cmd: %s returned '%s' instead of 0" % \
-(cmd_and_args, ret))
-
return out
def exec_native_cmd(cmd_and_args, native_sysroot, catch=3, pseudo=""):
@@ -126,12 +126,6 @@ def exec_native_cmd(cmd_and_args, native_sysroot, catch=3,
pseudo=""):
msg += "Wic failed to find a recipe to build native %s. Please "\
"file a bug against wic.\n" % prog
msger.error(msg)
-if out:
-msger.debug('"%s" output: %s' % (args[0], out))
-
-if ret != 0:
-msger.error("exec_cmd: '%s' returned '%s' instead of 0" % \
-(cmd_and_args, ret))
return ret, out
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/2] direct.py: fix getting image name
part.rootfs_dir was used as an image name in the code. However, when multi-rootfs feature is used this attribute points to the name of the rootfs, e.g. if --rootfs command line is rootfs1=core-image-minimal partf.rootfs_dir is 'rootfs1'. The code also fails when image name is not provided in wic commandline. For example, when wic is called with --rootfs-dir= part.rootfs_dir will contain path and wic will crash trying to call bitbake -e to get value of ROOTFS_SIZE variable. Fixed the code by getting image name properly and checking if it's not a path. [YOCTO #10815] Signed-off-by: Ed Bartosh --- scripts/lib/wic/imager/direct.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/wic/imager/direct.py b/scripts/lib/wic/imager/direct.py index 11ec15e..52828c1 100644 --- a/scripts/lib/wic/imager/direct.py +++ b/scripts/lib/wic/imager/direct.py @@ -270,8 +270,8 @@ class DirectImageCreator(BaseImageCreator): # get rootfs size from bitbake variable if it's not set in .ks file if not part.size: # and if rootfs name is specified for the partition -image_name = part.rootfs_dir -if image_name: +image_name = self.rootfs_dir.get(part.rootfs_dir) +if image_name and os.path.sep not in image_name: # Bitbake variable ROOTFS_SIZE is calculated in # Image._get_rootfs_size method from meta/lib/oe/image.py # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
