sqlite gets really angry if a database is open across a fork() call, and will give all sorts of messages ranging from I/O errors to database corruption errors. To deal with this, close all database connections before forking, and reopen them (lazily) on the other side.
[YOCTO #13030] Signed-off-by: Joshua Watt <[email protected]> --- bitbake/lib/bb/persist_data.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py index 4468facd18f..27ffb1ddaa4 100644 --- a/bitbake/lib/bb/persist_data.py +++ b/bitbake/lib/bb/persist_data.py @@ -30,6 +30,8 @@ from bb.compat import total_ordering from collections import Mapping import sqlite3 import contextlib +import bb.fork +import weakref sqlversion = sqlite3.sqlite_version_info if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): @@ -38,6 +40,28 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): logger = logging.getLogger("BitBake.PersistData") +# Carrying an open database connection across a fork() confuses sqlite and +# results in fun errors like 'database disk image is malformed'. +# To remedy this, close all connections before forking, then they will be +# (lazily) reopen them on the other side. This will cause a lot of problems if +# there are threads running and trying to access the database at the same time, +# but if you are mixing threads and fork() you have no one to blame but +# yourself. If that is discovered to be a problem in the future, some sort of +# per-table reader-writer lock could be used to block the fork() until all +# pending transactions complete +sql_table_weakrefs = [] +def _fork_before_handler(): + for ref in sql_table_weakrefs: + t = ref() + if t is not None and t.connection is not None: + t.connection.close() + t.connection = None + +bb.fork.register_at_fork(before=_fork_before_handler) + +def _remove_table_weakref(ref): + sql_table_weakrefs.remove(ref) + @total_ordering class SQLTable(collections.MutableMapping): class _Decorators(object): @@ -305,4 +329,10 @@ def persist(domain, d): bb.utils.mkdirhier(cachedir) cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3") - return SQLTable(cachefile, domain) + t = SQLTable(cachefile, domain) + + # Add a weak reference to the table list. The weak reference will not keep + # the object alive by itself, so it prevents circular reference counts + sql_table_weakrefs.append(weakref.ref(t, _remove_table_weakref)) + + return t -- 2.19.2 -- _______________________________________________ Openembedded-core mailing list [email protected] http://lists.openembedded.org/mailman/listinfo/openembedded-core
