Hello community,
here is the log from the commit of package baloo5 for openSUSE:Factory checked
in at 2016-10-18 09:56:52
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/baloo5 (Old)
and /work/SRC/openSUSE:Factory/.baloo5.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "baloo5"
Changes:
--------
--- /work/SRC/openSUSE:Factory/baloo5/baloo5.changes 2016-09-14
23:06:15.000000000 +0200
+++ /work/SRC/openSUSE:Factory/.baloo5.new/baloo5.changes 2016-10-18
09:56:53.000000000 +0200
@@ -1,0 +2,18 @@
+Fri Oct 7 21:12:19 UTC 2016 - hrvoje.sen...@gmail.com
+
+- Baloo housekeeping:
+ 0001-fix-baloo_file-crash-with-corrupted-database.patch
+ 0003-allow-ctime-mtime-0.patch (kde#355238)
+ 0004-Increase-size-limit-of-baloo-index-for-64-bit-machin.patch
+ (kde#364475)
+ 0005-Make-e.g.-Baloo-Query-thread-safe.patch
+ 0006-fix-typo.patch
+ 0007-fix-missing-m_env-nullptr-assignment.patch
+ 0008-Transaction-not-created-don-t-try-to-abort-them.patch
+ 0009-add-missing-close-env-as-pointed-out-by-Lo-c-Yhuel.patch
+ (kde#353783)
+ 0010-Fix-sorted-insert-aka-flat_map-like-insert.patch (kde#367991)
+ 0011-Open-baloo-lmdb-database-read-only-beside-in-baloo_f.patch
+ 0012-Use-correct-config-entry-in-autostart-condition.patch
+
+-------------------------------------------------------------------
New:
----
0001-fix-baloo_file-crash-with-corrupted-database.patch
0003-allow-ctime-mtime-0.patch
0004-Increase-size-limit-of-baloo-index-for-64-bit-machin.patch
0005-Make-e.g.-Baloo-Query-thread-safe.patch
0006-fix-typo.patch
0007-fix-missing-m_env-nullptr-assignment.patch
0008-Transaction-not-created-don-t-try-to-abort-them.patch
0009-add-missing-close-env-as-pointed-out-by-Lo-c-Yhuel.patch
0010-Fix-sorted-insert-aka-flat_map-like-insert.patch
0011-Open-baloo-lmdb-database-read-only-beside-in-baloo_f.patch
0012-Use-correct-config-entry-in-autostart-condition.patch
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ baloo5.spec ++++++
--- /var/tmp/diff_new_pack.hf1pU4/_old 2016-10-18 09:56:54.000000000 +0200
+++ /var/tmp/diff_new_pack.hf1pU4/_new 2016-10-18 09:56:54.000000000 +0200
@@ -30,6 +30,18 @@
Source1: baselibs.conf
# PATCH-FIX-OPENSUSE fix-cmakelists-comments.diff -- Change cmake 3.0 block
comments to work in cmake 2.x
Patch1: fix-cmakelists-comments.diff
+# PATCHES 2-12 ARE CONTAINED IN 5.27.0
+Patch2: 0001-fix-baloo_file-crash-with-corrupted-database.patch
+Patch3: 0003-allow-ctime-mtime-0.patch
+Patch4: 0004-Increase-size-limit-of-baloo-index-for-64-bit-machin.patch
+Patch5: 0005-Make-e.g.-Baloo-Query-thread-safe.patch
+Patch6: 0006-fix-typo.patch
+Patch7: 0007-fix-missing-m_env-nullptr-assignment.patch
+Patch8: 0008-Transaction-not-created-don-t-try-to-abort-them.patch
+Patch9: 0009-add-missing-close-env-as-pointed-out-by-Lo-c-Yhuel.patch
+Patch10: 0010-Fix-sorted-insert-aka-flat_map-like-insert.patch
+Patch11: 0011-Open-baloo-lmdb-database-read-only-beside-in-baloo_f.patch
+Patch12: 0012-Use-correct-config-entry-in-autostart-condition.patch
BuildRequires: extra-cmake-modules >= %{_tar_path}
BuildRequires: kf5-filesystem
BuildRequires: libattr-devel
@@ -145,6 +157,17 @@
%prep
%setup -q -n baloo-%{version}
%patch1 -p1
+%patch2 -p1
+%patch3 -p1
+%patch4 -p1
+%patch5 -p1
+%patch6 -p1
+%patch7 -p1
+%patch8 -p1
+%patch9 -p1
+%patch10 -p1
+%patch11 -p1
+%patch12 -p1
%build
%cmake_kf5 -d build -- -DCMAKE_INSTALL_LOCALEDIR=%{_kf5_localedir}
++++++ 0001-fix-baloo_file-crash-with-corrupted-database.patch ++++++
>From a03b0caa4ca1fbfc249bfc0c2730aac340bbf929 Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Fri, 9 Sep 2016 00:00:40 +0200
Subject: [PATCH 01/13] fix baloo_file crash with corrupted database
CHANGELOG: Handle corruption of index database for baloo_file, try to recreate
the database or abort if that fails.
REVIEW: 128865
---
src/file/main.cpp | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/src/file/main.cpp b/src/file/main.cpp
index 9410635..52c42e9 100644
--- a/src/file/main.cpp
+++ b/src/file/main.cpp
@@ -82,7 +82,23 @@ int main(int argc, char** argv)
QFile::remove(path + "/index-lock");
Baloo::Database *db = Baloo::globalDatabaseInstance();
- db->open(Baloo::Database::CreateDatabase);
+
+ /**
+ * try to open, if that fails, try to unlink the index db and retry
+ */
+ if (!db->open(Baloo::Database::CreateDatabase)) {
+ // delete old stuff, set to initial run!
+ qWarning() << "Failed to create database, removing corrupted
database.";
+ QFile::remove(path + "/index");
+ QFile::remove(path + "/index-lock");
+ indexerConfig.setInitialRun(true);
+
+ // try to create now after cleanup, if still no works => fail
+ if (!db->open(Baloo::Database::CreateDatabase)) {
+ qWarning() << "Failed to create database after deleting corrupted
one.";
+ return 1;
+ }
+ }
Baloo::MainHub hub(db, &indexerConfig);
return app.exec();
--
2.10.0
++++++ 0003-allow-ctime-mtime-0.patch ++++++
>From 628daced19b88d0c537736a14aea3287a4662609 Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 18:48:53 +0200
Subject: [PATCH 03/13] allow ctime/mtime == 0
Fix that baloo is instant killed by any file with timestamp 0. (which is OK and
can easily happen after unpacking some zip/tar/..)
REVIEW: 128887
BUG: 355238
---
autotests/unit/engine/documenttimedbtest.cpp | 17 +++++++++++++++++
src/engine/documenttimedb.cpp | 2 --
src/engine/writetransaction.cpp | 3 ---
3 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/autotests/unit/engine/documenttimedbtest.cpp
b/autotests/unit/engine/documenttimedbtest.cpp
index 18231bf..c509e5c 100644
--- a/autotests/unit/engine/documenttimedbtest.cpp
+++ b/autotests/unit/engine/documenttimedbtest.cpp
@@ -28,6 +28,7 @@ class DocumentTimeDBTest : public SingleDBTest
Q_OBJECT
private Q_SLOTS:
void test();
+ void testAllowZeroTime();
};
void DocumentTimeDBTest::test()
@@ -45,6 +46,22 @@ void DocumentTimeDBTest::test()
QCOMPARE(db.get(1), DocumentTimeDB::TimeInfo());
}
+void DocumentTimeDBTest::testAllowZeroTime()
+{
+ DocumentTimeDB db(DocumentTimeDB::create(m_txn), m_txn);
+
+ // we must be able to handle zero time, aka 1970...
+ DocumentTimeDB::TimeInfo info;
+ info.mTime = 0;
+ info.cTime = 0;
+
+ db.put(1, info);
+ QCOMPARE(db.get(1), info);
+
+ db.del(1);
+ QCOMPARE(db.get(1), DocumentTimeDB::TimeInfo());
+}
+
QTEST_MAIN(DocumentTimeDBTest)
#include "documenttimedbtest.moc"
diff --git a/src/engine/documenttimedb.cpp b/src/engine/documenttimedb.cpp
index aa0925a..ef744b9 100644
--- a/src/engine/documenttimedb.cpp
+++ b/src/engine/documenttimedb.cpp
@@ -58,8 +58,6 @@ MDB_dbi DocumentTimeDB::open(MDB_txn* txn)
void DocumentTimeDB::put(quint64 docId, const TimeInfo& info)
{
Q_ASSERT(docId > 0);
- Q_ASSERT(info.mTime);
- Q_ASSERT(info.cTime);
MDB_val key;
key.mv_size = sizeof(quint64);
diff --git a/src/engine/writetransaction.cpp b/src/engine/writetransaction.cpp
index 9ad7520..3808970 100644
--- a/src/engine/writetransaction.cpp
+++ b/src/engine/writetransaction.cpp
@@ -206,9 +206,6 @@ void WriteTransaction::replaceDocument(const Document& doc,
DocumentOperations o
}
if (operations & DocumentTime) {
- Q_ASSERT(doc.m_mTime);
- Q_ASSERT(doc.m_cTime);
-
DocumentTimeDB::TimeInfo info;
info.mTime = doc.m_mTime;
info.cTime = doc.m_cTime;
--
2.10.0
++++++ 0004-Increase-size-limit-of-baloo-index-for-64-bit-machin.patch ++++++
>From b0890aca71aa4f0fdabe65ee7b7fbd0bc844d8b8 Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 18:54:58 +0200
Subject: [PATCH 04/13] Increase size limit of baloo index for 64-bit machines
CHANGELOG: On 64-bit systems baloo allows now > 5 GB index storage.
Increase size limit of baloo index for 64-bit machines to avoid crashs after >
5GB of index size.
(Better would be additional out-of-space handling, but ATM baloo has zero
checks for that)
The size limit for 32-bit is still 1GB, like before (there was a silent
overflow from 5GB to 1GB in the computation), people with large homes will
still get random segfaults on 32-bit.
Patch based on patch from Hao Zhang, Bug 364475
REVIEW: 128885
BUG: 364475
---
src/engine/database.cpp | 12 +++++++++++-
src/engine/databasesize.h | 28 ++++++++++++++--------------
src/engine/transaction.cpp | 2 +-
src/tools/balooctl/statuscommand.cpp | 4 ++--
4 files changed, 28 insertions(+), 18 deletions(-)
diff --git a/src/engine/database.cpp b/src/engine/database.cpp
index 89e2e03..ec7ae2e 100644
--- a/src/engine/database.cpp
+++ b/src/engine/database.cpp
@@ -93,8 +93,18 @@ bool Database::open(OpenMode mode)
return false;
}
+ /**
+ * maximal number of allowed named databases, must match number of
databases we create below
+ * each additional one leads to overhead
+ */
mdb_env_set_maxdbs(m_env, 12);
- mdb_env_set_mapsize(m_env, static_cast<size_t>(1024) * 1024 * 1024 * 5);
// 5 gb
+
+ /**
+ * size limit for database == size limit of mmap
+ * use 1 GB on 32-bit, use 256 GB on 64-bit
+ */
+ const size_t maximalSizeInBytes = size_t((sizeof(size_t) == 4) ? 1 : 256)
* size_t(1024) * size_t(1024) * size_t(1024);
+ mdb_env_set_mapsize(m_env, maximalSizeInBytes);
// The directory needs to be created before opening the environment
QByteArray arr = QFile::encodeName(indexInfo.absoluteFilePath());
diff --git a/src/engine/databasesize.h b/src/engine/databasesize.h
index aa180fb..0a1b97e 100644
--- a/src/engine/databasesize.h
+++ b/src/engine/databasesize.h
@@ -31,30 +31,30 @@ public:
* This is the size which is computed with all the pages used from all the
* individual database pages
*/
- uint expectedSize;
+ size_t expectedSize;
/**
* This is the size based on the MDB_env and the total number of pages used
*/
- uint actualSize;
+ size_t actualSize;
- uint postingDb;
- uint positionDb;
+ size_t postingDb;
+ size_t positionDb;
- uint docTerms;
- uint docFilenameTerms;
- uint docXattrTerms;
+ size_t docTerms;
+ size_t docFilenameTerms;
+ size_t docXattrTerms;
- uint idTree;
- uint idFilename;
+ size_t idTree;
+ size_t idFilename;
- uint docTime;
- uint docData;
+ size_t docTime;
+ size_t docData;
- uint contentIndexingIds;
- uint failedIds;
+ size_t contentIndexingIds;
+ size_t failedIds;
- uint mtimeDb;
+ size_t mtimeDb;
};
}
diff --git a/src/engine/transaction.cpp b/src/engine/transaction.cpp
index 0af20be..908a81f 100644
--- a/src/engine/transaction.cpp
+++ b/src/engine/transaction.cpp
@@ -402,7 +402,7 @@ QVector<QByteArray> Transaction::documentXattrTerms(quint64
docId) const
//
// File Size
//
-static uint dbiSize(MDB_txn* txn, MDB_dbi dbi)
+static size_t dbiSize(MDB_txn* txn, MDB_dbi dbi)
{
MDB_stat stat;
mdb_stat(txn, dbi, &stat);
diff --git a/src/tools/balooctl/statuscommand.cpp
b/src/tools/balooctl/statuscommand.cpp
index 73289c4..1a56c64 100644
--- a/src/tools/balooctl/statuscommand.cpp
+++ b/src/tools/balooctl/statuscommand.cpp
@@ -92,8 +92,8 @@ int StatusCommand::exec(const QCommandLineParser& parser)
const QString path = fileIndexDbPath();
- QFileInfo indexInfo(path + QLatin1String("/index"));
- quint32 size = indexInfo.size();
+ const QFileInfo indexInfo(path + QLatin1String("/index"));
+ const auto size = indexInfo.size();
KFormat format(QLocale::system());
if (size) {
out << "Current size of index is " << format.formatByteSize(size,
2) << endl;
--
2.10.0
++++++ 0005-Make-e.g.-Baloo-Query-thread-safe.patch ++++++
>From e34da150d82a57cf417a59b8b632b2fecb32a6f7 Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 20:24:40 +0200
Subject: [PATCH 05/13] Make e.g. Baloo::Query thread safe.
lmdb itself is thread safe (e.g. you can use the same env in multiple threads).
Unfortunately, the Baloo:atabase itself not, as open() might race against other
open calls (we have one unique db object in baloo).
=> add non-recursive mutex (recursive mutex not needed, one just must avoid to
call isOpen() or path() inside open, that is done, else no unit test works).
REVIEW: 128890
---
src/engine/database.cpp | 21 ++++++++++++++++++---
src/engine/database.h | 47 ++++++++++++++++++++++++++++++++++++++++++-----
2 files changed, 60 insertions(+), 8 deletions(-)
diff --git a/src/engine/database.cpp b/src/engine/database.cpp
index ec7ae2e..8ae6b03 100644
--- a/src/engine/database.cpp
+++ b/src/engine/database.cpp
@@ -1,6 +1,7 @@
/*
This file is part of the KDE Baloo project.
* Copyright (C) 2015 Vishesh Handa <vha...@kde.org>
+ * Copyright (C) 2016 Christoph Cullmann <cullm...@kde.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -43,23 +44,30 @@
#include <QFile>
#include <QFileInfo>
#include <QDir>
+#include <QMutexLocker>
using namespace Baloo;
Database::Database(const QString& path)
: m_path(path)
- , m_env(0)
+ , m_env(nullptr)
{
}
Database::~Database()
{
- mdb_env_close(m_env);
+ // try only to close if we did open the DB successfully
+ if (m_env) {
+ mdb_env_close(m_env);
+ }
}
bool Database::open(OpenMode mode)
{
- if (isOpen()) {
+ QMutexLocker locker(&m_mutex);
+
+ // nop if already open!
+ if (m_env) {
return true;
}
@@ -216,7 +224,14 @@ bool Database::open(OpenMode mode)
return true;
}
+bool Database::isOpen() const
+{
+ QMutexLocker locker(&m_mutex);
+ return m_env != 0;
+}
+
QString Database::path() const
{
+ QMutexLocker locker(&m_mutex);
return m_path;
}
diff --git a/src/engine/database.h b/src/engine/database.h
index e3bb175..4d1f6a1 100644
--- a/src/engine/database.h
+++ b/src/engine/database.h
@@ -1,6 +1,7 @@
/*
This file is part of the KDE Baloo project.
* Copyright (C) 2015 Vishesh Handa <vha...@kde.org>
+ * Copyright (C) 2016 Christoph Cullmann <cullm...@kde.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -21,6 +22,8 @@
#ifndef BALOO_DATABASE_H
#define BALOO_DATABASE_H
+#include <QMutex>
+
#include "document.h"
#include "databasedbis.h"
@@ -31,21 +34,56 @@ class DatabaseTest;
class BALOO_ENGINE_EXPORT Database
{
public:
+ /**
+ * Init database for given DB path, will not open it.
+ * @param path db path
+ */
explicit Database(const QString& path);
- ~Database();
- QString path() const;
+ /**
+ * Destruct db, might close it, if opened.
+ */
+ ~Database();
+ /**
+ * Database open mode
+ */
enum OpenMode {
CreateDatabase,
OpenDatabase
};
+
+ /**
+ * Open database in given mode.
+ * Nop after open was done (even if mode differs).
+ * There is no close as this would invalidate the database for all threads
using it.
+ * @parmm mode create or open only?
+ * @return success?
+ */
bool open(OpenMode mode);
- bool isOpen() const { return m_env != 0; }
+ /**
+ * Is database open?
+ * @return database open?
+ */
+ bool isOpen() const;
+
+ /**
+ * Path to database.
+ * @return database path
+ */
+ QString path() const;
private:
- QString m_path;
+ /**
+ * serialize access, as open might be called from multiple threads
+ */
+ mutable QMutex m_mutex;
+
+ /**
+ * database path
+ */
+ const QString m_path;
MDB_env* m_env;
DatabaseDbis m_dbis;
@@ -56,5 +94,4 @@ private:
};
}
-
#endif // BALOO_DATABASE_H
--
2.10.0
++++++ 0006-fix-typo.patch ++++++
>From b5a93665cf6b44d53abf15302231f6e07f18df85 Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 20:28:04 +0200
Subject: [PATCH 06/13] fix typo
---
src/engine/database.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/engine/database.h b/src/engine/database.h
index 4d1f6a1..6ccb2a5 100644
--- a/src/engine/database.h
+++ b/src/engine/database.h
@@ -57,7 +57,7 @@ public:
* Open database in given mode.
* Nop after open was done (even if mode differs).
* There is no close as this would invalidate the database for all threads
using it.
- * @parmm mode create or open only?
+ * @param mode create or open only?
* @return success?
*/
bool open(OpenMode mode);
--
2.10.0
++++++ 0007-fix-missing-m_env-nullptr-assignment.patch ++++++
>From 988e5feb5de64ed25337fe2ff9b494eb30b15b47 Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 20:39:50 +0200
Subject: [PATCH 07/13] fix missing m_env = nullptr assignment REVIEW: 128890
---
src/engine/database.cpp | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/src/engine/database.cpp b/src/engine/database.cpp
index 8ae6b03..6a433c7 100644
--- a/src/engine/database.cpp
+++ b/src/engine/database.cpp
@@ -59,6 +59,7 @@ Database::~Database()
// try only to close if we did open the DB successfully
if (m_env) {
mdb_env_close(m_env);
+ m_env = nullptr;
}
}
@@ -97,7 +98,7 @@ bool Database::open(OpenMode mode)
int rc = mdb_env_create(&m_env);
if (rc) {
- m_env = 0;
+ m_env = nullptr;
return false;
}
@@ -118,7 +119,7 @@ bool Database::open(OpenMode mode)
QByteArray arr = QFile::encodeName(indexInfo.absoluteFilePath());
rc = mdb_env_open(m_env, arr.constData(), MDB_NOSUBDIR | MDB_NOMEMINIT,
0664);
if (rc) {
- m_env = 0;
+ m_env = nullptr;
return false;
}
@@ -126,6 +127,7 @@ bool Database::open(OpenMode mode)
Q_ASSERT_X(rc == 0, "Database::open reader_check", mdb_strerror(rc));
if (rc) {
mdb_env_close(m_env);
+ m_env = nullptr;
return false;
}
@@ -139,7 +141,7 @@ bool Database::open(OpenMode mode)
if (rc) {
mdb_txn_abort(txn);
mdb_env_close(m_env);
- m_env = 0;
+ m_env = nullptr;
return false;
}
@@ -165,7 +167,7 @@ bool Database::open(OpenMode mode)
if (!m_dbis.isValid()) {
mdb_txn_abort(txn);
mdb_env_close(m_env);
- m_env = 0;
+ m_env = nullptr;
return false;
}
@@ -173,7 +175,7 @@ bool Database::open(OpenMode mode)
Q_ASSERT_X(rc == 0, "Database::transaction ro commit",
mdb_strerror(rc));
if (rc) {
mdb_env_close(m_env);
- m_env = 0;
+ m_env = nullptr;
return false;
}
} else {
@@ -182,7 +184,7 @@ bool Database::open(OpenMode mode)
if (rc) {
mdb_txn_abort(txn);
mdb_env_close(m_env);
- m_env = 0;
+ m_env = nullptr;
return false;
}
@@ -208,7 +210,7 @@ bool Database::open(OpenMode mode)
if (!m_dbis.isValid()) {
mdb_txn_abort(txn);
mdb_env_close(m_env);
- m_env = 0;
+ m_env = nullptr;
return false;
}
@@ -216,11 +218,12 @@ bool Database::open(OpenMode mode)
Q_ASSERT_X(rc == 0, "Database::transaction commit", mdb_strerror(rc));
if (rc) {
mdb_env_close(m_env);
- m_env = 0;
+ m_env = nullptr;
return false;
}
}
+ Q_ASSERT(m_env);
return true;
}
--
2.10.0
++++++ 0008-Transaction-not-created-don-t-try-to-abort-them.patch ++++++
>From 54f7363048c7db41f63c85f637911a5598c30e9e Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 21:10:58 +0200
Subject: [PATCH 08/13] Transaction not created => don't try to abort them
REVIEW: 128891
---
src/engine/database.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/engine/database.cpp b/src/engine/database.cpp
index 6a433c7..37b8e97 100644
--- a/src/engine/database.cpp
+++ b/src/engine/database.cpp
@@ -139,7 +139,6 @@ bool Database::open(OpenMode mode)
int rc = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &txn);
Q_ASSERT_X(rc == 0, "Database::transaction ro begin",
mdb_strerror(rc));
if (rc) {
- mdb_txn_abort(txn);
mdb_env_close(m_env);
m_env = nullptr;
return false;
@@ -182,7 +181,6 @@ bool Database::open(OpenMode mode)
int rc = mdb_txn_begin(m_env, NULL, 0, &txn);
Q_ASSERT_X(rc == 0, "Database::transaction begin", mdb_strerror(rc));
if (rc) {
- mdb_txn_abort(txn);
mdb_env_close(m_env);
m_env = nullptr;
return false;
--
2.10.0
++++++ 0009-add-missing-close-env-as-pointed-out-by-Lo-c-Yhuel.patch ++++++
>From 377e62b0307839edb0245d65381a3f55f594ae4e Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 22:30:18 +0200
Subject: [PATCH 09/13] =?UTF-8?q?add=20missing=20close=20env,=20as=20point?=
=?UTF-8?q?ed=20out=20by=20Lo=C3=AFc=20Yhuel=20Missed=20in=20my=20review?=
=?UTF-8?q?=20diff=20:/?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
REVIEW: 128890
BUG: 353783
---
src/engine/database.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/engine/database.cpp b/src/engine/database.cpp
index 37b8e97..c83f540 100644
--- a/src/engine/database.cpp
+++ b/src/engine/database.cpp
@@ -119,6 +119,7 @@ bool Database::open(OpenMode mode)
QByteArray arr = QFile::encodeName(indexInfo.absoluteFilePath());
rc = mdb_env_open(m_env, arr.constData(), MDB_NOSUBDIR | MDB_NOMEMINIT,
0664);
if (rc) {
+ mdb_env_close(m_env);
m_env = nullptr;
return false;
}
--
2.10.0
++++++ 0010-Fix-sorted-insert-aka-flat_map-like-insert.patch ++++++
>From 6e5b41e88d92c90df8e54d99163cea08f17d0554 Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 22:37:40 +0200
Subject: [PATCH 10/13] Fix sorted insert (aka flat_map like insert).
REVIEW: 128893
BUG: 367991
unit test covers more cases, in any case: should not corrupt your memory ;=)
---
autotests/unit/engine/documenturldbtest.cpp | 47 +++++++++++++++++++++++++++++
src/engine/documenturldb.cpp | 14 ++-------
src/engine/idutils.h | 27 +++++++++++++++++
src/engine/writetransaction.cpp | 34 +++------------------
4 files changed, 81 insertions(+), 41 deletions(-)
diff --git a/autotests/unit/engine/documenturldbtest.cpp
b/autotests/unit/engine/documenturldbtest.cpp
index 448821b..db919b8 100644
--- a/autotests/unit/engine/documenturldbtest.cpp
+++ b/autotests/unit/engine/documenturldbtest.cpp
@@ -147,6 +147,53 @@ private Q_SLOTS:
QCOMPARE(db.getId(id, QByteArray("file")), id1);
QCOMPARE(db.getId(id, QByteArray("file2")), id2);
}
+
+ void testSortedIdInsert()
+ {
+ // test sorted insert used in Baloo::DocumentUrlDB::add, bug 367991
+ std::vector<quint64> test;
+ test.push_back(9);
+
+ // shall not crash
+ sortedIdInsert(test, quint64(1));
+
+ // stuff shall be ok inserted
+ QVERIFY(test.size() == 2);
+ QVERIFY(test[0] == 1);
+ QVERIFY(test[1] == 9);
+
+ // shall not crash
+ sortedIdInsert(test, quint64(1));
+
+ // no insert please
+ QVERIFY(test.size() == 2);
+
+ // shall not crash
+ sortedIdInsert(test, quint64(10));
+
+ // stuff shall be ok inserted
+ QVERIFY(test.size() == 3);
+ QVERIFY(test[0] == 1);
+ QVERIFY(test[1] == 9);
+ QVERIFY(test[2] == 10);
+
+ // shall not crash
+ sortedIdInsert(test, quint64(2));
+
+ // stuff shall be ok inserted
+ QVERIFY(test.size() == 4);
+ QVERIFY(test[0] == 1);
+ QVERIFY(test[1] == 2);
+ QVERIFY(test[2] == 9);
+ QVERIFY(test[3] == 10);
+
+ // shall not crash
+ sortedIdInsert(test, quint64(2));
+
+ // no insert please
+ QVERIFY(test.size() == 4);
+ }
+
protected:
MDB_env* m_env;
MDB_txn* m_txn;
diff --git a/src/engine/documenturldb.cpp b/src/engine/documenturldb.cpp
index 5083e7a..ef2a22c 100644
--- a/src/engine/documenturldb.cpp
+++ b/src/engine/documenturldb.cpp
@@ -112,18 +112,8 @@ void DocumentUrlDB::add(quint64 id, quint64 parentId,
const QByteArray& name)
QVector<quint64> subDocs = idTreeDb.get(parentId);
- // Find if the id exists
- if (subDocs.isEmpty()) {
- subDocs.append(id);
- } else {
- auto it = std::upper_bound(subDocs.begin(), subDocs.end(), id);
-
- // Merge the id if it does not
- auto prev = it - 1;
- if (*prev != id) {
- subDocs.insert(it, id);
- }
- }
+ // insert if not there
+ sortedIdInsert(subDocs, id);
idTreeDb.put(parentId, subDocs);
diff --git a/src/engine/idutils.h b/src/engine/idutils.h
index cc7da9c..e1d513f 100644
--- a/src/engine/idutils.h
+++ b/src/engine/idutils.h
@@ -85,6 +85,33 @@ inline quint32 idToDeviceId(quint64 id)
return arr[0];
}
+
+template<typename T, typename V>
+inline void sortedIdInsert(T& vec, const V& id)
+{
+ /**
+ * search with normal <
+ */
+ const auto i(std::lower_bound(vec.begin(), vec.end(), id));
+
+ /**
+ * end reached or element found smaller?
+ * => insert new element!
+ */
+ if (i == vec.end() || (id != *i))
+ vec.insert(i, id);
+}
+
+template<typename T, typename V>
+inline void sortedIdRemove(T& vec, const V& id)
+{
+ const int idx = vec.indexOf(id);
+ if (idx >= 0) {
+ vec.remove(idx);
+ }
+}
+
+
}
#endif
diff --git a/src/engine/writetransaction.cpp b/src/engine/writetransaction.cpp
index 3808970..171f5ba 100644
--- a/src/engine/writetransaction.cpp
+++ b/src/engine/writetransaction.cpp
@@ -30,6 +30,7 @@
#include "documenttimedb.h"
#include "documentdatadb.h"
#include "mtimedb.h"
+#include "idutils.h"
using namespace Baloo;
@@ -243,31 +244,6 @@ QVector< QByteArray >
WriteTransaction::replaceTerms(quint64 id, const QVector<Q
return addTerms(id, terms);
}
-template<typename T>
-static void insert(QVector<T>& vec, const T& id)
-{
- if (vec.isEmpty()) {
- vec.append(id);
- } else {
- auto it = std::upper_bound(vec.begin(), vec.end(), id);
-
- // Merge the id if it does not
- auto prev = it - 1;
- if (*prev != id) {
- vec.insert(it, id);
- }
- }
-}
-
-template<typename T>
-static void removeOne(QVector<T>& vec, const T& id)
-{
- const int idx = vec.indexOf(id);
- if (idx >= 0) {
- vec.remove(idx);
- }
-}
-
void WriteTransaction::commit()
{
PostingDB postingDB(m_dbis.postingDbi, m_txn);
@@ -289,23 +265,23 @@ void WriteTransaction::commit()
quint64 id = op.data.docId;
if (op.type == AddId) {
- insert(list, id);
+ sortedIdInsert(list, id);
if (!op.data.positions.isEmpty()) {
if (!fetchedPositionList) {
positionList = positionDB.get(term);
fetchedPositionList = true;
}
- insert(positionList, op.data);
+ sortedIdInsert(positionList, op.data);
}
}
else {
- removeOne(list, id);
+ sortedIdRemove(list, id);
if (!fetchedPositionList) {
positionList = positionDB.get(term);
fetchedPositionList = true;
}
- removeOne(positionList, PositionInfo(id));
+ sortedIdRemove(positionList, PositionInfo(id));
}
}
--
2.10.0
++++++ 0011-Open-baloo-lmdb-database-read-only-beside-in-baloo_f.patch ++++++
>From 02047b524a176da447d8c96e15c7e2abae8339ae Mon Sep 17 00:00:00 2001
From: Christoph Cullmann <cullm...@kde.org>
Date: Sun, 11 Sep 2016 23:36:27 +0200
Subject: [PATCH 11/13] Open baloo lmdb database read-only beside in
baloo_file/baloo_file_extractor + balooctl (for some commands) + unit tests
At the moment, any application that uses baloo can corrupt the db.
Now, only the things that need to write to it open it with read-write.
This only works as long as the library exposes only read-only things like
Query/...
REVIEW: 128892
---
src/engine/database.cpp | 6 +++---
src/engine/database.h | 14 +++++++++++++-
src/file/extractor/app.cpp | 2 +-
src/lib/file.cpp | 2 +-
src/lib/searchstore.cpp | 2 +-
src/lib/taglistjob.cpp | 2 +-
src/qml/experimental/monitor.cpp | 2 +-
src/tools/balooctl/main.cpp | 8 ++++----
src/tools/balooctl/statuscommand.cpp | 2 +-
src/tools/balooshow/main.cpp | 2 +-
10 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/src/engine/database.cpp b/src/engine/database.cpp
index c83f540..0c91d0e 100644
--- a/src/engine/database.cpp
+++ b/src/engine/database.cpp
@@ -79,7 +79,7 @@ bool Database::open(OpenMode mode)
}
QFileInfo indexInfo(dir, QStringLiteral("index"));
- if (mode == OpenDatabase && !indexInfo.exists()) {
+ if ((mode != CreateDatabase) && !indexInfo.exists()) {
return false;
}
@@ -117,7 +117,7 @@ bool Database::open(OpenMode mode)
// The directory needs to be created before opening the environment
QByteArray arr = QFile::encodeName(indexInfo.absoluteFilePath());
- rc = mdb_env_open(m_env, arr.constData(), MDB_NOSUBDIR | MDB_NOMEMINIT,
0664);
+ rc = mdb_env_open(m_env, arr.constData(), MDB_NOSUBDIR | MDB_NOMEMINIT |
((mode == ReadOnlyDatabase) ? MDB_RDONLY : 0), 0664);
if (rc) {
mdb_env_close(m_env);
m_env = nullptr;
@@ -136,7 +136,7 @@ bool Database::open(OpenMode mode)
// Individual Databases
//
MDB_txn* txn;
- if (mode == OpenDatabase) {
+ if (mode != CreateDatabase) {
int rc = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &txn);
Q_ASSERT_X(rc == 0, "Database::transaction ro begin",
mdb_strerror(rc));
if (rc) {
diff --git a/src/engine/database.h b/src/engine/database.h
index 6ccb2a5..735ecd5 100644
--- a/src/engine/database.h
+++ b/src/engine/database.h
@@ -49,8 +49,20 @@ public:
* Database open mode
*/
enum OpenMode {
+ /**
+ * Create + open read-write dabase.
+ */
CreateDatabase,
- OpenDatabase
+
+ /**
+ * Read-Write Database, only works if database exists.
+ */
+ ReadWriteDatabase,
+
+ /**
+ * Read-Only Database, only works if database exists.
+ */
+ ReadOnlyDatabase
};
/**
diff --git a/src/file/extractor/app.cpp b/src/file/extractor/app.cpp
index 0ca7276..6843165 100644
--- a/src/file/extractor/app.cpp
+++ b/src/file/extractor/app.cpp
@@ -55,7 +55,7 @@ App::App(QObject* parent)
void App::slotNewInput()
{
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadWriteDatabase)) {
qCritical() << "Failed to open the database";
exit(1);
}
diff --git a/src/lib/file.cpp b/src/lib/file.cpp
index cbbc912..1176f35 100644
--- a/src/lib/file.cpp
+++ b/src/lib/file.cpp
@@ -96,7 +96,7 @@ bool File::load()
}
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadOnlyDatabase)) {
return false;
}
diff --git a/src/lib/searchstore.cpp b/src/lib/searchstore.cpp
index 060a4fd..265016b 100644
--- a/src/lib/searchstore.cpp
+++ b/src/lib/searchstore.cpp
@@ -48,7 +48,7 @@ SearchStore::SearchStore()
: m_db(0)
{
m_db = globalDatabaseInstance();
- if (!m_db->open(Database::OpenDatabase)) {
+ if (!m_db->open(Database::ReadOnlyDatabase)) {
m_db = 0;
}
diff --git a/src/lib/taglistjob.cpp b/src/lib/taglistjob.cpp
index 76ac8ff..3d1b0c8 100644
--- a/src/lib/taglistjob.cpp
+++ b/src/lib/taglistjob.cpp
@@ -46,7 +46,7 @@ TagListJob::~TagListJob()
void TagListJob::start()
{
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadOnlyDatabase)) {
setError(UserDefinedError);
setErrorText(QStringLiteral("Failed to open the database"));
emitResult();
diff --git a/src/qml/experimental/monitor.cpp b/src/qml/experimental/monitor.cpp
index 11c06ae..679b914 100644
--- a/src/qml/experimental/monitor.cpp
+++ b/src/qml/experimental/monitor.cpp
@@ -126,7 +126,7 @@ void Monitor::balooStarted(const QString& service)
void Monitor::fetchTotalFiles()
{
Baloo::Database *db = Baloo::globalDatabaseInstance();
- if (db->open(Baloo::Database::OpenDatabase)) {
+ if (db->open(Baloo::Database::ReadOnlyDatabase)) {
Baloo::Transaction tr(db, Baloo::Transaction::ReadOnly);
m_totalFiles = tr.size();
m_filesIndexed = tr.size() - tr.phaseOneSize();
diff --git a/src/tools/balooctl/main.cpp b/src/tools/balooctl/main.cpp
index 2a6b175..b12b39d 100644
--- a/src/tools/balooctl/main.cpp
+++ b/src/tools/balooctl/main.cpp
@@ -191,7 +191,7 @@ int main(int argc, char* argv[])
}
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadWriteDatabase)) {
out << "Baloo Index could not be opened\n";
return 1;
}
@@ -230,7 +230,7 @@ int main(int argc, char* argv[])
}
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadWriteDatabase)) {
out << "Baloo Index could not be opened\n";
return 1;
}
@@ -260,7 +260,7 @@ int main(int argc, char* argv[])
if (command == QStringLiteral("indexSize")) {
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadOnlyDatabase)) {
out << "Baloo Index could not be opened\n";
return 1;
}
@@ -311,7 +311,7 @@ int main(int argc, char* argv[])
if (command == QStringLiteral("checkDb")) {
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadOnlyDatabase)) {
out << "Baloo Index could not be opened\n";
return 1;
}
diff --git a/src/tools/balooctl/statuscommand.cpp
b/src/tools/balooctl/statuscommand.cpp
index 1a56c64..fede101 100644
--- a/src/tools/balooctl/statuscommand.cpp
+++ b/src/tools/balooctl/statuscommand.cpp
@@ -56,7 +56,7 @@ int StatusCommand::exec(const QCommandLineParser& parser)
}
Database *db = globalDatabaseInstance();
- if (!db->open(Database::OpenDatabase)) {
+ if (!db->open(Database::ReadOnlyDatabase)) {
out << i18n("Baloo Index could not be opened") << endl;
return 1;
}
diff --git a/src/tools/balooshow/main.cpp b/src/tools/balooshow/main.cpp
index f45f2e0..54c64df 100644
--- a/src/tools/balooshow/main.cpp
+++ b/src/tools/balooshow/main.cpp
@@ -101,7 +101,7 @@ int main(int argc, char* argv[])
QString text;
Baloo::Database *db = Baloo::globalDatabaseInstance();
- if (!db->open(Baloo::Database::OpenDatabase)) {
+ if (!db->open(Baloo::Database::ReadOnlyDatabase)) {
stream << i18n("The Baloo index could not be opened. Please run
\"balooctl status\" to see if Baloo is enabled and working.")
<< endl;
return 1;
--
2.10.0
++++++ 0012-Use-correct-config-entry-in-autostart-condition.patch ++++++
>From e73849e052c1a2ca174efe18d4a4958c6576fc33 Mon Sep 17 00:00:00 2001
From: David Edmundson <k...@davidedmundson.co.uk>
Date: Wed, 28 Sep 2016 12:41:03 +0100
Subject: [PATCH 12/13] Use correct config entry in autostart condition
Balooctl and the KCMs write the config entry "Indexing-Enabled" not just
"Enabled"
This saves baloo launching when it is only going to immediately quit
again anyway.
REVIEW: 129059
---
src/file/baloo_file.desktop | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/file/baloo_file.desktop b/src/file/baloo_file.desktop
index d79000c..7f73500 100644
--- a/src/file/baloo_file.desktop
+++ b/src/file/baloo_file.desktop
@@ -2,7 +2,7 @@
Type=Service
Exec=baloo_file
X-KDE-StartupNotify=false
-X-KDE-autostart-condition=baloofilerc:Basic Settings:Enabled:true
+X-KDE-autostart-condition=baloofilerc:Basic Settings:Indexing-Enabled:true
X-KDE-autostart-phase=0
X-GNOME-Autostart-enabled=true
OnlyShowIn=KDE;GNOME;Unity;XFCE
--
2.10.0
