Hello community, here is the log from the commit of package kdepim4-runtime for openSUSE:Factory checked in at 2012-02-22 15:53:06 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/kdepim4-runtime (Old) and /work/SRC/openSUSE:Factory/.kdepim4-runtime.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "kdepim4-runtime", Maintainer is "[email protected]" Changes: -------- --- /work/SRC/openSUSE:Factory/kdepim4-runtime/kdepim4-runtime.changes 2012-01-27 15:20:52.000000000 +0100 +++ /work/SRC/openSUSE:Factory/.kdepim4-runtime.new/kdepim4-runtime.changes 2012-02-22 15:53:09.000000000 +0100 @@ -1,0 +2,7 @@ +Sun Feb 19 19:09:04 UTC 2012 - [email protected] + +- Add trueg's patch to fix virtuoso cpu thrashing and generally + make the indexing feeders behave better. + * see https://bugs.kde.org/show_bug.cgi?id=289932 + +------------------------------------------------------------------- New: ---- kdepim-runtime-nepomuk-feeder-fix.diff ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ kdepim4-runtime.spec ++++++ --- /var/tmp/diff_new_pack.wh2Crj/_old 2012-02-22 15:53:10.000000000 +0100 +++ /var/tmp/diff_new_pack.wh2Crj/_new 2012-02-22 15:53:10.000000000 +0100 @@ -15,6 +15,8 @@ # Please submit bugfixes or comments via http://bugs.opensuse.org/ # + + Name: kdepim4-runtime Version: 4.8.0 Release: 0 @@ -26,6 +28,7 @@ Source1: akonadi.png Patch0: 4_8_BRANCH.diff Patch1: disable-knut.diff +Patch2: kdepim-runtime-nepomuk-feeder-fix.diff BuildRequires: fdupes BuildRequires: libkdepimlibs4-devel Suggests: kontact @@ -45,6 +48,7 @@ %setup -q -n kdepim-runtime-%{version} %patch0 -p1 %patch1 +%patch2 -p1 %build %ifarch ppc64 ++++++ kdepim-runtime-nepomuk-feeder-fix.diff ++++++ diff --git a/agents/nepomukfeeder/CMakeLists.txt b/agents/nepomukfeeder/CMakeLists.txt index 82ea7c9..07ffbdf 100644 --- a/agents/nepomukfeeder/CMakeLists.txt +++ b/agents/nepomukfeeder/CMakeLists.txt @@ -5,6 +5,7 @@ include_directories( dms-copy plugin ./ + ${CMAKE_CURRENT_BINARY_DIR} ) include( SopranoAddOntology ) @@ -17,6 +18,9 @@ add_subdirectory( plugin ) add_subdirectory( plugins ) add_subdirectory( test ) +set(CMAKE_NEPOMUK_FEEDER_INDEX_COMPAT_LEVEL 3) +configure_file(nepomukfeeder-config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/nepomukfeeder-config.h) + kde4_add_executable(akonadi_nepomuk_feeder nepomukfeederagent.cpp pluginloader.cpp feederqueue.cpp nepomukhelpers.cpp) set( NEPOMUKFEEDER_SHARED_LIBS diff --git a/agents/nepomukfeeder/feederqueue.cpp b/agents/nepomukfeeder/feederqueue.cpp index 9274daf..f19fdd1 100644 --- a/agents/nepomukfeeder/feederqueue.cpp +++ b/agents/nepomukfeeder/feederqueue.cpp @@ -23,6 +23,7 @@ #include "dms-copy/simpleresource.h" #include <Nepomuk/ResourceManager> #include <Soprano/Model> +#include <Soprano/QueryResultIterator> #include <nie.h> #include <Akonadi/ItemFetchJob> @@ -31,9 +32,12 @@ #include <KUrl> #include <KJob> +#include <QDateTime> + #include <aneo.h> #include "nepomukhelpers.h" +#include "nepomukfeeder-config.h" using namespace Akonadi; @@ -72,7 +76,34 @@ void FeederQueue::setOnline( bool online ) //kDebug() << online; mOnline = online; if (online) - continueIndexing(); + continueIndexing(); +} + +void FeederQueue::setIndexingSpeed(FeederQueue::IndexingSpeed speed) +{ + const int s_reducedSpeedDelay = 500; // ms + const int s_snailPaceDelay = 3000; // ms + + kDebug() << speed; + + // + // The high priority queue is never suspended entirely, even if we are offline. + // The low prio queue is always throttled a little more than the high prio one + // + if ( speed == FullSpeed ) { + lowPrioQueue.setProcessingDelay(0); + highPrioQueue.setProcessingDelay(0); + setOnline(true); + } + else if ( speed == ReducedSpeed ) { + lowPrioQueue.setProcessingDelay(s_snailPaceDelay); + highPrioQueue.setProcessingDelay(s_reducedSpeedDelay); + setOnline(true); + } + else if ( speed == SnailPace ) { + highPrioQueue.setProcessingDelay(s_snailPaceDelay); + setOnline(false); + } } void FeederQueue::addCollection( const Akonadi::Collection &collection ) @@ -122,16 +153,24 @@ void FeederQueue::itemHeadersReceived( const Akonadi::Item::List& items ) if ( item.storageCollectionId() != mCurrentCollection.id() ) continue; // stay away from links - // update item if it does not exist - if ( !Nepomuk::ResourceManager::instance()->mainModel()->containsAnyStatement( Soprano::Node(), Vocabulary::NIE::url(), item.url() ) ) { + // update item if it does not exist or does not have a proper id + // we check if the item already exists with the following values: + // - nie:url needs to be set + // - aneo:akonadiItemId needs to be set + // - nie:lastModified needs to match the item's modification time + // - aneo:akonadiIndexCompatLevel needs to match the indexer's level + if ( mReIndex || + !Nepomuk::ResourceManager::instance()->mainModel()->executeQuery(QString::fromLatin1("ask where { ?r %1 %2 ; %3 %4 ; %5 %6 ; %7 %8 . }") + .arg(Soprano::Node::resourceToN3(Vocabulary::NIE::url()), + Soprano::Node::resourceToN3(item.url()), + Soprano::Node::resourceToN3(Vocabulary::ANEO::akonadiItemId()), + Soprano::Node::literalToN3(QString::number(item.id())), + Soprano::Node::resourceToN3(Vocabulary::NIE::lastModified()), + Soprano::Node::literalToN3(item.modificationTime()), + Soprano::Node::resourceToN3(Vocabulary::ANEO::akonadiIndexCompatLevel()), + Soprano::Node::literalToN3(NEPOMUK_FEEDER_INDEX_COMPAT_LEVEL)), + Soprano::Query::QueryLanguageSparql).boolValue() ) { itemsToUpdate.append( item ); - - // the item exists. Check if it has an item ID property, otherwise re-index it. - } else { //TODO maybe reindex anyways to be sure that type etc is correct - if ( !Nepomuk::ResourceManager::instance()->mainModel()->containsAnyStatement( Soprano::Node(), - Vocabulary::ANEO::akonadiItemId(), Soprano::LiteralValue( QUrl( QString::number( item.id() ) ) ) ) || mReIndex ) { - itemsToUpdate.append( item ); - } } } @@ -183,15 +222,15 @@ void FeederQueue::processItemQueue() if ( (mProcessedAmount % 100) == 0 && mTotalAmount > 0 && mProcessedAmount <= mTotalAmount ) emit progress( (mProcessedAmount * 100) / mTotalAmount ); - if ( !highPrioQueue.isEmpty() ) { + if ( !mOnline ) { + kDebug() << "not Online, stopping processing"; + return; + } else if ( !highPrioQueue.isEmpty() ) { //kDebug() << "high"; if (!highPrioQueue.processItem()) { return; } - } else if ( !mOnline ) { - kDebug() << "not Online, stopping processing"; - return; - } else if ( !lowPrioQueue.isEmpty() ){ + } else if ( !lowPrioQueue.isEmpty() ){ //kDebug() << "low"; if (!lowPrioQueue.processItem()) { return; @@ -254,7 +293,8 @@ ItemQueue::ItemQueue(int batchSize, int fetchSize, QObject* parent) mPendingRemoveDataJobs( 0 ), mBatchSize( batchSize ), mFetchSize( fetchSize ), - mRunningJobs( 0 ) + mRunningJobs( 0 ), + mProcessingDelay( 0 ) { if ( fetchSize < batchSize ) { kWarning() << "fetchSize must be >= batchsize"; @@ -384,15 +424,13 @@ void ItemQueue::batchJobResult(KJob* job) //kDebug() << "pipline size: " << mItemPipeline.size(); //kDebug() << "fetchedItemList : " << mFetchedItemList.size(); Q_ASSERT(mBatch.isEmpty()); - int timeout = 0; if ( job->error() ) { /*foreach( const Nepomuk::SimpleResource &res, m_debugGraph.toList() ) { kWarning() << res; }*/ kWarning() << job->errorString(); - timeout = 0; //This timeout is here in case nepomuk is still processing and the dbus connection just timed out (to avoid just adding more work). Since we have now a huge dbus timeout that timeout is probably not needed anymore. } - QTimer::singleShot(timeout, this, SLOT(continueProcessing())); + QTimer::singleShot(mProcessingDelay, this, SLOT(continueProcessing())); mRunningJobs++; } @@ -415,7 +453,12 @@ void ItemQueue::continueProcessing() bool ItemQueue::isEmpty() { - return mItemPipeline.isEmpty() && mFetchedItemList.isEmpty(); + return mItemPipeline.isEmpty() && mFetchedItemList.isEmpty(); +} + +void ItemQueue::setProcessingDelay(int ms) +{ + mProcessingDelay = ms; } diff --git a/agents/nepomukfeeder/feederqueue.h b/agents/nepomukfeeder/feederqueue.h index ecbd8b7..13d4afc 100644 --- a/agents/nepomukfeeder/feederqueue.h +++ b/agents/nepomukfeeder/feederqueue.h @@ -52,6 +52,10 @@ public: bool processItem(); /** queue is empty */ bool isEmpty(); + + /** the delay between two batches */ + void setProcessingDelay(int ms); + signals: /** all items processed */ void finished(); @@ -79,6 +83,8 @@ private: int mBatchSize; //Size of Nepomuk batch, number of items stored together in nepomuk int mFetchSize; //Maximum number of items fetched with full payload (defines ram usage of feeder), must be >= mBatchSize, ideally a multiple of it int mRunningJobs; + + int mProcessingDelay; }; /** @@ -128,6 +134,34 @@ public: /** start/stop indexing */ void setOnline(bool); + enum IndexingSpeed { + /** + * Index at full speed, i.e. do not use any artificial + * delays. + * + * This is the mode used if the user is "away". + */ + FullSpeed = 0, + + /** + * Reduce the indexing speed mildly. This is the normal + * mode used while the user works. The indexer uses small + * delay between indexing two batches in order to keep the + * load on CPU and IO down. + */ + ReducedSpeed, + + /** + * Like ReducedSpeed delays are used but they are much longer + * to get even less CPU and IO load. This mode is used for the + * first 2 minutes after startup to give the KDE session manager + * time to start up the KDE session rapidly. + */ + SnailPace + }; + + void setIndexingSpeed( IndexingSpeed speed ); + signals: void fullyIndexed(); void progress(int); diff --git a/agents/nepomukfeeder/nepomukfeeder-config.h.cmake b/agents/nepomukfeeder/nepomukfeeder-config.h.cmake new file mode 100644 index 0000000..dcea476 --- /dev/null +++ b/agents/nepomukfeeder/nepomukfeeder-config.h.cmake @@ -0,0 +1,27 @@ +/* + This file is part of the Nepomuk KDE project. + Copyright (C) 2012 Sebastian Trueg <[email protected]> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) version 3, or any + later version accepted by the membership of KDE e.V. (or its + successor approved by the membership of KDE e.V.), which shall + act as a proxy defined in Section 6 of version 3 of the license. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library. If not, see <http://www.gnu.org/licenses/>. +*/ + +#ifndef NEPOMUK_FEEDER_CONFIG_H +#define NEPOMUK_FEEDER_CONFIG_H + +#define NEPOMUK_FEEDER_INDEX_COMPAT_LEVEL ${CMAKE_NEPOMUK_FEEDER_INDEX_COMPAT_LEVEL} + +#endif diff --git a/agents/nepomukfeeder/nepomukfeederagent.cpp b/agents/nepomukfeeder/nepomukfeederagent.cpp index edb3e29..a3be04b 100644 --- a/agents/nepomukfeeder/nepomukfeederagent.cpp +++ b/agents/nepomukfeeder/nepomukfeederagent.cpp @@ -53,6 +53,7 @@ #include <nepomukfeederutils.h> #include "pluginloader.h" #include "nepomukhelpers.h" +#include "nepomukfeeder-config.h" namespace Akonadi { @@ -73,7 +74,6 @@ static inline bool indexingDisabled( const Collection &collection ) NepomukFeederAgent::NepomukFeederAgent(const QString& id) : AgentBase(id), - mIndexCompatLevel( 3 ), mNepomukStartupAttempted( false ), mInitialUpdateDone( false ), mSelfTestPassed( false ), @@ -101,7 +101,8 @@ NepomukFeederAgent::NepomukFeederAgent(const QString& id) : connect( KIdleTime::instance(), SIGNAL(timeoutReached(int)), SLOT(systemIdle()) ); connect( KIdleTime::instance(), SIGNAL(resumingFromIdle()), SLOT(systemResumed()) ); - KIdleTime::instance()->addIdleTimeout( 10 * 1000 ); + KIdleTime::instance()->addIdleTimeout( 2 * 60 * 1000 ); // only go full-speed after 2 min + KConfig config( "akonadi_nepomuk_feederrc" ); KConfigGroup cfgGrp( &config, identifier() ); kDebug() << "DisableIdleDetection: " << cfgGrp.readEntry( "DisableIdleDetection", false ); @@ -112,6 +113,7 @@ NepomukFeederAgent::NepomukFeederAgent(const QString& id) : QTimer::singleShot( 0, this, SLOT(selfTest()) ); mQueue.setItemFetchScope(changeRecorder()->itemFetchScope()); + mQueue.setIndexingSpeed( FeederQueue::ReducedSpeed ); connect(&mQueue, SIGNAL(progress(int)), SIGNAL(percent(int))); connect(&mQueue, SIGNAL(idle(QString)), this, SLOT(idle(QString))); @@ -126,7 +128,7 @@ NepomukFeederAgent::~NepomukFeederAgent() void NepomukFeederAgent::itemAdded(const Akonadi::Item& item, const Akonadi::Collection& collection) { - //kDebug() << item.id(); + kDebug() << item.id(); if ( indexingDisabled( collection ) ) return; @@ -136,7 +138,7 @@ void NepomukFeederAgent::itemAdded(const Akonadi::Item& item, const Akonadi::Col void NepomukFeederAgent::itemChanged(const Akonadi::Item& item, const QSet< QByteArray >& partIdentifiers) { - //kDebug() << item.id(); + kDebug() << item.id(); Q_UNUSED( partIdentifiers ); if ( indexingDisabled( item.parentCollection() ) ) return; @@ -173,7 +175,6 @@ void NepomukFeederAgent::collectionRemoved(const Akonadi::Collection& collection void NepomukFeederAgent::updateAll() { - mQueue.setReindexing(true); CollectionFetchJob *collectionFetch = new CollectionFetchJob( Collection::root(), CollectionFetchJob::Recursive, this ); connect( collectionFetch, SIGNAL(collectionsReceived(Akonadi::Collection::List)), SLOT(collectionsReceived(Akonadi::Collection::List)) ); } @@ -235,8 +236,14 @@ void NepomukFeederAgent::selfTest() mNepomukStartupAttempted = false; // everything worked, we can try again if the server goes down later mNepomukStartupTimeout.stop(); checkOnline(); - if ( !mInitialUpdateDone && needsReIndexing() ) { + const KConfigGroup grp( componentData().config(), "InitialIndexing" ); + const bool initialUpdateComplete = grp.readEntry( "InitialIndexingComplete", false ); + const int indexCompatLevelIncreased = grp.readEntry( "IndexCompatLevel", 0 ) < NEPOMUK_FEEDER_INDEX_COMPAT_LEVEL; + if ( !mInitialUpdateDone && ( !initialUpdateComplete || indexCompatLevelIncreased ) ) { mInitialUpdateDone = true; + // we actually never need to reindex everything in normal operation + // we leave the setting in anyway in case we ever introduce a manual override or whatever + mQueue.setReindexing(false); QTimer::singleShot( 0, this, SLOT(updateAll()) ); } else { emit status( Idle, i18n( "Ready to index data." ) ); @@ -258,16 +265,11 @@ void NepomukFeederAgent::disableIdleDetection( bool value ) } } -bool NepomukFeederAgent::needsReIndexing() const -{ - const KConfigGroup grp( componentData().config(), "InitialIndexing" ); - return mIndexCompatLevel > grp.readEntry( "IndexCompatLevel", 0 ); -} - void NepomukFeederAgent::slotFullyIndexed() { KConfigGroup grp( componentData().config(), "InitialIndexing" ); - grp.writeEntry( "IndexCompatLevel", mIndexCompatLevel ); + grp.writeEntry( "InitialIndexingComplete", true ); + grp.writeEntry( "IndexCompatLevel", NEPOMUK_FEEDER_INDEX_COMPAT_LEVEL ); grp.sync(); } @@ -307,7 +309,7 @@ void NepomukFeederAgent::systemIdle() emit status( Idle, i18n( "System idle, ready to index data." ) ); mSystemIsIdle = true; KIdleTime::instance()->catchNextResumeEvent(); - setRunning( mSystemIsIdle ); + mQueue.setIndexingSpeed( FeederQueue::FullSpeed ); } void NepomukFeederAgent::systemResumed() @@ -317,7 +319,7 @@ void NepomukFeederAgent::systemResumed() emit status( Idle, i18n( "System busy, indexing suspended." ) ); mSystemIsIdle = false; - setRunning( mSystemIsIdle ); + mQueue.setIndexingSpeed( FeederQueue::ReducedSpeed ); } void NepomukFeederAgent::idle(const QString &string) diff --git a/agents/nepomukfeeder/nepomukfeederagent.h b/agents/nepomukfeeder/nepomukfeederagent.h index 94ff1d2..e1390fc 100644 --- a/agents/nepomukfeeder/nepomukfeederagent.h +++ b/agents/nepomukfeeder/nepomukfeederagent.h @@ -107,13 +107,6 @@ class NepomukFeederAgent : public Akonadi::AgentBase, public Akonadi::AgentBase: void doSetOnline(bool online); private: - - /** - Overrides in subclasses to cause re-indexing on startup to only happen - when the format changes, for example. Base implementation checks the index compatibility level. - */ - virtual bool needsReIndexing() const; - void checkOnline(); void setRunning( bool running ); @@ -128,7 +121,6 @@ class NepomukFeederAgent : public Akonadi::AgentBase, public Akonadi::AgentBase: private: QTimer mNepomukStartupTimeout; - int mIndexCompatLevel; bool mNepomukStartupAttempted; bool mInitialUpdateDone; bool mSelfTestPassed; diff --git a/agents/nepomukfeeder/nepomukhelpers.cpp b/agents/nepomukfeeder/nepomukhelpers.cpp index 294516c..89d05fd 100644 --- a/agents/nepomukfeeder/nepomukhelpers.cpp +++ b/agents/nepomukfeeder/nepomukhelpers.cpp @@ -34,10 +34,13 @@ #include <KJob> +#include <QDateTime> + #include <nepomukfeederplugin.h> #include <nepomukfeederutils.h> #include "pluginloader.h" +#include "nepomukfeeder-config.h" using namespace Nepomuk::Vocabulary; @@ -99,6 +102,8 @@ void addItemToGraph( const Akonadi::Item &item, Nepomuk::SimpleResourceGraph &gr Nepomuk::SimpleResource res; res.setTypes(QList <QUrl>() << Vocabulary::ANEO::AkonadiDataObject() << NIE::InformationElement()); res.setProperty( NIE::url(), QUrl(item.url()) ); + res.setProperty( NIE::lastModified(), item.modificationTime() ); + res.setProperty( Vocabulary::ANEO::akonadiIndexCompatLevel(), NEPOMUK_FEEDER_INDEX_COMPAT_LEVEL ); Q_ASSERT(res.property( NIE::url() ).first().toUrl() == QUrl(item.url())); res.setProperty( Vocabulary::ANEO::akonadiItemId(), QString::number( item.id() ) ); setParentCollection( item, res, graph); @@ -127,4 +132,4 @@ KJob *addGraphToNepomuk( const Nepomuk::SimpleResourceGraph &graph ) } -} \ No newline at end of file +} diff --git a/agents/ontologies/aneo.trig b/agents/ontologies/aneo.trig index f19d068..54eaaf8 100644 --- a/agents/ontologies/aneo.trig +++ b/agents/ontologies/aneo.trig @@ -1,5 +1,6 @@ # # Copyright (c) 2011 Christian Mollekopf <[email protected]> +# Copyright (c) 2012 Sebastian Trueg <[email protected]> # All rights reserved, licensed under either CC-BY or BSD. # # You are free: @@ -59,6 +60,14 @@ aneo: { rdfs:subPropertyOf nao:identifier ; nrl:maxCardinality 1 ; nao:userVisible false . + + aneo:akonadiIndexCompatLevel a rdf:Property ; + rdfs:label "Akonadi index compat level" ; + rdfs:comment "Saves the version of the indexer used to write this item." ; + rdfs:range xsd:int ; + rdfs:domain aneo:AkonadiDataObject ; + nrl:cardinality 1 ; + nao:userVisible false . } @@ -68,7 +77,7 @@ aneo: { nao:prefLabel "Akonadi Ontology" ; nao:hasDefaultNamespace "http://akonadi-project.org/ontologies/aneo#"; ; nao:hasDefaultNamespaceAbbreviation "aneo" ; - nao:lastModified "2011-06-23T14:31:43Z" ; + nao:lastModified "2012-02-15T09:56:43Z" ; nao:serializationLanguage "TriG" ; nao:status "Unstable" ; nrl:updatable "0" ; -- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
