Author: sevein
Date: Fri May 13 16:05:55 2011
New Revision: 9098
Log:
Restart index given a position (index_segfault_r9036.patch), see issue 1128.
Optimize the search index each 500 records, fixes issue 1986.
Modified:
trunk/lib/search/QubitSearch.class.php
trunk/plugins/sfSearchPlugin/lib/index/xfIndexSingle.class.php
trunk/plugins/sfSearchPlugin/lib/task/xfPopulateTask.class.php
Modified: trunk/lib/search/QubitSearch.class.php
==============================================================================
--- trunk/lib/search/QubitSearch.class.php Wed May 11 00:02:49 2011
(r9097)
+++ trunk/lib/search/QubitSearch.class.php Fri May 13 16:05:55 2011
(r9098)
@@ -31,30 +31,95 @@
/**
* @see xfIndex
*/
- public function populate()
+ public function qubitPopulate($options)
{
- $this->optimize();
-
$start = microtime(true);
$this->getLogger()->log('Populating index...', $this->getName());
- $this->getEngine()->erase();
- $this->getLogger()->log('Index erased.', $this->getName());
- $criteria = new Criteria;
- foreach (QubitActor::get($criteria) as $actor)
+ if (!isset($options['actorOffset']) && !isset($options['ioOffset']))
+ {
+ $this->getEngine()->erase();
+ $this->getLogger()->log('Index erased.', $this->getName());
+ }
+ else
{
- $actor->updateLuceneIndex();
- $this->getLogger()->log('"'.$actor->__toString().'" inserted
('.round(microtime(true) - $start, 2).'s)', $this->getName());
+ $this->optimize();
}
- $criteria = new Criteria;
- foreach (QubitInformationObject::get($criteria) as $informationObject)
+ $actorOffset = intval($options['actorOffset']);
+ $ioOffset = intval($options['ioOffset']);
+
+ if (-1 < $actorOffset)
{
- SearchIndex::updateTranslatedLanguages($informationObject);
- $this->getLogger()->log('"'.$informationObject->__toString().'" inserted
('.round(microtime(true) - $start, 2).'s)', $this->getName());
+ $actors = QubitActor::getAll();
+ $counter = count($actors);
+ foreach ($actors as $key => $actor)
+ {
+ if (isset($actorOffset) && $key < $actorOffset)
+ {
+ if ($key == 0)
+ {
+ $this->getLogger()->log('Ignoring first '.$actorOffset.' actors.');
+ }
+
+ continue;
+ }
+
+ $actor->updateLuceneIndex();
+ $this->getLogger()->log('"'.$actor->__toString().'" inserted
('.round(microtime(true) - $start, 2).'s) ('.($key + 1).'/'.$counter.')',
$this->getName());
+
+ self::keepSearchIndexOptimized();
+ }
+ }
+ else
+ {
+ $this->getLogger()->log('Actors are ignored.');
+ } return;
+
+ if (-1 < $ioOffset)
+ {
+ $informationObjects = QubitInformationObject::getAll();
+ $counter = count($informationObjects);
+ for ($i = $ioOffset; $i < $counter; $i++)
+ {
+ $informationObject = $informationObjects[$i];
+
+ SearchIndex::updateTranslatedLanguages($informationObject);
+ $this->getLogger()->log('"'.$informationObject->__toString().'"
inserted ('.round(microtime(true) - $start, 2).'s) ('.($i +
1).'/'.$counter.')', $this->getName());
+
+ self::keepSearchIndexOptimized();
+ }
+ }
+ else
+ {
+ $this->getLogger()->log('Information objects are ignored.');
}
- $this->optimize();
$this->getLogger()->log('Index populated in "'.round(microtime(true) -
$start, 2).'" seconds.', $this->getName());
+
+ $this->optimize();
+ $this->getLogger()->log('Done!');
+ }
+
+ public function optimize()
+ {
+ $this->getEngine()->close();
+
+ parent::optimize();
+ }
+
+ /**
+ * Optimize the index when the function is called N times
+ */
+ public function keepSearchIndexOptimized()
+ {
+ global $_counter;
+
+ $_counter++;
+
+ if (0 == $_counter % 500)
+ {
+ $this->optimize();
+ }
}
}
Modified: trunk/plugins/sfSearchPlugin/lib/index/xfIndexSingle.class.php
==============================================================================
--- trunk/plugins/sfSearchPlugin/lib/index/xfIndexSingle.class.php Wed May
11 00:02:49 2011 (r9097)
+++ trunk/plugins/sfSearchPlugin/lib/index/xfIndexSingle.class.php Fri May
13 16:05:55 2011 (r9098)
@@ -137,7 +137,7 @@
/**
* @see xfIndex
*/
- final public function optimize()
+ public function optimize()
{
$this->setup();
Modified: trunk/plugins/sfSearchPlugin/lib/task/xfPopulateTask.class.php
==============================================================================
--- trunk/plugins/sfSearchPlugin/lib/task/xfPopulateTask.class.php Wed May
11 00:02:49 2011 (r9097)
+++ trunk/plugins/sfSearchPlugin/lib/task/xfPopulateTask.class.php Fri May
13 16:05:55 2011 (r9098)
@@ -26,7 +26,9 @@
$this->addOptions(array(
new sfCommandOption('application', null,
sfCommandOption::PARAMETER_OPTIONAL, 'The application name', true),
- new sfCommandOption('optimize', 'o', sfCommandOption::PARAMETER_NONE,
'If passed, the index is optimized after population')));
+ new sfCommandOption('optimize', 'o', sfCommandOption::PARAMETER_NONE,
'If passed, the index is optimized after population'),
+ new sfCommandOption('actorOffset', null,
sfCommandOption::PARAMETER_REQUIRED, 'Segmentation fault workaround (actors)'),
+ new sfCommandOption('ioOffset', null,
sfCommandOption::PARAMETER_REQUIRED, 'Segmentation fault workaround
(information objects)')));
$this->namespace = 'search';
$this->name = 'populate';
@@ -60,7 +62,7 @@
$index = new $index;
$index->setLogger(new xfLoggerTask($this->dispatcher, $this->formatter));
- $index->populate();
+ $index->qubitPopulate($options);
if ($options['optimize'])
{
--
You received this message because you are subscribed to the Google Groups
"Qubit Toolkit Commits" group.
To post to this group, send email to [email protected].
To unsubscribe from this group, send email to
[email protected].
For more options, visit this group at
http://groups.google.com/group/qubit-commits?hl=en.
