Author: fortim
Date: Mon Sep 21 04:10:32 2009
New Revision: 3455
Log:
Stabilize OAI Harvests
Modified:
trunk/apps/qubit/modules/oai/actions/harvesterHarvestAction.class.php
trunk/apps/qubit/modules/oai/templates/harvesterListSuccess.php
trunk/lib/model/QubitOaiHarvest.php
Modified: trunk/apps/qubit/modules/oai/actions/harvesterHarvestAction.class.php
==============================================================================
--- trunk/apps/qubit/modules/oai/actions/harvesterHarvestAction.class.php
Mon Sep 21 00:02:49 2009 (r3454)
+++ trunk/apps/qubit/modules/oai/actions/harvesterHarvestAction.class.php
Mon Sep 21 04:10:32 2009 (r3455)
@@ -39,6 +39,9 @@
//Set no records match to false to start with
$this->noRecordsMatch = false;
+
+ //Keep track of number of records harvested
+ $this->recordCount = 0;
//If the request did not go through the proper routing, forward to 404
if (!$request->hasParameter('harvestId'))
@@ -64,38 +67,34 @@
$oaiSimpleRes = array();
$from = date('Y-m-d\TH:i:s\Z', strtotime($harvestInfo->getLastHarvest()));
$until = gmdate('Y-m-d\TH:i:s\Z');
- while ($resumptionToken)
+
+ //Create the base request
+ $verb = 'verb=ListRecords';
+ if ($harvestInfo->getLastHarvest() != null)
{
- //Create the base request
- $verb = 'verb=ListRecords';
- if ($harvestInfo->getLastHarvest() != null)
- {
- $verb .= '&from='.$from;
- }
- $verb .= '&until='.$until;
- $verb .= '&metadataPrefix='.$harvestInfo->getMetadataPrefix();
-
- //Add the set parameter if supplied
- if ($harvestInfo->getSet() != null)
- {
- $verb .= '&set='.$harvestInfo->getSet();
- }
+ $verb .= '&from='.$from;
+ }
+ $verb .= '&until='.$until;
+ $verb .= '&metadataPrefix='.$harvestInfo->getMetadataPrefix();
- //Add the resumption token if token supplied by harvestee [set at end of
loop]
- if ($resumptionToken != 1 && $resumptionToken != false)
- {
- $verb .= '&resumptionToken='.$resumptionToken;
- }
+ //Add the set parameter if supplied
+ if ($harvestInfo->getSet() != null)
+ {
+ $verb .= '&set='.$harvestInfo->getSet();
+ }
+ while ($resumptionToken)
+ {
//Open a socket to the specified repository, with correct parameters
$socket = new QubitSocket($rep->getUri().'?'.$verb);
-
$oaiResponse = $socket->get_data();
//If communication failed 404
if ($oaiResponse == false)
{
- QubitOai::mailErrors('Unable to connect to repository :
'.$this->repositoryName);
+ // Add mail to admin if can't connect.
+ //QubitOai::mailErrors('Unable to connect to repository :
'.$this->repositoryName);
+ print_r("Can't connect to repository");
$this->forward404();
}
@@ -116,55 +115,65 @@
}
}
- if ($oaiSimple->xpath('//c:ListRecords/c:record'))
+ if (!$this->noRecordsMatch)
{
- $oaiSimpleRes = array_merge($oaiSimpleRes,
$oaiSimple->xpath('//c:ListRecords/c:record'));
+
+ //Container for xml import errors
+ $this->errorsFound = array();
+ $this->errorsXML = array();
+
+ //Create header and footer for XML record for it to validate
+ $oaiHeader = '<?xml version="1.0" encoding="UTF-8" ?>';
+
+ $oaiFooter = '';
+ $oaiRecords = $oaiSimple->xpath('//c:ListRecords/c:record');
+ foreach ($oaiRecords as $oaiRec)
+ {
+ $oaiRec = $oaiHeader.$oaiRec->asXML().$oaiFooter."\n";
+
+ $options = array();
+ $options = $options['strictXmlParsing'] = false;
+ $importer = QubitXmlImport::execute($oaiRec, $options);
+ if ($importer->hasErrors())
+ {
+ $this->errorsFound[] = $importer->getErrors();
+ $this->errorsXML[] = $oaiRec;
+ }
+ }
+
+
+ // Increment record count to keep track of number of records harvested
+ $this->recordCount += count($oaiRecords);
+
+ $nbrErrors = count($this->errorsFound);
+ $errorReport = '';
+ $errorReportHTML = '';
+ for ($i = 0; $i < $nbrErrors; $i++)
+ {
+ $errorReport .= "Error when importing
record:\n\n".$this->errorsXML[$i]."\n\nError message:\n".$this->errorsFound[$i];
+ $errorReport .=
"\n**************************************************\n\n";
+ $errorReportHTML .= "Error when importing record:\n\n
<br>".$this->errorsXML[$i]."\n\n <br> Error message: <br>
\n".$this->errorsFound[$i];
+ }
+
}
//Check for resumption token which will also be the while loop indicator
- $resumptionToken = QubitOai::getResumptionToken($oaiResponse);
+ $oaiResumptionToken =
$oaiSimple->xpath('//c:ListRecords/c:resumptionToken');
+ if($oaiResumptionToken == false || count($oaiResumptionToken) > 1)
+ {
+ $resumptionToken = false;
+ } else {
+ $resumptionToken = $oaiResumptionToken[0];
+ $verb = 'verb=ListRecords&resumptionToken='.$resumptionToken;
+ }
+
}
// Update last harvest date
$harvestInfo->setLastHarvest($until);
$harvestInfo->save();
- if (!$this->noRecordsMatch)
- {
-
- //Container for xml import errors
- $this->errorsFound = array();
- $this->errorsXML = array();
-
- //Create header and footer for XML record for it to validate
- $oaiHeader = '<?xml version="1.0" encoding="UTF-8" ?>';
- $oaiFooter = '';
- foreach ($oaiSimpleRes as $oaiRec)
- {
- $oaiRec = $oaiHeader.$oaiRec->asXML().$oaiFooter;
- $options = array();
- $options = $options['strictXmlParsing'] = false;
- $importer = QubitXmlImport::execute($oaiRec, $options);
- if ($importer->hasErrors())
- {
- $this->errorsFound[] = $importer->getErrors();
- $this->errorsXML[] = $oaiRec;
- }
- }
- $this->recordCount = count($oaiSimpleRes);
- $nbrErrors = count($this->errorsFound);
- $errorReport = '';
- $errorReportHTML = '';
- for ($i = 0; $i < $nbrErrors; $i++)
- {
- $errorReport .= "Error when importing
record:\n\n".$this->errorsXML[$i]."\n\nError message:\n".$this->errorsFound[$i];
- $errorReport .=
"\n**************************************************\n\n";
- $errorReportHTML .= "Error when importing record:\n\n
<br>".$this->errorsXML[$i]."\n\n <br> Error message: <br>
\n".$this->errorsFound[$i];
- }
- $returnMail = QubitOai::mailErrors($errorReport);
-
- }
}
}
\ No newline at end of file
Modified: trunk/apps/qubit/modules/oai/templates/harvesterListSuccess.php
==============================================================================
--- trunk/apps/qubit/modules/oai/templates/harvesterListSuccess.php Mon Sep
21 00:02:49 2009 (r3454)
+++ trunk/apps/qubit/modules/oai/templates/harvesterListSuccess.php Mon Sep
21 04:10:32 2009 (r3455)
@@ -18,7 +18,7 @@
<tr>
<td>
<div>
- <a href="<?php echo $rep->getUri();?>?verb=Identify"><?php echo
$rep->getName(); ?></a><br>Last Harvest: <?php $harvest =
QubitOaiHarvest::getLastHarvest($rep->getId()); echo
$harvest->getLastHarvestAttempt();;?>
+ <a href="<?php echo $rep->getUri();?>?verb=Identify"><?php echo
$rep->getName(); ?></a><br>Last Harvest: <?php $harvest =
QubitOaiHarvest::getLastHarvestByID($rep->getId()); echo
$harvest->getLastHarvest();;?>
</div>
</td>
<td>
Modified: trunk/lib/model/QubitOaiHarvest.php
==============================================================================
--- trunk/lib/model/QubitOaiHarvest.php Mon Sep 21 00:02:49 2009 (r3454)
+++ trunk/lib/model/QubitOaiHarvest.php Mon Sep 21 04:10:32 2009 (r3455)
@@ -33,7 +33,7 @@
* @var int id, the id for the repository
* @return date the last harvest date for that repository
*/
- public static function getLastHarvest($id)
+ public static function getLastHarvestByID($id)
{
$criteria = new Criteria;
$criteria->add(QubitOaiHarvest::OAI_REPOSITORY_ID, $id);
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"Qubit Toolkit Commits" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to
[email protected]
For more options, visit this group at
http://groups.google.ca/group/qubit-commits?hl=en
-~----------~----~----~----~------~----~------~--~---
