Eileen has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/377664 )
Change subject: Update Omnimail to use offset
......................................................................
Update Omnimail to use offset
Bug: T175665
Change-Id: If520df67b3a00a557ec849bc367406d0ec22b330
---
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
M sites/default/civicrm/extensions/org.wikimedia.omnimail/composer.lock
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
5 files changed, 139 insertions(+), 15 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm
refs/changes/64/377664/1
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
index e38e65f..3e8c48f 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
@@ -41,6 +41,13 @@
}
$request = Omnimail::create($params['mail_provider'],
$mailerCredentials)->getGroupMembers($jobParameters);
+ $offset = CRM_Utils_Array::value('offset', $jobSettings, 0);
+ if (isset($params['options']['offset'])) {
+ $offset = $params['options']['offset'];
+ }
+ if ($offset) {
+ $request->setOffset((int) $offset);
+ }
$startTimestamp = self::getStartTimestamp($params, $jobSettings);
$this->endTimeStamp =
self::getEndTimestamp(CRM_Utils_Array::value('end_date', $params), $settings,
$startTimestamp);
@@ -58,6 +65,7 @@
$request->setGroupIdentifier($params['group_identifier']);
$result = $request->getResponse();
+ $this->setRetrievalParameters($result->getRetrievalParameters());
for ($i = 0; $i < $settings['omnimail_job_retry_number']; $i++) {
if ($result->isCompleted()) {
$data = $result->getData();
@@ -69,7 +77,7 @@
}
throw new CRM_Omnimail_IncompleteDownloadException('Download incomplete',
0, array(
- 'retrieval_parameters' => $result->getRetrievalParameters(),
+ 'retrieval_parameters' => $this->getRetrievalParameters(),
'mail_provider' => $params['mail_provider'],
'end_date' => $this->endTimeStamp,
));
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
index b372980..04a548f 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
@@ -26,6 +26,26 @@
*/
public $endTimeStamp;
+
+ /**
+ * @var array
+ */
+ protected $retrievalParameters;
+
+ /**
+ * @return array
+ */
+ public function getRetrievalParameters() {
+ return $this->retrievalParameters;
+ }
+
+ /**
+ * @param array $retrievalParameters
+ */
+ public function setRetrievalParameters($retrievalParameters) {
+ $this->retrievalParameters = $retrievalParameters;
+ }
+
/**
* Get the timestamp to start from.
*
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
index 7744a1a..6ce2547 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
@@ -22,17 +22,18 @@
$rowsLeftBeforeThrottle = $throttleCount;
$job = new CRM_Omnimail_Omnigroupmembers();
+ $jobSettings = $job->getJobSettings($params);
try {
$contacts = $job->getResult($params);
}
catch (CRM_Omnimail_IncompleteDownloadException $e) {
- $jobSettings = $job->getJobSettings($params);
civicrm_api3('Setting', 'create', array(
'omnimail_omnigroupmembers_load' => array(
$params['mail_provider'] => array(
'last_timestamp' => $jobSettings['last_timestamp'],
'retrieval_parameters' => $e->getRetrievalParameters(),
'progress_end_date' => $e->getEndTimestamp(),
+ 'offset' => 0,
),
),
));
@@ -42,7 +43,31 @@
$defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
$locationTypeID = $defaultLocationType->id;
+ if (isset($params['options']['offset'])) {
+ $offset = $params['options']['offset'];
+ }
+ else {
+ $offset = CRM_Utils_Array::value('offset', $jobSettings, 0);
+ }
+ $limit = (isset($params['options']['limit'])) ? $params['options']['limit']
: NULL;
+ $count = 0;
+
foreach ($contacts as $contact) {
+ if ($count === $limit) {
+ civicrm_api3('Setting', 'create', array(
+ 'omnimail_omnigroupmembers_load' => array(
+ $params['mail_provider'] => array(
+ 'last_timestamp' => $jobSettings['last_timestamp'],
+ 'retrieval_parameters' => $job->getRetrievalParameters(),
+ 'progress_end_date' => $job->endTimeStamp,
+ 'offset' => $offset,
+ ),
+ ),
+ ));
+ // Do this here - ie. before processing a new row rather than at the end
of the last row
+ // to avoid thinking a job is incomplete if the limit co-incides with
available rows.
+ return civicrm_api3_create_success($values);
+ }
$groupMember = $job->formatRow($contact, $params['custom_data_map']);
if (!empty($groupMember['email']) && !civicrm_api3('email', 'getcount',
array('email' => $groupMember['email']))) {
// If there is already a contact with this email we will skip for now.
@@ -76,12 +101,23 @@
}
$values[$contact['id']] = reset($contact['values']);
}
+ $offset++;
+ $count++;
+ // Every row seems extreme but perhaps not in this performance monitoring
phase.
+ civicrm_api3('Setting', 'create', array(
+ 'omnimail_omnigroupmembers_load' => array(
+ $params['mail_provider'] => array_merge(
+ $jobSettings, array('offset' => $offset)
+ )
+ ),
+ ));
$rowsLeftBeforeThrottle--;
if ($throttleStagePoint && (strtotime('now') > $throttleStagePoint)) {
$throttleStagePoint = strtotime('+ ' . (int) $throttleSeconds .
'seconds');
$rowsLeftBeforeThrottle = $throttleCount;
}
+
if ($throttleSeconds && $rowsLeftBeforeThrottle <= 0) {
sleep(ceil($throttleStagePoint - strtotime('now')));
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/composer.lock
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/composer.lock
index 65a33d2..545b430 100644
--- a/sites/default/civicrm/extensions/org.wikimedia.omnimail/composer.lock
+++ b/sites/default/civicrm/extensions/org.wikimedia.omnimail/composer.lock
@@ -707,7 +707,7 @@
"source": {
"type": "git",
"url":
"https://github.com/eileenmcnaughton/omnimail-silverpop.git";,
- "reference": "e06e50192ae17fe93f1045ee7562aa286272cbcc"
+ "reference": "aac29243defa1dfc78cff7f51bbb4f2c30de39ec"
},
"require": {
"league/csv": "^8.0",
@@ -746,7 +746,7 @@
"omnimail",
"silverpop"
],
- "time": "2017-07-05 04:12:48"
+ "time": "2017-09-12 13:57:41"
}
],
"packages-dev": [
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
index 54ee55f..451e119 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
@@ -47,15 +47,7 @@
civicrm_api3('Omnigroupmember', 'load', array('mail_provider' =>
'Silverpop', 'username' => 'Shrek', 'password' => 'Fiona', 'options' =>
array('limit' => 3), 'client' => $client, 'group_identifier' => 123, 'group_id'
=> $group['id']));
$groupMembers = civicrm_api3('GroupContact', 'get', array('group_id' =>
$group['id']));
$this->assertEquals(3, $groupMembers['count']);
- $contactIDs = array('IN' => array());
- foreach ($groupMembers['values'] as $groupMember) {
- $contactIDs['IN'][] = $groupMember['contact_id'];
- }
- $contacts = civicrm_api3('Contact', 'get', array(
- 'contact_id' => $contactIDs,
- 'sequential' => 1,
- 'return' => array('contact_source', 'email', 'country', 'created_date',
'preferred_language', 'is_opt_out')
- ));
+ $contacts = $this->getGroupMemberDetails($groupMembers);
$this->assertEquals('fr_FR', $contacts['values'][0]['preferred_language']);
$this->assertEquals('e...@example.com', $contacts['values'][0]['email']);
$this->assertEquals('France', $contacts['values'][0]['country']);
@@ -63,6 +55,45 @@
$this->assertEquals('Silverpop Added by WebForms 10/18/16',
$contacts['values'][0]['contact_source']);
$this->assertEquals('Silverpop clever place 07/04/17',
$contacts['values'][2]['contact_source']);
+ $this->cleanupGroup($group);
+ }
+
+ /**
+ * Example: Test that offset is respected.
+ */
+ public function testOmnigroupmemberLoadOffset() {
+ $client = $this->setupSuccessfulDownloadClient();
+ $group = civicrm_api3('Group', 'create', array('name' => 'Omnimailers',
'title' => 'Omni'));
+
+ civicrm_api3('Omnigroupmember', 'load', array('mail_provider' =>
'Silverpop', 'username' => 'Shrek', 'password' => 'Fiona', 'options' =>
array('offset' => 1), 'client' => $client, 'group_identifier' => 123,
'group_id' => $group['id']));
+ $groupMembers = civicrm_api3('GroupContact', 'get', array('group_id' =>
$group['id']));
+ $this->assertEquals(2, $groupMembers['count']);
+ $contacts = $this->getGroupMemberDetails($groupMembers);
+ $this->assertEquals('sa...@example.com', $contacts['values'][0]['email']);
+ $this->assertEquals('l...@example.com', $contacts['values'][1]['email']);
+ $this->cleanupGroup($group);
+ }
+
+ /**
+ * Example: Test that offset is respected.
+ */
+ public function testOmnigroupmemberLoadUseOffsetSetting() {
+ $client = $this->setupSuccessfulDownloadClient();
+ $group = civicrm_api3('Group', 'create', array('name' => 'Omnimailers',
'title' => 'Omni'));
+
+ civicrm_api3('Omnigroupmember', 'load', array('mail_provider' =>
'Silverpop', 'username' => 'Shrek', 'password' => 'Fiona', 'options' =>
array('limit' => 1), 'client' => $client, 'group_identifier' => 123, 'group_id'
=> $group['id']));
+ $groupMembers = civicrm_api3('GroupContact', 'get', array('group_id' =>
$group['id']));
+ $this->assertEquals(1, $groupMembers['count']);
+ $contacts = $this->getGroupMemberDetails($groupMembers);
+ $this->assertEquals('e...@example.com', $contacts['values'][0]['email']);
+
+ // Re-run. Offset is now 1 in settings & we are passing in limit =1. Sarah
should be created.
+ $client = $this->setupSuccessfulDownloadClient(FALSE);
+ civicrm_api3('Omnigroupmember', 'load', array('mail_provider' =>
'Silverpop', 'username' => 'Shrek', 'password' => 'Fiona', 'options' =>
array('limit' => 1), 'client' => $client, 'group_identifier' => 123, 'group_id'
=> $group['id']));
+ $groupMembers = civicrm_api3('GroupContact', 'get', array('group_id' =>
$group['id']));
+ $this->assertEquals(2, $groupMembers['count']);
+ $contacts = $this->getGroupMemberDetails($groupMembers);
+ $this->assertEquals('sa...@example.com', $contacts['values'][1]['email']);
$this->cleanupGroup($group);
}
@@ -140,16 +171,21 @@
}
/**
+ * Set up the mock client to emulate a successful download.
+ * @param bool $isUpdateSetting
+ *
* @return \GuzzleHttp\Client
*/
- protected function setupSuccessfulDownloadClient() {
+ protected function setupSuccessfulDownloadClient($isUpdateSetting = TRUE) {
$responses = array(
file_get_contents(__DIR__ . '/Responses/ExportListResponse.txt'),
file_get_contents(__DIR__ . '/Responses/JobStatusCompleteResponse.txt'),
);
copy(__DIR__ . '/Responses/20170509_noCID - All - Jul 5 2017 06-27-45
AM.csv', sys_get_temp_dir() . '/20170509_noCID - All - Jul 5 2017 06-27-45
AM.csv');
fopen(sys_get_temp_dir() . '/20170509_noCID - All - Jul 5 2017 06-27-45
AM.csv.complete', 'c');
- $this->createSetting('omnimail_omnigroupmembers_load', array('Silverpop'
=> array('last_timestamp' => '1487890800')));
+ if ($isUpdateSetting) {
+ $this->createSetting('omnimail_omnigroupmembers_load', array('Silverpop'
=> array('last_timestamp' => '1487890800')));
+ }
$client = $this->getMockRequest($responses);
return $client;
@@ -177,4 +213,28 @@
}
+ /**
+ * @param $groupMembers
+ * @return array
+ */
+ protected function getGroupMemberDetails($groupMembers) {
+ $contactIDs = array('IN' => array());
+ foreach ($groupMembers['values'] as $groupMember) {
+ $contactIDs['IN'][] = $groupMember['contact_id'];
+ }
+ $contacts = civicrm_api3('Contact', 'get', array(
+ 'contact_id' => $contactIDs,
+ 'sequential' => 1,
+ 'return' => array(
+ 'contact_source',
+ 'email',
+ 'country',
+ 'created_date',
+ 'preferred_language',
+ 'is_opt_out'
+ )
+ ));
+ return $contacts;
+ }
+
}
--
To view, visit https://gerrit.wikimedia.org/r/377664
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If520df67b3a00a557ec849bc367406d0ec22b330
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/crm
Gerrit-Branch: master
Gerrit-Owner: Eileen <emcnaugh...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
