Eileen has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/369830 )
Change subject: Omnimail::omnigroupmember.get job: add slow download handling.
......................................................................
Omnimail::omnigroupmember.get job: add slow download handling.
This adds handling to for when the download takes a while, same as the
recipient job.
Bug T160949
Change-Id: Id65ab87235de86000393e66b0f7e7679692115bf
---
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Get.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
4 files changed, 152 insertions(+), 25 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm
refs/changes/30/369830/1
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
index fd9a6b7..b5007ab 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnigroupmembers.php
@@ -66,7 +66,7 @@
$params['mail_provider'] => array('last_timestamp' =>
$endTimestamp),
),
));
- return $data;
+ return $this->formatResult($params, $data);
}
else {
sleep($settings['omnimail_job_retry_interval']);
@@ -80,4 +80,34 @@
}
+ /**
+ * @param $params
+ * @param $result
+ * @return array
+ */
+ public function formatResult($params, $result) {
+ $options = _civicrm_api3_get_options_from_params($params);
+ $values = array();
+ foreach ($result as $groupMember) {
+ $value = array(
+ 'email' => (string) $groupMember->getEmail(),
+ 'is_opt_out' => (string) $groupMember->isOptOut(),
+ 'opt_in_date' => (string) $groupMember->getOptInIsoDateTime(),
+ 'opt_in_source' => (string) $groupMember->getOptInSource(),
+ 'opt_out_source' => (string) $groupMember->getOptOutSource(),
+ 'opt_out_date' => (string) $groupMember->getOptOutIsoDateTime(),
+ 'contact_id' => (string) $groupMember->getContactReference(),
+ );
+ foreach ($params['custom_data_map'] as $fieldName => $dataKey) {
+ $value[$fieldName] = (string) $groupMember->getCustomData($dataKey);
+ }
+ $values[] = $value;
+ if ($options['limit'] > 0 && count($values) === (int) $options['limit'])
{
+ break;
+ }
+ }
+ return $values;
+ }
+
+
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Get.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Get.php
index 0281f62..8e114d8 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Get.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Get.php
@@ -15,27 +15,7 @@
*/
function civicrm_api3_omnigroupmember_get($params) {
$job = new CRM_Omnimail_Omnigroupmembers();
- $result = $job->getResult($params);
- $options = _civicrm_api3_get_options_from_params($params);
- $values = array();
- foreach ($result as $groupMember) {
- $value = array(
- 'email' => (string) $groupMember->getEmail(),
- 'is_opt_out' => (string) $groupMember->isOptOut(),
- 'opt_in_date' => (string) $groupMember->getOptInIsoDateTime(),
- 'opt_in_source' => (string) $groupMember->getOptInSource(),
- 'opt_out_source' => (string) $groupMember->getOptOutSource(),
- 'opt_out_date' => (string) $groupMember->getOptOutIsoDateTime(),
- 'contact_id' => (string) $groupMember->getContactReference(),
- );
- foreach ($params['custom_data_map'] as $fieldName => $dataKey) {
- $value[$fieldName] = (string) $groupMember->getCustomData($dataKey);
- }
- $values[] = $value;
- if ($options['limit'] > 0 && count($values) === (int) $options['limit']) {
- break;
- }
- }
+ $values = $job->getResult($params);
return civicrm_api3_create_success($values);
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
index 5fcc26f..59e0cd7 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
@@ -14,11 +14,32 @@
* @return array
*/
function civicrm_api3_omnigroupmember_load($params) {
- $contacts = civicrm_api3('Omnigroupmember', 'get', $params);
+ $omnimail = new CRM_Omnimail_Omnigroupmembers($params);
+ if (!isset($params['options']) || !isset($params['options'])) {
+ $params['options'] = 0;
+ }
+ try {
+ $job = new CRM_Omnimail_Omnigroupmembers();
+ $contacts = $job->getResult($params);
+ }
+ catch (CRM_Omnimail_IncompleteDownloadException $e) {
+ $jobSettings = $omnimail->getJobSettings($params);
+ civicrm_api3('Setting', 'create', array(
+ 'omnimail_omnigroupmembers_load' => array(
+ $params['mail_provider'] => array(
+ 'last_timestamp' => $jobSettings['last_timestamp'],
+ 'retrieval_parameters' => $e->getRetrievalParameters(),
+ 'progress_end_date' => $e->getEndTimestamp(),
+ ),
+ ),
+ ));
+ return civicrm_api3_create_success(1);
+ }
+
$defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
$locationTypeID = $defaultLocationType->id;
- foreach ($contacts['values'] as $groupMember) {
+ foreach ($contacts as $groupMember) {
if (!empty($groupMember['email']) && !civicrm_api3('email', 'getcount',
array('email' => $groupMember['email']))) {
// If there is already a contact with this email we will skip for now.
// It might that we want to create duplicates, update contacts or do
other actions later
@@ -126,7 +147,6 @@
);
$params['start_date'] = array(
'title' => ts('Date to fetch from'),
- 'api.default' => '3 days ago',
'type' => CRM_Utils_Type::T_TIMESTAMP,
);
$params['end_date'] = array(
@@ -141,5 +161,22 @@
$params['retrieval_parameters'] = array(
'title' => ts('Additional information for retrieval of pre-stored
requests'),
);
+ $params['custom_data_map'] = array(
+ 'type' => CRM_Utils_Type::T_STRING,
+ 'title' => ts('Custom fields map'),
+ 'description' => array('custom mappings pertaining to the mail provider
fields'),
+ 'api.default' => array(
+ 'language' => 'rml_language',
+ 'source' => 'rml_source',
+ 'created_date' => 'rml_submitdate',
+ 'country' => 'rml_country',
+ ),
+ );
+ $params['is_opt_in_only'] = array(
+ 'type' => CRM_Utils_Type::T_BOOLEAN,
+ 'title' => ts('Opted in contacts only'),
+ 'description' => array('Restrict to opted in contacts'),
+ 'api.default' => 1,
+ );
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
index 27c9c94..a4752a0 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
@@ -66,6 +66,75 @@
}
+ /**
+ * Test when download does not complete in time.
+ */
+ public function testOmnigroupmemberLoadIncomplete() {
+ civicrm_api3('Setting', 'create', array(
+ 'omnimail_omnigroupmembers_load' => array(
+ 'Silverpop' => array('last_timestamp' => '1487890800'),
+ ),
+ ));
+ $responses = array(
+ file_get_contents(__DIR__ . '/Responses/ExportListResponse.txt'),
+ );
+ for ($i = 0; $i < 15; $i++) {
+ $responses[] = file_get_contents(__DIR__ .
'/Responses/JobStatusWaitingResponse.txt');
+ }
+ civicrm_api3('setting', 'create', array('omnimail_job_retry_interval' =>
0.01));
+ $group = civicrm_api3('Group', 'create', array('name' => 'Omnimailers',
'title' => 'Omni'));
+
+ civicrm_api3('Omnigroupmember', 'load', array('mail_provider' =>
'Silverpop', 'username' => 'Donald', 'password' => 'Duck', 'client' =>
$this->getMockRequest($responses), 'group_identifier' => 123, 'group_id' =>
$group['id'], 'start_date' => '2017-02-20'));
+
+ $groupMembers = civicrm_api3('GroupContact', 'get', array('group_id' =>
$group['id']));
+ $this->assertEquals(0, $groupMembers['count']);
+
+ $this->assertEquals(array(
+ 'last_timestamp' => '1487890800',
+ 'retrieval_parameters' => array(
+ 'jobId' => '101719657',
+ 'filePath' => '/download/20170509_noCID - All - Jul 5 2017 06-27-45
AM.csv',
+ ),
+ 'progress_end_date' => '1488150000',
+ ), $this->getJobSettings());
+ }
+
+ /**
+ * After completing an incomplete download the end date should be the
progress end date.
+ */
+ public function testCompleteIncomplete() {
+ $client = $this->setupSuccessfulDownloadClient();
+ $group = civicrm_api3('Group', 'create', array('name' => 'Omnimailers',
'title' => 'Omni'));
+ civicrm_api3('setting', 'create', array(
+ 'omnimail_omnigroupmembers_load' => array(
+ 'Silverpop' => array(
+ 'last_timestamp' => '1487890800',
+ 'retrieval_parameters' => array(
+ 'jobId' => '101719657',
+ 'filePath' => '/download/20170509_noCID - All - Jul 5 2017
06-27-45 AM.csv',
+ ),
+ 'progress_end_date' => '1488150000',
+ ),
+ ),
+ ));
+
+ civicrm_api3('Omnigroupmember', 'load', array(
+ 'mail_provider' => 'Silverpop',
+ 'username' => 'Shrek',
+ 'password' => 'Fiona',
+ 'options' => array('limit' => 3),
+ 'client' => $client,
+ 'group_identifier' => 123,
+ 'group_id' => $group['id'],
+ ));
+
+ $groupMembers = civicrm_api3('GroupContact', 'get', array('group_id' =>
$group['id']));
+ $this->assertEquals(3, $groupMembers['count']);
+
+ $this->assertEquals(array(
+ 'last_timestamp' => '1488495600',
+ ), $this->getJobSettings(array('mail_provider' => 'Silverpop')));
+ }
/**
* @return \GuzzleHttp\Client
@@ -82,4 +151,15 @@
$client = $this->getMockRequest($responses);
return $client;
}
+
+ /**
+ * Get job settings.
+ *
+ * @return array
+ */
+ public function getJobSettings() {
+ $omnimail = new CRM_Omnimail_Omnigroupmembers();
+ return $omnimail->getJobSettings(array('mail_provider' => 'Silverpop'));
+ }
+
}
--
To view, visit https://gerrit.wikimedia.org/r/369830
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id65ab87235de86000393e66b0f7e7679692115bf
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/crm
Gerrit-Branch: master
Gerrit-Owner: Eileen <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
