Eileen has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/379184 )
Change subject: Update Omnimail recipient load to support parameterised
batching.
......................................................................
Update Omnimail recipient load to support parameterised batching.
Jeff suggested that many small inserts puts more load on replication. This
gives us
'insert_batch_size' as another lever to tinker with. It's not reall applicable
to
the groupmember job as we use the api for that, as we do with the other import
jobs.
Bug: T176255
Change-Id: I191e8ddaf21f388732c360c372ad5a93fe63c6fa
---
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
2 files changed, 87 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm
refs/changes/84/379184/1
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
index 5a328cc..06f8c9e 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
@@ -41,17 +41,21 @@
$rowsLeftBeforeThrottle = $throttleCount;
$limit = (isset($params['options']['limit'])) ?
$params['options']['limit'] : NULL;
$count = 0;
+ $insertBatchSize = CRM_Utils_Array::value('insert_batch_size', $params, 1);
+ $valueStrings = array();
foreach ($recipients as $recipient) {
if ($count === $limit) {
+ // Do this here - ie. before processing a new row rather than at the
end of the last row
+ // to avoid thinking a job is incomplete if the limit co-incides with
available rows.
+ // Also write any remaining rows to the DB before exiting.
+ _civicrm_api3_omnirecipient_load_write_remainder_rows($valueStrings);
$omnimail->saveJobSetting(array(
'last_timestamp' => $jobSettings['last_timestamp'],
'retrieval_parameters' => $omnimail->getRetrievalParameters(),
'progress_end_date' => $omnimail->endTimeStamp,
'offset' => $omnimail->getOffset() + $count,
));
- // Do this here - ie. before processing a new row rather than at the
end of the last row
- // to avoid thinking a job is incomplete if the limit co-incides with
available rows.
return civicrm_api3_create_success(1);
}
$insertValues = array(
@@ -68,11 +72,8 @@
),
6 => array((string) $recipient->getContactReference(), 'String'),
);
- CRM_Core_DAO::executeQuery("
- INSERT IGNORE INTO civicrm_mailing_provider_data
- (`contact_identifier`, `mailing_identifier`, `email`, `event_type`,
`recipient_action_datetime`, `contact_id`)
- values(%1, %2, %3, %4, %5, %6 )",
- $insertValues);
+ $valueStrings[] = CRM_Core_DAO::composeQuery("(%1, %2, %3, %4, %5, %6
)", $insertValues);
+ $valueStrings =
_civicrm_api3_omnirecipient_load_batch_write_to_db($valueStrings,
$insertBatchSize);
$rowsLeftBeforeThrottle--;
$count++;
@@ -84,6 +85,7 @@
sleep(ceil($throttleStagePoint - strtotime('now')));
}
}
+ _civicrm_api3_omnirecipient_load_write_remainder_rows($valueStrings);
$omnimail->saveJobSetting(array('last_timestamp' =>
$omnimail->endTimeStamp));
return civicrm_api3_create_success(1);
}
@@ -96,6 +98,38 @@
return civicrm_api3_create_success(1);
}
+}
+
+/**
+ * @param $valueStrings
+ */
+function _civicrm_api3_omnirecipient_load_write_remainder_rows($valueStrings) {
+ if (count($valueStrings)) {
+ _civicrm_api3_omnirecipient_load_batch_write_to_db($valueStrings,
count($valueStrings));
+ }
+}
+
+/**
+ * Write the imported values to the DB, batching per parameter.
+ *
+ * Save the values to the DB in batch sizes accordant with the insertBatchSize
+ * parameter.
+ *
+ * @param array $valueStrings
+ * @param int $insertBatchSize
+ *
+ * @return array
+ */
+function _civicrm_api3_omnirecipient_load_batch_write_to_db($valueStrings,
$insertBatchSize) {
+ if (count($valueStrings) === $insertBatchSize) {
+ CRM_Core_DAO::executeQuery("
+ INSERT IGNORE INTO civicrm_mailing_provider_data
+ (`contact_identifier`, `mailing_identifier`, `email`, `event_type`,
`recipient_action_datetime`, `contact_id`)
+ values" . implode(',', $valueStrings)
+ );
+ $valueStrings = array();
+ }
+ return $valueStrings;
}
/**
@@ -150,5 +184,11 @@
'type' => CRM_Utils_Type::T_STRING,
'api.default' => '',
);
+ $params['insert_batch_size'] = array(
+ 'title' => ts('Number of rows to insert in each DB write'),
+ 'description' => ts('Set this to increase row batching.'),
+ 'type' => CRM_Utils_Type::T_INT,
+ 'api.default' => 1,
+ );
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
index 7d94d41..f502fdd 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
@@ -93,6 +93,46 @@
}
/**
+ * Test for no errors when using 'insert_batch_size' parameter.
+ *
+ * It's hard to test that it does batch, but at least we can check it
+ * succeeds.
+ */
+ public function testOmnirecipientLoadIncreasedBatchInsert() {
+ $client = $this->setupSuccessfulDownloadClient();
+
+ civicrm_api3('Omnirecipient', 'load', array(
+ 'mail_provider' => 'Silverpop',
+ 'username' => 'Donald',
+ 'password' => 'Duck',
+ 'debug' => 1,
+ 'client' => $client,
+ 'insert_batch_size' => 4,
+ ));
+ $this->assertEquals(4, CRM_Core_DAO::singleValueQuery('SELECT count(*)
FROM civicrm_mailing_provider_data'));
+ }
+
+ /**
+ * Test for no errors when using 'insert_batch_size' parameter.
+ *
+ * It's hard to test that it does batch, but at least we can check it
+ * succeeds.
+ */
+ public function testOmnirecipientLoadIncreasedBatchInsertExceedsAvailable() {
+ $client = $this->setupSuccessfulDownloadClient();
+
+ civicrm_api3('Omnirecipient', 'load', array(
+ 'mail_provider' => 'Silverpop',
+ 'username' => 'Donald',
+ 'password' => 'Duck',
+ 'debug' => 1,
+ 'client' => $client,
+ 'insert_batch_size' => 6,
+ ));
+ $this->assertEquals(4, CRM_Core_DAO::singleValueQuery('SELECT count(*)
FROM civicrm_mailing_provider_data'));
+ }
+
+ /**
* Example: Test that a version is returned.
*/
public function testOmnirecipientLoadLimitAndOffset() {
--
To view, visit https://gerrit.wikimedia.org/r/379184
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I191e8ddaf21f388732c360c372ad5a93fe63c6fa
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/crm
Gerrit-Branch: master
Gerrit-Owner: Eileen <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
