Jgleeson has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/392425 )
Change subject: WIP added first attempt of measuring donation processing stats
......................................................................
WIP added first attempt of measuring donation processing stats
Change-Id: Id053e7e6392dd480d59b933d47eea99d421be3c3
---
M sites/all/modules/queue2civicrm/DonationStats.php
M sites/all/modules/queue2civicrm/queue2civicrm.module
2 files changed, 50 insertions(+), 11 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm
refs/changes/25/392425/1
diff --git a/sites/all/modules/queue2civicrm/DonationStats.php
b/sites/all/modules/queue2civicrm/DonationStats.php
index 3870c2e..2fbb0f0 100644
--- a/sites/all/modules/queue2civicrm/DonationStats.php
+++ b/sites/all/modules/queue2civicrm/DonationStats.php
@@ -34,6 +34,8 @@
* 7) Gateway specific moving average of (6)
* 8) Overall moving average of (6)
*
+ * This method is called within @see DonationQueueConsumer::processMessage()
+ *
* @param array $message
* @param array $contribution
*/
@@ -78,7 +80,19 @@
}
/**
- * Export overall & average stats data to Prometheus out files using the
+ * Record DonationQueueConsumer total batch processing time.
+ *
+ * Also trigger the call to generate and record average-donations-processed
stats.
+ *
+ * @param $batchProcessingTime
+ */
+ public function recordBatchProcessingTime($batchProcessingTime) {
+ $this->statsCollector->add("batch_processing_time", $batchProcessingTime);
+
$this->generateAndRecordAverageDonationsProcessedStats($batchProcessingTime);
+ }
+
+ /**
+ * Export donation stats data to Prometheus out files using the
* Statistics\Exporter\Prometheus exporter.
*
* @param string $filename
@@ -86,6 +100,7 @@
*/
public function exportToPrometheus($filename, $path) {
$prometheusExporter = new PrometheusStatsExporter($filename, $path);
+
// define the stats we want export.
// NOTE: we could pass the entire statsCollector object here to be
export() but individual records which are
// averaged as part of the queue consumer process have no individual value
so they are omitted
@@ -178,5 +193,23 @@
);
}
+ /*
+ * Record the average number of donations processed within a per-second time
period using a
+ * calculation of the average donations processed per second as the base and
extrapolating upwards to
+ * estimate averages over longer periods.
+ *
+ * This method of blind average scaling with no will distort the *actual*
messages-processed
+ * per-time-window so will not be useful or accurate above short time
periods.
+ *
+ * @param $batchProcessingTime
+ */
+ protected function
generateAndRecordAverageDonationsProcessedStats($batchProcessingTime) {
+ $totalDonationsProcessed = $this->statsCollector->get("overall.donations");
+ $donationsProcesedPerSecond = $totalDonationsProcessed /
$batchProcessingTime;
+ $this->statsCollector->add("average.processed.per_second",
$donationsProcesedPerSecond);
+ $this->statsCollector->add("average.processed.per_5_seconds",
$donationsProcesedPerSecond * 5);
+ $this->statsCollector->add("average.processed.per_10_seconds",
$donationsProcesedPerSecond * 10);
+ $this->statsCollector->add("average.processed.per_30_seconds",
$donationsProcesedPerSecond * 30);
+ }
}
\ No newline at end of file
diff --git a/sites/all/modules/queue2civicrm/queue2civicrm.module
b/sites/all/modules/queue2civicrm/queue2civicrm.module
index f6aef56..ca6efa3 100644
--- a/sites/all/modules/queue2civicrm/queue2civicrm.module
+++ b/sites/all/modules/queue2civicrm/queue2civicrm.module
@@ -1,9 +1,10 @@
<?php
+
use queue2civicrm\DonationQueueConsumer;
use DonationStats;
// include common functions
-require_once( drupal_get_path( 'module', 'queue2civicrm' ) .
'/queue2civicrm_common.inc' );
+require_once(drupal_get_path('module', 'queue2civicrm') .
'/queue2civicrm_common.inc');
/**
* Implementation of hook_menu().
@@ -38,7 +39,7 @@
}
function queue2civicrm_stomp_url() {
- return variable_get( 'queue2civicrm_url', 'tcp://localhost:61613' );
+ return variable_get('queue2civicrm_url', 'tcp://localhost:61613');
}
/**
@@ -62,7 +63,7 @@
'#type' => 'checkbox',
'#title' => t('Disable job'),
'#description' => t('If checked, no message processing will be
performed.'),
- '#default_value' => variable_get('queue2civicrm_disable', false),
+ '#default_value' => variable_get('queue2civicrm_disable', FALSE),
);
$form['queue2civicrm_batch'] = array(
@@ -111,20 +112,21 @@
// We only want to initialize the SmashPig stuff once, so we use the default
// config section rather than a processor-specific section. This means all
// processors have to use the same database for pending donation info.
- wmf_common_create_smashpig_context( 'queue2civicrm' );
+ wmf_common_create_smashpig_context('queue2civicrm');
wmf_civicrm_boost_performance();
- if ( variable_get( 'queue2civicrm_disable', false ) ) {
- watchdog( 'queue2civicrm', 'Job is disabled. Exiting.', NULL,
WATCHDOG_INFO );
+ if (variable_get('queue2civicrm_disable', FALSE)) {
+ watchdog('queue2civicrm', 'Job is disabled. Exiting.', NULL,
WATCHDOG_INFO);
return;
}
- $batch_time = variable_get( 'queue2civicrm_batch_time', 0 );
+ $batch_time = variable_get('queue2civicrm_batch_time', 0);
+
// If we're running on a time limit, try to account for drush's startup time.
- if ( $batch_time > 0 && isset( $_SERVER['REQUEST_TIME'] ) ) {
+ if ($batch_time > 0 && isset($_SERVER['REQUEST_TIME'])) {
$already_elapsed = time() - $_SERVER['REQUEST_TIME'];
- if ( $already_elapsed < $batch_time ) {
+ if ($already_elapsed < $batch_time) {
$batch_time = $batch_time - $already_elapsed;
}
}
@@ -132,9 +134,12 @@
$consumer = new DonationQueueConsumer(
'donations',
$batch_time,
- variable_get( 'queue2civicrm_batch', 0 )
+ 50//variable_get('queue2civicrm_batch', 0)
);
+
+ $processingStartTime = microtime(TRUE);
$processed = $consumer->dequeueMessages();
+ $processingEndTime = microtime(TRUE);
/**
* this may some day supersede the process counts handled above...
@@ -152,6 +157,7 @@
'metrics_reporting_prometheus_path', '/var/spool/prometheus'
);
$DonationStats = new DonationStats();
+ $DonationStats->recordBatchProcessingTime($processingEndTime -
$processingStartTime);
$DonationStats->exportToPrometheus("donations", $prometheusPath);
if ($processed > 0) {
--
To view, visit https://gerrit.wikimedia.org/r/392425
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id053e7e6392dd480d59b933d47eea99d421be3c3
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/crm
Gerrit-Branch: master
Gerrit-Owner: Jgleeson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
