jenkins-bot has submitted this change and it was merged.

Change subject: Batch stats to statsd
......................................................................


Batch stats to statsd

This sends all stats at the end of each main loop run.
This will reduce the number of packets being sent to statsd!

Bug: T132327
Change-Id: I216eadc7eeb0d64a5ee782463c69017a61ab72ec
---
M redisJobChronService
M redisJobRunnerService
M src/RedisJobService.php
3 files changed, 75 insertions(+), 2 deletions(-)

Approvals:
  Aaron Schulz: Looks good to me, approved
  Addshore: Looks good to me, but someone else must approve
  jenkins-bot: Verified



diff --git a/redisJobChronService b/redisJobChronService
index e62d1ad..c17cac8 100755
--- a/redisJobChronService
+++ b/redisJobChronService
@@ -51,6 +51,7 @@
                        $memCurrent = memory_get_usage();
                        $this->debug( "Memory usage: $memCurrent bytes." );
                        $this->incrStats( "memory.$host", $memCurrent - 
$memLast );
+                       $this->sendStats();
                        $memLast = $memCurrent;
                }
        }
diff --git a/redisJobRunnerService b/redisJobRunnerService
index 433bdee..263a744 100755
--- a/redisJobRunnerService
+++ b/redisJobRunnerService
@@ -114,6 +114,7 @@
                        $memCurrent = memory_get_usage();
                        $this->debug( "Memory usage: $memCurrent bytes." );
                        $this->incrStats( "memory.$host", $memCurrent - 
$memLast );
+                       $this->sendStats();
                        $memLast = $memCurrent;
                }
        }
diff --git a/src/RedisJobService.php b/src/RedisJobService.php
index f479a32..af5f10f 100755
--- a/src/RedisJobService.php
+++ b/src/RedisJobService.php
@@ -4,6 +4,8 @@
  * Base class for job services with main() implemented by subclasses
  */
 abstract class RedisJobService {
+       const MAX_UDP_SIZE_STR = 512;
+
        /** @var array List of IP:<port> entries */
        protected $queueSrvs = array();
        /** @var array List of IP:<port> entries */
@@ -12,6 +14,8 @@
        protected $password;
        /** @var string IP address or hostname */
        protected $statsdHost;
+       /** @var array statsd packets pending sending */
+       private $statsdPackets = array();
        /** @var integer Port number */
        protected $statsdPort;
 
@@ -375,10 +379,77 @@
                if ( !$this->statsdHost || $delta == 0 ) {
                        return; // nothing to do
                }
+               $this->statsdPackets[] = $this->getStatPacket( $event, $delta );
+       }
 
-               static $format = "%s:%s|c\n";
-               $packet = sprintf( $format, "jobrunner.$event", $delta );
+       /**
+        * @param string $event
+        * @param integer $delta
+        *
+        * @return string
+        */
+       private function getStatPacket( $event, $delta ) {
+               return sprintf( "%s:%s|c\n", "jobrunner.$event", $delta );
+       }
 
+       /**
+        * Actually send the stats that have been saved in $this->statsdPackets
+        */
+       protected function sendStats() {
+               if ( $this->statsdHost ) {
+                       $packets = array_reduce(
+                               $this->statsdPackets,
+                               [ __CLASS__, 'reduceStatPackets' ],
+                               array()
+                       );
+                       foreach ( $packets as $packet ) {
+                               $this->sendStatsPacket( $packet );
+                       }
+               }
+               $this->statsdPackets = array();
+       }
+
+       /**
+        * This is called from this->sendStats()
+        *
+        * This is taken from StatsdClient::doReduce in 
https://github.com/liuggio/statsd-php-client
+        * @license MIT
+        * @copyright (c) Giulio De Donato
+        *
+        * This function reduces the number of packets,the reduced has the 
maximum dimension of self::MAX_UDP_SIZE_STR
+        * Reference:
+        * https://github.com/etsy/statsd/blob/master/README.md
+        * All metrics can also be batch send in a single UDP packet, separated 
by a newline character.
+        *
+        * @param string[] $reducedMetrics
+        * @param string $metric
+        *
+        * @return string[]
+        */
+       private static function reduceStatPackets( array $reducedMetrics, 
$metric ) {
+               $lastReducedMetric = end( $reducedMetrics );
+               if ( strlen( $metric ) >= self::MAX_UDP_SIZE_STR || 
$lastReducedMetric === false ) {
+                       $reducedMetrics[] = $metric; // full packet sized 
metric or first metric
+               } else {
+                       $newMetric = "$lastReducedMetric\n$metric";
+                       if ( strlen( $newMetric ) > self::MAX_UDP_SIZE_STR ) {
+                               // Merging into the last metric yields too 
large a packet
+                               $reducedMetrics[] = $metric;
+                       } else {
+                               // Merge this metric into the last one since 
the packet size is OK
+                               array_pop( $reducedMetrics );
+                               $reducedMetrics[] = $newMetric;
+                       }
+               }
+
+               return $reducedMetrics;
+       }
+
+       /**
+        * @param string $packet
+        * @return void
+        */
+       private function sendStatsPacket( $packet ) {
                if ( !function_exists( 'socket_create' ) ) {
                        $this->debug( 'No "socket_create" method available.' );
                        return;

-- 
To view, visit https://gerrit.wikimedia.org/r/259660
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I216eadc7eeb0d64a5ee782463c69017a61ab72ec
Gerrit-PatchSet: 9
Gerrit-Project: mediawiki/services/jobrunner
Gerrit-Branch: master
Gerrit-Owner: Addshore <addshorew...@gmail.com>
Gerrit-Reviewer: Aaron Schulz <asch...@wikimedia.org>
Gerrit-Reviewer: Addshore <addshorew...@gmail.com>
Gerrit-Reviewer: BryanDavis <bda...@wikimedia.org>
Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org>
Gerrit-Reviewer: Ori.livneh <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to