Gergő Tisza has uploaded a new change for review.
https://gerrit.wikimedia.org/r/226639
Change subject: Sample StatsD messages when instructed
......................................................................
Sample StatsD messages when instructed
Bug: T106457
Change-Id: I8ddb5a53dcbaf398f5a114d2a4a862842e980db4
---
M autoload.php
M includes/GlobalFunctions.php
M includes/libs/BufferingStatsdDataFactory.php
A includes/libs/SamplingStatsdClient.php
A tests/phpunit/includes/libs/SamplingStatsdClientTest.php
5 files changed, 182 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/39/226639/1
diff --git a/autoload.php b/autoload.php
index a8940c4..2f3fdd1 100644
--- a/autoload.php
+++ b/autoload.php
@@ -1056,6 +1056,7 @@
'SQLiteField' => __DIR__ . '/includes/db/DatabaseSqlite.php',
'SVGMetadataExtractor' => __DIR__ .
'/includes/media/SVGMetadataExtractor.php',
'SVGReader' => __DIR__ . '/includes/media/SVGMetadataExtractor.php',
+ 'SamplingStatsdClient' => __DIR__ .
'/includes/libs/SamplingStatsdClient.php',
'Sanitizer' => __DIR__ . '/includes/Sanitizer.php',
'SavepointPostgres' => __DIR__ . '/includes/db/DatabasePostgres.php',
'ScopedCallback' => __DIR__ . '/includes/libs/ScopedCallback.php',
diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php
index 167305d..b7b733d 100644
--- a/includes/GlobalFunctions.php
+++ b/includes/GlobalFunctions.php
@@ -1259,7 +1259,7 @@
$statsdHost = $statsdServer[0];
$statsdPort = isset( $statsdServer[1] ) ?
$statsdServer[1] : 8125;
$statsdSender = new SocketSender( $statsdHost,
$statsdPort );
- $statsdClient = new StatsdClient( $statsdSender, true,
false );
+ $statsdClient = new SamplingStatsdClient(
$statsdSender, true, false );
$statsdClient->send( $context->getStats()->getBuffer()
);
} catch ( Exception $ex ) {
MWExceptionHandler::logException( $ex );
diff --git a/includes/libs/BufferingStatsdDataFactory.php
b/includes/libs/BufferingStatsdDataFactory.php
index 3d7fad5..100d2a4 100644
--- a/includes/libs/BufferingStatsdDataFactory.php
+++ b/includes/libs/BufferingStatsdDataFactory.php
@@ -20,6 +20,7 @@
* @file
*/
+use Liuggio\StatsdClient\Entity\StatsdData;
use Liuggio\StatsdClient\Entity\StatsdDataInterface;
use Liuggio\StatsdClient\Factory\StatsdDataFactory;
@@ -75,6 +76,9 @@
return $entity;
}
+ /**
+ * @return StatsdData[]
+ */
public function getBuffer() {
return $this->buffer;
}
diff --git a/includes/libs/SamplingStatsdClient.php
b/includes/libs/SamplingStatsdClient.php
new file mode 100644
index 0000000..27ae240
--- /dev/null
+++ b/includes/libs/SamplingStatsdClient.php
@@ -0,0 +1,133 @@
+<?php
+/**
+ * Copyright 2015
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+use Liuggio\StatsdClient\StatsdClient;
+use Liuggio\StatsdClient\Entity\StatsdData;
+use Liuggio\StatsdClient\Entity\StatsdDataInterface;
+
+/**
+ * A statsd client that applies the sampling rate to the data items before
sending them.
+ *
+ * @since 1.26
+ */
+class SamplingStatsdClient extends StatsdClient {
+ /**
+ * Sets sampling rate for all items in $data.
+ * The sample rate specified in a StatsdData entity overrides the
sample rate specified here.
+ *
+ * {@inheritDoc}
+ */
+ public function appendSampleRate( $data, $sampleRate = 1 )
+ {
+ if ( $sampleRate < 1 ) {
+ array_walk( $data, function( &$item, $key ) use (
$sampleRate ) {
+ /** @var $item StatsdData */
+ if ( $item->getSampleRate() === 1 ) {
+ $item->setSampleRate( $sampleRate );
+ }
+ });
+ }
+
+ return $data;
+ }
+
+ /**
+ * Sample the metrics according to their sample rate and send the
remaining ones.
+ *
+ * {@inheritDoc}
+ */
+ public function send( $data, $sampleRate = 1 )
+ {
+ if ( !is_array( $data ) ) {
+ $data = array( $data );
+ }
+ if ( empty( $data ) ) {
+ return;
+ }
+ foreach ( $data as $item ) {
+ if ( !( $item instanceof StatsdDataInterface ) ) {
+ throw new InvalidArgumentException(
+ 'SamplingStatsdClient does not accept
stringified messages' );
+ }
+ }
+
+ // add sampling
+ if ( $sampleRate < 1 ) {
+ $data = $this->appendSampleRate( $data, $sampleRate );
+ }
+ $data = $this->sampleData( $data );
+
+ $messages = array_map( 'strval', $data );
+
+ // reduce number of packets
+ if ( $this->getReducePacket() ) {
+ $data = $this->reduceCount( $data );
+ }
+ //failures in any of this should be silently ignored if ..
+ $written = 0;
+ try {
+ $fp = $this->getSender()->open();
+ if ( !$fp ) {
+ return;
+ }
+ foreach ( $messages as $message ) {
+ $written += $this->getSender()->write( $fp,
$message );
+ }
+ $this->getSender()->close( $fp );
+ } catch ( \Exception $e ) {
+ $this->throwException( $e );
+ }
+
+ return $written;
+ }
+
+ /**
+ * Throw away some of the data according to the sample rate.
+ * @param StatsdDataInterface[] $data
+ * @return array
+ * @throws LogicException
+ */
+ protected function sampleData( $data ) {
+ $newData = array();
+ $mt_rand_max = mt_getrandmax();
+ foreach ( $data as $item ) {
+ $samplingRate = $item->getSampleRate();
+ if ($samplingRate <= 0.0 || 1.0 < $samplingRate) {
+ throw new \LogicException('Sampling rate shall
be within ]0, 1]');
+ }
+ if ( $samplingRate === 1 || mt_rand() / $mt_rand_max <=
$samplingRate ) {
+ $newData[] = $item;
+ }
+ }
+ return $newData;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ protected function throwException( \Exception $exception )
+ {
+ if ( !$this->getFailSilently() ) {
+ throw $exception;
+ }
+ }
+}
diff --git a/tests/phpunit/includes/libs/SamplingStatsdClientTest.php
b/tests/phpunit/includes/libs/SamplingStatsdClientTest.php
new file mode 100644
index 0000000..be6732d
--- /dev/null
+++ b/tests/phpunit/includes/libs/SamplingStatsdClientTest.php
@@ -0,0 +1,43 @@
+<?php
+
+use Liuggio\StatsdClient\Entity\StatsdData;
+
+class SamplingStatsdClientTest extends PHPUnit_Framework_TestCase {
+ /**
+ * @dataProvider samplingDataProvider
+ */
+ public function testSampling( $data, $sampleRate, $seed, $expectWrite )
{
+ $sender = $this->getMock(
'Liuggio\StatsdClient\Sender\SenderInterface' );
+ $sender->expects( $this->any() )->method( 'open' )->will(
$this->returnValue( true ) );
+ if ( $expectWrite ) {
+ $sender->expects( $this->once() )->method( 'write' )
+ ->with( $this->anything(), $this->equalTo(
$data ) );
+ } else {
+ $sender->expects( $this->never() )->method( 'write' );
+ }
+ mt_srand( $seed );
+ $client = new SamplingStatsdClient( $sender );
+ $client->send( $data, $sampleRate );
+ }
+
+ public function samplingDataProvider() {
+ $unsampled = new StatsdData();
+ $unsampled->setKey( 'foo' );
+ $unsampled->setValue( 1 );
+
+ $sampled = new StatsdData();
+ $sampled->setKey( 'foo' );
+ $sampled->setValue( 1 );
+ $sampled->setSampleRate( '0.1' );
+
+ return array(
+ // $data, $sampleRate, $seed, $expectWrite
+ array( $unsampled, 1, 0 /*0.44*/, $unsampled ),
+ array( $sampled, 1, 0 /*0.44*/, null ),
+ array( $sampled, 1, 4 /*0.03*/, $sampled ),
+ array( $unsampled, 0.1, 4 /*0.03*/, $sampled ),
+ array( $sampled, 0.5, 0 /*0.44*/, null ),
+ array( $sampled, 0.5, 4 /*0.03*/, $sampled ),
+ );
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/226639
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8ddb5a53dcbaf398f5a114d2a4a862842e980db4
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Gergő Tisza <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
