jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/398651 )
Change subject: Introduce ScoreStorage and its Sql implementetion
......................................................................
Introduce ScoreStorage and its Sql implementetion
It will take over some works of the Cache.php
Bug: T181334
Change-Id: I94025f3ee38ee723a4f9e8c08d353be27b3d7558
---
M includes/ServiceWiring.php
A includes/Storage/ScoreStorage.php
A includes/Storage/SqlScoreStorage.php
A tests/phpunit/includes/Storage/SqlScoreStorageTest.php
4 files changed, 304 insertions(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Awight: Looks good to me, approved
diff --git a/includes/ServiceWiring.php b/includes/ServiceWiring.php
index 458c902..0d114ad 100644
--- a/includes/ServiceWiring.php
+++ b/includes/ServiceWiring.php
@@ -19,6 +19,7 @@
use MediaWiki\Logger\LoggerFactory;
use MediaWiki\MediaWikiServices;
use ORES\Storage\SqlModelLookup;
+use ORES\Storage\SqlScoreStorage;
return [
'ORESModelLookup' => function ( MediaWikiServices $services ) {
@@ -31,6 +32,13 @@
$services->getMainWANObjectCache(),
LoggerFactory::getInstance( 'ORES' )
);
- }
+ },
+
+ 'ORESScoreStorage' => function ( MediaWikiServices $services ) {
+ return new SqlScoreStorage(
+ $services->getDBLoadBalancer(),
+ $services->getService( 'ORESModelLookup' )
+ );
+ },
];
diff --git a/includes/Storage/ScoreStorage.php
b/includes/Storage/ScoreStorage.php
new file mode 100644
index 0000000..fe9a79c
--- /dev/null
+++ b/includes/Storage/ScoreStorage.php
@@ -0,0 +1,35 @@
+<?php
+/**
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace ORES\Storage;
+
+/**
+ * Service interface to store score data in storage.
+ *
+ * @license GPL-2.0+
+ */
+interface ScoreStorage {
+
+ /**
+ * Save scores to the database
+ *
+ * @param array[] $scores in the same structure as is returned by ORES.
+ * @param callable $errorCallback This callback is called when we
cannot parse a revision score
+ * response. The signature is errorCallback( string
$errorMessage, string $revisionID )
+ */
+ public function storeScores( $scores, callable $errorCallback = null );
+
+}
diff --git a/includes/Storage/SqlScoreStorage.php
b/includes/Storage/SqlScoreStorage.php
new file mode 100644
index 0000000..472faa4
--- /dev/null
+++ b/includes/Storage/SqlScoreStorage.php
@@ -0,0 +1,77 @@
+<?php
+/**
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace ORES\Storage;
+
+use InvalidArgumentException;
+use ORES\Parser\ScoreParser;
+use RuntimeException;
+use Wikimedia\Rdbms\LoadBalancer;
+
+class SqlScoreStorage implements ScoreStorage {
+
+ private $loadBalancer;
+
+ private $modelLookup;
+
+ public function __construct(
+ LoadBalancer $loadBalancer,
+ ModelLookup $modelLookup
+ ) {
+ $this->loadBalancer = $loadBalancer;
+ $this->modelLookup = $modelLookup;
+ }
+
+ /**
+ * @see ModelLookup::getModelId()
+ *
+ * @param array[] $scores
+ * @param callable $errorCallback
+ * @return int
+ */
+ public function storeScores( $scores, callable $errorCallback = null ) {
+ // TODO: Make it an argument and deprecate the whole config
variable
+ global $wgOresModelClasses;
+
+ if ( $errorCallback === null ) {
+ $errorCallback = function ( $mssg, $revision ) {
+ throw new RuntimeException( "Model contains an
error for $revision: $mssg" );
+ };
+ }
+
+ $dbData = [];
+
+ $scoreParser = new ScoreParser( $this->modelLookup,
$wgOresModelClasses );
+ foreach ( $scores as $revision => $revisionData ) {
+ try {
+ $dbDataPerRevision =
$scoreParser->processRevision( $revision, $revisionData );
+ } catch ( InvalidArgumentException $exception ) {
+ call_user_func( $errorCallback,
$exception->getMessage(), $revision );
+ continue;
+ }
+
+ $dbData = array_merge( $dbData, $dbDataPerRevision );
+ }
+
+ $this->loadBalancer->getConnection( DB_MASTER )->insert(
+ 'ores_classification',
+ $dbData,
+ __METHOD__,
+ [ 'IGNORE' ]
+ );
+ }
+
+}
diff --git a/tests/phpunit/includes/Storage/SqlScoreStorageTest.php
b/tests/phpunit/includes/Storage/SqlScoreStorageTest.php
new file mode 100644
index 0000000..12dd8af
--- /dev/null
+++ b/tests/phpunit/includes/Storage/SqlScoreStorageTest.php
@@ -0,0 +1,183 @@
+<?php
+
+namespace ORES\Tests;
+
+use MediaWiki\MediaWikiServices;
+use MediaWikiLangTestCase;
+use MWException;
+use ORES\Storage\HashModelLookup;
+use ORES\Storage\SqlScoreStorage;
+
+/**
+ * @group ORES
+ * @group Database
+ * @covers ORES\Storage\SqlScoreStorage
+ */
+class SqlScoreStorageTest extends MediaWikiLangTestCase {
+
+ const GOODFAITH = 1;
+ const REVERTED = 2;
+ const DAMAGING = 3;
+
+ /**
+ * @var SqlScoreStorage
+ */
+ protected $storage;
+
+ protected function setUp() {
+ parent::setUp();
+
+ $this->tablesUsed[] = 'ores_classification';
+ $modelData = [
+ 'reverted' => [ 'id' => self::REVERTED, 'version' =>
'0.0.1' ],
+ 'damaging' => [ 'id' => self::DAMAGING, 'version' =>
'0.0.2' ],
+ 'goodfaith' => [ 'id' => self::GOODFAITH, 'version' =>
'0.0.3' ],
+ ];
+ $this->storage = new SqlScoreStorage(
+ MediaWikiServices::getInstance()->getDBLoadBalancer(),
+ new HashModelLookup( $modelData )
+ );
+ }
+
+ public function storeScoresProvider() {
+ return [
+ [
+ [
+ 'wiki' => [
+ 'models' => [
+ 'damaging' => [
+ 'version' =>
'0.0.2',
+ ],
+ 'goodfaith' => [
+ 'version' =>
'0.0.3',
+ ],
+ ],
+ 'scores' => [
+ '12345' => [
+ 'damaging' => [
+ 'score'
=> [
+
'prediction' => false,
+
'probability' => [
+
'false' => 0.933,
+
'true' => 0.067,
+
],
+ ],
+ ],
+ 'reverted' => [
+ 'score'
=> [
+
'prediction' => true,
+
'probability' => [
+
'false' => 0.124,
+
'true' => 0.876,
+
],
+ ],
+ ],
+ ],
+ ],
+ ],
+ ],
+ [
+ (object)[
+ 'oresc_rev' => '12345',
+ 'oresc_model' =>
(string)self::REVERTED,
+ 'oresc_class' => '1',
+ 'oresc_probability' => '0.876',
+ 'oresc_is_predicted' => '1'
+ ],
+ (object)[
+ 'oresc_rev' => '12345',
+ 'oresc_model' =>
(string)self::DAMAGING,
+ 'oresc_class' => '1',
+ 'oresc_probability' => '0.067',
+ 'oresc_is_predicted' => '0'
+ ],
+ ],
+ [ 12345 ],
+ ],
+ ];
+ }
+
+ /**
+ * @dataProvider storeScoresProvider
+ */
+ public function testStoreScores( $scores, $expected, $revIds ) {
+ $this->setMwGlobals( [
+ 'wgOresWikiId' => 'wiki',
+ ] );
+
+ $scores = $scores['wiki']['scores'];
+
+ $this->storage->storeScores( $scores );
+
+ $dbr = \wfGetDB( DB_REPLICA );
+ $res = $dbr->select(
+ 'ores_classification',
+ [
+ 'oresc_rev',
+ 'oresc_model',
+ 'oresc_class',
+ 'oresc_probability',
+ 'oresc_is_predicted'
+ ],
+ [ 'oresc_rev' => $revIds ],
+ __METHOD__,
+ 'ORDER BY oresc_probability'
+ );
+
+ $this->assertEquals( $expected, iterator_to_array( $res, false
) );
+ }
+
+ public function storeScoresInvalidProvider() {
+ return [
+ [
+ [ 1111 => [
+ 'non existing model' => [
+ 'score' => [
+ 'prediction' => true,
+ 'probability' => [
'true' => 0.9, 'false' => 0.1 ]
+ ],
+ ],
+ ] ]
+ ],
+ [
+ [ 12345 => [
+ 'damaging' => [
+ 'error' => [
+ 'message' => 'YAY, it
failed',
+ ],
+ ],
+ 'reverted' => [
+ 'score' => [
+ 'prediction' => false,
+ 'probability' => [
'true' => 0.3, 'false' => 0.7 ]
+ ],
+ ]
+ ] ]
+ ],
+ [
+ [ 12345 => [
+ 'reverted' => [
+ 'score' => [
+ 'prediction' => false,
+ 'probability' => [ 'It
should fail' => 0.3, 'false' => 0.7 ]
+ ],
+ ]
+ ] ]
+ ],
+ ];
+ }
+
+ /**
+ * @dataProvider storeScoresInvalidProvider
+ */
+ public function testProcessRevisionInvalid( array $data ) {
+ $this->setExpectedException( MWException::class,
'processRevisionInvalid failure' );
+ $this->storage->storeScores(
+ $data,
+ function () {
+ throw new MWException( 'processRevisionInvalid
failure' );
+ }
+ );
+ }
+
+}
--
To view, visit https://gerrit.wikimedia.org/r/398651
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I94025f3ee38ee723a4f9e8c08d353be27b3d7558
Gerrit-PatchSet: 3
Gerrit-Project: mediawiki/extensions/ORES
Gerrit-Branch: master
Gerrit-Owner: Ladsgroup <[email protected]>
Gerrit-Reviewer: Awight <[email protected]>
Gerrit-Reviewer: Catrope <[email protected]>
Gerrit-Reviewer: Ladsgroup <[email protected]>
Gerrit-Reviewer: Legoktm <[email protected]>
Gerrit-Reviewer: Thiemo Kreuz (WMDE) <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
