MaxSem has submitted this change and it was merged.
Change subject: Count pages with geo tags
......................................................................
Count pages with geo tags
Bug: T149722
Bug: T148812
Change-Id: I1fa2d69549af5527105efa4403329182399ab238
---
M bin/hourly.sh
M config.json
A geo-tag-counts.php
A src/Graphite.php
A src/Mysql.php
M vendor/composer/autoload_classmap.php
M vendor/composer/autoload_static.php
7 files changed, 120 insertions(+), 0 deletions(-)
Approvals:
MaxSem: Verified; Looks good to me, approved
diff --git a/bin/hourly.sh b/bin/hourly.sh
index 8ba6b00..d3f53c2 100755
--- a/bin/hourly.sh
+++ b/bin/hourly.sh
@@ -3,3 +3,4 @@
BASEDIR=`dirname "$0"`/..
/usr/bin/php $BASEDIR/tracking-category-count.php
+/usr/bin/php $BASEDIR/geo-tag-counts.php
diff --git a/config.json b/config.json
index 8c3c83c..07ad537 100644
--- a/config.json
+++ b/config.json
@@ -4,5 +4,11 @@
"categories": {
"kartographer-tracking-category": "kartographer.pages.%WIKI%.hourly",
"graph-tracking-category": "graph.pages.%WIKI%.hourly"
+ },
+ "geoCoordinates": {
+ "contentNamespaces": {
+ "default": [ 0 ],
+ "commonswiki": [ 6 ]
+ }
}
}
diff --git a/geo-tag-counts.php b/geo-tag-counts.php
new file mode 100644
index 0000000..37495d3
--- /dev/null
+++ b/geo-tag-counts.php
@@ -0,0 +1,59 @@
+<?php
+
+namespace DiscoveryStats;
+
+require_once( __DIR__ . '/vendor/autoload.php' );
+
+$config = json_decode( file_get_contents( __DIR__ . '/config.json' ) );
+$wikiBlacklist = [
+ 'labswiki',
+ 'labtestwiki',
+];
+
+$matrix = new SiteMatrix();
+$db = Mysql::connect( '/etc/mysql/conf.d/analytics-research-client.cnf',
+ 'analytics-store.eqiad.wmnet'
+);
+$graphite = new Graphite( $config );
+
+foreach ( $matrix->getSites() as $site ) {
+ $dbName = $site->getDbName();
+ // Can't quote it, have to validate
+ if ( !preg_match( '/^[a-z0-9_]+$/', $dbName ) ) {
+ throw new \Exception( "Invalid database '$dbName'" );
+ }
+ if ( $site->isPrivate() || in_array( $dbName, $wikiBlacklist ) ) {
+ continue;
+ }
+
+ query( "USE $dbName" );
+ $siteKey = $site->getFamily() . '.' . $site->getCode();
+
+ $res = query( 'SELECT count(*) AS num FROM geo_tags WHERE gt_primary=1' );
+ if ( $res && ( $row = $res->fetch() ) ) {
+ $graphite->record( "geodata.pages.$siteKey.hourly", $row['num'] );
+ }
+
+ $ns = isset( $config->geoCoordinates->contentNamespaces->$dbName )
+ ? $config->geoCoordinates->contentNamespaces->$dbName
+ : $config->geoCoordinates->contentNamespaces->default;
+ $ns = implode( ', ', $ns );
+ $res = query( 'SELECT count(*) AS num FROM geo_tags, page WHERE
page_id=gt_page_id '
+ . "AND page_namespace IN ($ns) AND gt_primary=1"
+ );
+ if ( $res && ( $row = $res->fetch() ) ) {
+ $graphite->record( "geodata.content.$siteKey.hourly", $row['num'] );
+ }
+}
+
+function query( $sql ) {
+ global $db;
+
+ $res = $db->query( $sql );
+ if ( !$res ) {
+ $err = $db->errorInfo();
+ throw new \Exception( "{$err[0]}: {$err[2]}" );
+ }
+
+ return $res;
+}
diff --git a/src/Graphite.php b/src/Graphite.php
new file mode 100644
index 0000000..7a65164
--- /dev/null
+++ b/src/Graphite.php
@@ -0,0 +1,26 @@
+<?php
+
+namespace DiscoveryStats;
+
+class Graphite {
+ /** @var string */
+ private $host;
+ /** @var int */
+ private $port;
+ /* @var int */
+ private $timestamp;
+
+ public function __construct( $config ) {
+ $this->host = $config->graphiteHost;
+ $this->port = $config->graphitePort;
+ $this->timestamp = time();
+ }
+
+ public function record( $metric, $value ) {
+ $packet = "{$metric} {$value} {$this->timestamp}";
+ $nc = "nc -q0 {$this->host} {$this->port}";
+ $command = "echo \"$packet\" | $nc";
+
+ exec( $command );
+ }
+}
diff --git a/src/Mysql.php b/src/Mysql.php
new file mode 100644
index 0000000..271c398
--- /dev/null
+++ b/src/Mysql.php
@@ -0,0 +1,24 @@
+<?php
+
+namespace DiscoveryStats;
+
+use Exception;
+use PDO;
+
+class Mysql {
+ /**
+ *
+ */
+ public static function connect( $config, $host ) {
+ $ini = parse_ini_file( $config );
+
+ if ( !$ini ) {
+ throw new Exception( "Error opening mysql config $config" );
+ }
+
+ return new PDO( "mysql:host=$host",
+ $ini['user'],
+ $ini['password']
+ );
+ }
+}
diff --git a/vendor/composer/autoload_classmap.php
b/vendor/composer/autoload_classmap.php
index 19c5ac2..a23625b 100644
--- a/vendor/composer/autoload_classmap.php
+++ b/vendor/composer/autoload_classmap.php
@@ -7,6 +7,8 @@
return array(
'DiscoveryStats\\Api' => $baseDir . '/src/Api.php',
+ 'DiscoveryStats\\Graphite' => $baseDir . '/src/Graphite.php',
+ 'DiscoveryStats\\Mysql' => $baseDir . '/src/Mysql.php',
'DiscoveryStats\\NormalSite' => $baseDir . '/src/NormalSite.php',
'DiscoveryStats\\Site' => $baseDir . '/src/Site.php',
'DiscoveryStats\\SiteMatrix' => $baseDir . '/src/SiteMatrix.php',
diff --git a/vendor/composer/autoload_static.php
b/vendor/composer/autoload_static.php
index 43a02a2..9b64739 100644
--- a/vendor/composer/autoload_static.php
+++ b/vendor/composer/autoload_static.php
@@ -8,6 +8,8 @@
{
public static $classMap = array (
'DiscoveryStats\\Api' => __DIR__ . '/../..' . '/src/Api.php',
+ 'DiscoveryStats\\Graphite' => __DIR__ . '/../..' . '/src/Graphite.php',
+ 'DiscoveryStats\\Mysql' => __DIR__ . '/../..' . '/src/Mysql.php',
'DiscoveryStats\\NormalSite' => __DIR__ . '/../..' .
'/src/NormalSite.php',
'DiscoveryStats\\Site' => __DIR__ . '/../..' . '/src/Site.php',
'DiscoveryStats\\SiteMatrix' => __DIR__ . '/../..' .
'/src/SiteMatrix.php',
--
To view, visit https://gerrit.wikimedia.org/r/319262
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1fa2d69549af5527105efa4403329182399ab238
Gerrit-PatchSet: 3
Gerrit-Project: analytics/discovery-stats
Gerrit-Branch: production
Gerrit-Owner: MaxSem <[email protected]>
Gerrit-Reviewer: MaxSem <[email protected]>
Gerrit-Reviewer: Yurik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
