Joal has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/395732 )
Change subject: Update mediawiki-history endpoints
......................................................................
Update mediawiki-history endpoints
In order to match original wikistats data, some changes are needed
in druid queries served by AQS. This patch implements those changes
and does some unused-code/formatting cleaning.
Bug: T178478
Change-Id: Ia12b41e63009a0ff0f6edbebea6900c463dcc3e0
---
M sys/mediawiki-history-metrics.js
M sys/mediawiki-history-schemas.yaml
M test/features/mediawiki-history-metrics/fixtures.js
3 files changed, 29 insertions(+), 16 deletions(-)
Approvals:
Joal: Verified; Looks good to me, approved
diff --git a/sys/mediawiki-history-metrics.js b/sys/mediawiki-history-metrics.js
index 4d6068e..7e466f6 100644
--- a/sys/mediawiki-history-metrics.js
+++ b/sys/mediawiki-history-metrics.js
@@ -228,8 +228,13 @@
D.datasource,
A2D.granularity[rp.granularity],
druidUtil.makeAndFilter(
- [ druidQueriesBlocks.filter.pages ]
- .concat(eventsFiltersFromRequestParams(rp))),
+ [
+ druidQueriesBlocks.filter.pages,
+ druidUtil.makeNotFilter(
+ druidUtil.makeSelectorFilter(
+ D.dimension.otherTags,
+ D.otherTags.redirect))
+ ].concat(eventsFiltersFromRequestParams(rp))),
[
druidUtil.makeFilteredAggregation(
druidQueriesBlocks.filter.create,
@@ -334,8 +339,10 @@
D.datasource,
A2D.granularity[rp.granularity],
druidUtil.makeAndFilter(
- [ eventEntityFilter, digestGranularityFilter(rp.granularity) ]
- .concat(digestsFiltersFromRequestParams(rp))),
+ [
+ eventEntityFilter,
+ digestGranularityFilter(rp.granularity)
+ ].concat(digestsFiltersFromRequestParams(rp))),
[ eventsCountingAggregation(outputMetric) ],
[], // No post-aggregation
druidUtil.makeInterval(rp.start, rp.end)
@@ -384,8 +391,14 @@
D.datasource,
A2D.granularity[rp.granularity],
druidUtil.makeAndFilter(
- [ druidQueriesBlocks.filter.revisions,
druidQueriesBlocks.filter.create ]
- .concat(eventsFiltersFromRequestParams(rp))),
+ [
+ druidQueriesBlocks.filter.revisions,
+ druidQueriesBlocks.filter.create,
+ druidUtil.makeNotFilter(
+ druidUtil.makeSelectorFilter(
+ D.dimension.otherTags,
+ D.otherTags.deleted))
+ ].concat(eventsFiltersFromRequestParams(rp))),
[ aggregation ],
[], // No post-aggregation
druidUtil.makeInterval(rp.start, rp.end)
@@ -447,8 +460,10 @@
TOP_THRESHOLD, // Get top 100
outputMetric,
druidUtil.makeAndFilter(
- [ druidQueriesBlocks.filter.revisions,
druidQueriesBlocks.filter.create ]
- .concat(eventsFiltersFromRequestParams(rp))),
+ [
+ druidQueriesBlocks.filter.revisions,
+ druidQueriesBlocks.filter.create
+ ].concat(eventsFiltersFromRequestParams(rp))),
[ aggregation ],
[], // No post-aggregation
druidUtil.makeInterval(rp.start, rp.end)
diff --git a/sys/mediawiki-history-schemas.yaml
b/sys/mediawiki-history-schemas.yaml
index 1bb6ee7..b1bcd83 100644
--- a/sys/mediawiki-history-schemas.yaml
+++ b/sys/mediawiki-history-schemas.yaml
@@ -66,8 +66,8 @@
nonContent: non_content
otherTags:
- deletedDay: deleted_day
- deletedMonth: deleted_month
+ redirect: redirect
+ deleted: deleted
selfCreated: self_created
@@ -128,7 +128,3 @@
granularity-digest:
daily: daily_digest
monthly: monthly_digest
-
- granularity-deleted_currents:
- daily: [ 'deleted_day' ]
- monthly: [ 'deleted_day', 'deleted_month' ]
diff --git a/test/features/mediawiki-history-metrics/fixtures.js
b/test/features/mediawiki-history-metrics/fixtures.js
index 5aff921..e24a4a9 100644
--- a/test/features/mediawiki-history-metrics/fixtures.js
+++ b/test/features/mediawiki-history-metrics/fixtures.js
@@ -182,7 +182,8 @@
var makeNewPagesDruidQuery = function(granularity, additionalFilters) {
var defaultFilters = [
- { type: 'selector', dimension: 'event_entity', value: 'page' }
+ { type: 'selector', dimension: 'event_entity', value: 'page' },
+ { type: 'not', field: { type: 'selector', dimension: 'other_tags',
value: 'redirect' } }
];
return {
queryType: 'timeseries',
@@ -550,7 +551,8 @@
var makeRevisionsDruidQuery = function(aggType, granularity,
additionalFilters) {
var defaultFilters = [
{ type: 'selector', dimension: 'event_entity', value: 'revision' },
- { type: 'selector', dimension: 'event_type', value: 'create' }
+ { type: 'selector', dimension: 'event_type', value: 'create' },
+ { type: 'not', field: { type: 'selector', dimension: 'other_tags',
value: 'deleted' } }
];
var agg;
--
To view, visit https://gerrit.wikimedia.org/r/395732
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia12b41e63009a0ff0f6edbebea6900c463dcc3e0
Gerrit-PatchSet: 1
Gerrit-Project: analytics/aqs
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
