Joal has uploaded a new change for review.
https://gerrit.wikimedia.org/r/264940
Change subject: Clean repo from mobile partition leftovers
......................................................................
Clean repo from mobile partition leftovers
Change mobile to text in comments
Remove mobile as checked partition in CamusChecker
Bug: T122651
Change-Id: I0c02da3fe2be0f9672967e2e587e7f84bdd048a2
---
M
refinery-camus/src/test/scala/org/wikimedia/analytics/refinery/camus/TestCamusStatusReader.scala
M
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
M
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsLegacyPageviewUDF.java
M
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
M
refinery-job/src/test/scala/org/wikimedia/analytics/refinery/job/TestCamusPartitionChecker.scala
5 files changed, 10 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery/source
refs/changes/40/264940/1
diff --git
a/refinery-camus/src/test/scala/org/wikimedia/analytics/refinery/camus/TestCamusStatusReader.scala
b/refinery-camus/src/test/scala/org/wikimedia/analytics/refinery/camus/TestCamusStatusReader.scala
index acafdd4..dde5dce 100644
---
a/refinery-camus/src/test/scala/org/wikimedia/analytics/refinery/camus/TestCamusStatusReader.scala
+++
b/refinery-camus/src/test/scala/org/wikimedia/analytics/refinery/camus/TestCamusStatusReader.scala
@@ -17,16 +17,15 @@
"A CamusStatusReader" should "read EtlKey values in offset-m-XXXXX sequence
file" in {
- val file: String = camusHistoryTestFolder + "/" + runFolder + "/" +
"offsets-m-00001"
+ val file: String = camusHistoryTestFolder + "/" + runFolder + "/" +
"offsets-m-00035"
val path: Path = new Path(file)
-
val keys = cr.readEtlKeys(path)
- val expectedKey = new EtlKey("webrequest_mobile", "", 10,
1222082461L,1222082462L, 104667347L)
- expectedKey.setTime(1439661130000L)
+ val expectedKey = new EtlKey("webrequest_maps", "", 1, 29811L, 29812L,
1198780952L)
+ expectedKey.setTime(1439661148000L)
expectedKey.setServer("unknown_server")
expectedKey.setService("unknown_service")
-
+
keys.size should equal (1)
// Terrible hack because equality function returns false while objects are
equal ...
// TODO: investigate why equals method return false in that case
@@ -43,7 +42,6 @@
keys.size should equal (72)
keys.filter(_.getTopic.equals("webrequest_upload")).size should equal (12)
keys.filter(_.getTopic.equals("webrequest_text")).size should equal (12)
- keys.filter(_.getTopic.equals("webrequest_mobile")).size should equal (12)
keys.filter(_.getTopic.equals("webrequest_misc")).size should equal (12)
keys.filter(_.getTopic.equals("webrequest_maps")).size should equal (12)
}
@@ -58,7 +56,6 @@
val keys = cr.readEtlKeys(paths)
keys.size should equal (36)
- keys.filter(_.getTopic.equals("webrequest_mobile")).size should equal (12)
keys.filter(_.getTopic.equals("webrequest_misc")).size should equal (12)
keys.filter(_.getTopic.equals("webrequest_maps")).size should equal (12)
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
index 9a233fb..0977f57 100644
---
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
+++
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
@@ -51,7 +51,7 @@
* FROM
* wmf_raw.webrequest
* WHERE
- * webrequest_source = 'mobile'
+ * webrequest_source = 'text'
* AND year=2014
* AND month=12
* AND day=7
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsLegacyPageviewUDF.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsLegacyPageviewUDF.java
index 81ce6e0..a339cd2 100644
---
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsLegacyPageviewUDF.java
+++
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsLegacyPageviewUDF.java
@@ -39,7 +39,7 @@
* FROM
* wmf_raw.webrequest
* WHERE
- * webrequest_source = 'mobile'
+ * webrequest_source = 'text'
* AND year=2014
* AND month=12
* AND day=7
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
index d704a4f..8014b3b 100644
---
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
+++
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
@@ -47,7 +47,7 @@
* FROM
* wmf_raw.webrequest
* WHERE
- * webrequest_source = 'mobile'
+ * webrequest_source = 'text'
* AND year=2014
* AND month=12
* AND day=7
diff --git
a/refinery-job/src/test/scala/org/wikimedia/analytics/refinery/job/TestCamusPartitionChecker.scala
b/refinery-job/src/test/scala/org/wikimedia/analytics/refinery/job/TestCamusPartitionChecker.scala
index 3694850..e6839e6 100644
---
a/refinery-job/src/test/scala/org/wikimedia/analytics/refinery/job/TestCamusPartitionChecker.scala
+++
b/refinery-job/src/test/scala/org/wikimedia/analytics/refinery/job/TestCamusPartitionChecker.scala
@@ -83,7 +83,7 @@
// correct Whitelist, no blacklist --> Should work, one topic in
historical data needs to be left aside
CamusPartitionChecker.props.setProperty(CamusPartitionChecker.WHITELIST_TOPICS,
-
"webrequest_maps,webrequest_mobile,webrequest_text,webrequest_upload,webrequest_misc")
+ "webrequest_maps,webrequest_text,webrequest_upload,webrequest_misc")
val topicsAndHours = CamusPartitionChecker.getTopicsAndHoursToFlag(path)
@@ -141,7 +141,7 @@
// correct Whitelist/blacklist config
CamusPartitionChecker.props.setProperty(CamusPartitionChecker.WHITELIST_TOPICS,
-
"webrequest_maps,webrequest_mobile,webrequest_text,webrequest_upload,webrequest_misc")
+ "webrequest_maps,webrequest_text,webrequest_upload,webrequest_misc")
val topicsAndHours = CamusPartitionChecker.getTopicsAndHoursToFlag(path)
@@ -157,7 +157,7 @@
// correct Whitelist/blacklist config
CamusPartitionChecker.props.setProperty(CamusPartitionChecker.WHITELIST_TOPICS,
-
"webrequest_maps,webrequest_mobile,webrequest_text,webrequest_upload,webrequest_misc")
+ "webrequest_maps,webrequest_text,webrequest_upload,webrequest_misc")
intercept[IllegalStateException] {
CamusPartitionChecker.getTopicsAndHoursToFlag(path)
--
To view, visit https://gerrit.wikimedia.org/r/264940
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0c02da3fe2be0f9672967e2e587e7f84bdd048a2
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery/source
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
