Ottomata has submitted this change and it was merged.
Change subject: Add get pageview_info udf and underlying functions
......................................................................
Add get pageview_info udf and underlying functions
Update getProjectFromHost function in PageviewDefinition
Add getDialectFromPath and getPageTitleFromUri in PageviewDefinition
Modify test data to test new functions
Adapt existing test functions to modified test data
Change-Id: Ieed48b6c520c09e62d3bc05085e67f11ed3a96b7
---
M
refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
M
refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java
M
refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
M refinery-core/src/test/resources/pageview_test_data.csv
A
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
D
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java
A
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java
D
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java
M
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
M
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java
M
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
11 files changed, 578 insertions(+), 166 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git
a/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
b/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
index 808c857..96f46e3 100644
---
a/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
+++
b/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
@@ -16,9 +16,9 @@
package org.wikimedia.analytics.refinery.core;
-import java.util.regex.Pattern;
-import java.util.HashSet;
import java.util.Arrays;
+import java.util.HashSet;
+import java.util.regex.Pattern;
/**
* Static functions to work with Wikimedia webrequest data.
@@ -105,6 +105,15 @@
"www",
"download"
));
+
+ /**
+ * Static values for project, dialect and article
+ */
+ public static final String UNKNOWN_PROJECT_VALUE = "-";
+ public static final String UNKNOWN_DIALECT_VALUE = "-";
+ public static final String UNKNOWN_PAGE_TITLE_VALUE = "-";
+ public static final String DEFAULT_DIALECT_VALUE = "default";
+
/**
* All API request uriPaths will contain this
@@ -212,7 +221,7 @@
* @return The project identifier in format [xxx.]xxxx (en.wikipedia or
wikisource for instance)
*/
public String getProjectFromHost(String uriHost) {
- if (uriHost == null) return "-";
+ if (uriHost == null) return UNKNOWN_PROJECT_VALUE;
String[] uri_parts = uriHost.toLowerCase().split("\\.");
switch (uri_parts.length) {
// case wikixxx.org
@@ -237,7 +246,181 @@
else
return uri_parts[0] + "." + uri_parts[3];
default:
- return "-";
+ return UNKNOWN_PROJECT_VALUE;
}
}
+
+ /**
+ * Normalize uriPath to maximize dialect and page title extraction
correctness
+ * Normalization export path if uriPath is a complete URL, and removes
double backslashes
+ *
+ * @param uriPath The url's path
+ * @return The normalized uriPath
+ */
+ private String normalizeUriPath(String uriPath) {
+ // Prevent null pointer exception
+ String normPath = (uriPath == null) ? "" : uriPath;
+
+ // Special case where full url ends-up in uriPath
+ // Extract path manually to prevent url encoding issues
+ int idxpathBeginning = 0;
+ if (normPath.startsWith("http"))
+ idxpathBeginning = normPath.indexOf("/", 9); // look for "/"
after http(s)://
+
+ int idxPathEnding = normPath.indexOf("?", idxpathBeginning); // look
for query "?" after path
+ idxPathEnding = (idxPathEnding < 0) ? normPath.length() :
idxPathEnding;
+
+ normPath = normPath.substring(idxpathBeginning, idxPathEnding);
+
+
+ // Clean uriPath of double backslashes
+ normPath = normPath.replaceAll("//+", "/");
+ normPath = normPath.trim();
+
+ return normPath;
+ }
+
+ /**
+ * Identifies the dialect from a pageview uriPath
+ * NOTE: Provides correct result only if used with is_pageview = true
+ *
+ * @param uriPath The url's path
+ * @return The dialect name (if any)
+ */
+ public String getDialectFromPath(String uriPath) {
+ // Normalize uriPath
+ String normPath = normalizeUriPath(uriPath);
+
+ // In case of api, unknown dialect
+ if (normPath.startsWith("/w/api.php"))
+ return UNKNOWN_DIALECT_VALUE;
+
+ // Default wiki urls, default dialect
+ if (normPath.equals("/") || normPath.equals("/wiki") ||
normPath.equals("/w")
+ || normPath.startsWith("/wiki/") || normPath.startsWith("/w/"))
+ return DEFAULT_DIALECT_VALUE;
+
+ // Special dialect case
+ // Dialect examples are zh-hans, zh-hk, or sr-rc or sr-el
+ // return dialect value if it contains a "-"
+ // or return default dialect if it doesn't (zh alone for instance)
+ // uriPath example with dialect /zh-hant/Wikipedia:首页
+ // uriPath example with default dialect /zh/Wikipedia:首页
+ // Manual extraction instead of regex for performance.
+ int startIdx = normPath.indexOf("/");
+ startIdx = (startIdx >= 0) ? (startIdx + 1) : startIdx;
+ int middleIdx = normPath.indexOf("-", startIdx);
+ int endIdx = normPath.indexOf("/", startIdx);
+ endIdx = (endIdx > 0) ? endIdx : (normPath.length());
+ if ((startIdx >= 0) && (startIdx < endIdx)) {
+ if ((middleIdx > 0) && (middleIdx < endIdx))
+ return normPath.substring(startIdx, endIdx);
+ else
+ return DEFAULT_DIALECT_VALUE;
+ }
+
+ // extraction failed, unknown dialect
+ return UNKNOWN_DIALECT_VALUE;
+
+ }
+
+ /**
+ * Extracts the page title name from uriPath
+ * NOTE: - Assumes that the page is not "index.*".
+ * - Provides correct result only if used with
+ * is_pageview = true (this method is supposedly
+ * called only by getPageTitleFromUri.
+ *
+ * @param path The url's path
+ * @return The page title name or UNKNOWN_PAGE_TITLE_VALUE
+ */
+ private String getPageTitleFromPath(String path) {
+
+ // If the path contains an anchor
+ // remove it from the result substring
+ int endIdx = path.indexOf("#");
+ endIdx = (endIdx > 0)?endIdx:(path.length());
+
+ // If path contains /wiki/, extract substring from there
+ // for instance: /wiki/horseshoe_crab -> horseshoe_crab
+ int startIdx = path.indexOf("/wiki/") ;
+ startIdx = (startIdx >= 0) ? (startIdx + "/wiki/".length()) : startIdx;
+ if ((startIdx >= 0) && (startIdx < endIdx))
+ return path.substring(startIdx, endIdx);
+
+ // If path contains /w/, extract substring from there
+ // for instance /w/horseshoe_crab -> horseshoe_crab
+ startIdx = path.indexOf("/w/");
+ startIdx = (startIdx >= 0) ? (startIdx + "/w/".length()) : startIdx;
+ if ((startIdx >= 0) && (startIdx < endIdx))
+ return path.substring(startIdx, endIdx);
+
+ // Else assume we are in /dialect/Page_title case
+ // for instance /zh-hant/Wikipedia:首页
+ // Find second "/" in path as substring beginning
+ startIdx = path.indexOf("/", path.indexOf("/") + 1);
+ startIdx = (startIdx >= 0) ? (startIdx + 1) : startIdx;
+ if ((startIdx > 0) && (startIdx < endIdx))
+ return path.substring(startIdx, endIdx);
+
+ //Case not covered, return unknown value
+ return UNKNOWN_PAGE_TITLE_VALUE;
+ }
+
+ /**
+ * Identifies a page title from a pageview uriPath and uriQuery.
+ * Normalize page title by lowering case and decoding URI characters.
+ * NOTE: Provides correct result only if used with is_pageview = true
+ *
+ * @param uriPath The url's path
+ * @param uriQuery The url's query
+ * @return The decoded page title name or UNKNOWN_PROJECT_VALUE
+ */
+ public String getPageTitleFromUri(String uriPath, String uriQuery) {
+ // Normalize uriPath
+ String normPath = normalizeUriPath(uriPath);
+
+ // General case of page title in path
+ boolean pathWiki = ((normPath.contains("/wiki/") && (!
normPath.contains("index.")))
+ || normPath.contains("/w/index.php/"));
+
+ String titleQueryParam = null;
+ String pageQueryParam = null;
+
+
+ // Extract first instance of title and page query parameters
+ // Done manually to decode using our own percent decoder
+ // Explicitly replace '+' by '_' to get closer to page titles from path
+ uriQuery = (uriQuery == null) ? "" : uriQuery; // Prevent null pointer
exception
+ uriQuery = (uriQuery.startsWith("?")) ? uriQuery.substring(1):
uriQuery; // Remove ? if any
+ final String[] pairs = uriQuery.split("&");
+ for (String pair : pairs) {
+ final int idx = pair.indexOf("=");
+ final String key = idx > 0 ? pair.substring(0, idx) : pair;
+ if ((key.equals("title") && (titleQueryParam == null)))
+ titleQueryParam = idx > 0 && pair.length() > idx + 1 ?
pair.substring(idx + 1).replace("+", "_") : null;
+ if ((key.equals("page")) && (pageQueryParam == null))
+ pageQueryParam = idx > 0 && pair.length() > idx + 1 ?
pair.substring(idx + 1).replace("+", "_") : null;
+ if ((titleQueryParam != null) && (pageQueryParam != null))
+ break;
+ }
+
+ String pageTitle = UNKNOWN_PAGE_TITLE_VALUE;
+
+ // Depending on case, extract page title from path or query parameter
+
+ if (titleQueryParam != null)
+ pageTitle = titleQueryParam;
+ else if (pathWiki)
+ pageTitle = getPageTitleFromPath(normPath);
+ else if (pageQueryParam != null)
+ pageTitle = pageQueryParam;
+ else if (! normPath.contains("index."))
+ pageTitle = getPageTitleFromPath(normPath);
+
+ // Normalize Decoding URL percent characters (if any)
+ return PercentDecoder.decode(pageTitle);
+
+ }
+
}
diff --git
a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java
b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java
index ca5d6c7..e83a872 100644
---
a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java
+++
b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java
@@ -34,6 +34,8 @@
public void testIsLegacyPageview(
String test_description,
String project,
+ String dialect,
+ String page_title,
boolean is_pageview,
boolean is_legacy_pageview,
boolean is_app_pageview,
diff --git
a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
index 684fead..d885f1e 100644
---
a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
+++
b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
@@ -33,6 +33,8 @@
public void testIsPageview(
String test_description,
String project,
+ String dialect,
+ String page_title,
boolean is_pageview,
boolean is_legacy_pageview,
boolean is_app_pageview,
@@ -68,6 +70,8 @@
public void testIsAppPageview(
String test_description,
String project,
+ String dialect,
+ String page_title,
boolean is_pageview,
boolean is_legacy_pageview,
boolean is_app_pageview,
@@ -101,6 +105,8 @@
public void testGetProjectFromHost(
String test_description,
String project,
+ String dialect,
+ String page_title,
boolean is_pageview,
boolean is_legacy_pageview,
boolean is_app_pageview,
@@ -120,4 +126,69 @@
PageviewDefinitionInstance.getProjectFromHost(uri_host)
);
}
+
+ @Test
+ @FileParameters(
+ value = "src/test/resources/pageview_test_data.csv",
+ mapper = CsvWithHeaderMapper.class
+ )
+ public void testGetDialectFromPath(
+ String test_description,
+ String project,
+ String dialect,
+ String page_title,
+ boolean is_pageview,
+ boolean is_legacy_pageview,
+ boolean is_app_pageview,
+ String ip_address,
+ String x_forwarded_for,
+ String uri_host,
+ String uri_path,
+ String uri_query,
+ String http_status,
+ String content_type,
+ String user_agent
+ ) {
+ PageviewDefinition PageviewDefinitionInstance =
PageviewDefinition.getInstance();
+ if (is_pageview) {
+ assertEquals(
+ test_description,
+ dialect,
+ PageviewDefinitionInstance.getDialectFromPath(uri_path)
+ );
+ }
+ }
+
+ @Test
+ @FileParameters(
+ value = "src/test/resources/pageview_test_data.csv",
+ mapper = CsvWithHeaderMapper.class
+ )
+ public void testGetPageTitleFromUri(
+ String test_description,
+ String project,
+ String dialect,
+ String page_title,
+ boolean is_pageview,
+ boolean is_legacy_pageview,
+ boolean is_app_pageview,
+ String ip_address,
+ String x_forwarded_for,
+ String uri_host,
+ String uri_path,
+ String uri_query,
+ String http_status,
+ String content_type,
+ String user_agent
+ ) {
+ PageviewDefinition PageviewDefinitionInstance =
PageviewDefinition.getInstance();
+ if (is_pageview) {
+ assertEquals(
+ test_description,
+ page_title,
+ PageviewDefinitionInstance.getPageTitleFromUri(uri_path,
uri_query)
+ );
+ }
+ }
+
}
\ No newline at end of file
diff --git a/refinery-core/src/test/resources/pageview_test_data.csv
b/refinery-core/src/test/resources/pageview_test_data.csv
index 49f9124..2add55a 100644
--- a/refinery-core/src/test/resources/pageview_test_data.csv
+++ b/refinery-core/src/test/resources/pageview_test_data.csv
@@ -1,32 +1,32 @@
-test_description,project,is_pageview,is_legacy_pageview,is_app_pageview,ip_address,x_forwarded_for,uri_host,uri_path,uri_query,http_status,content_type,user_agent
-Is Pageview -
Desktop,en.wikipedia,true,true,false,174.62.175.82,-,en.wikipedia.org,
/wiki/Horseshoe_crab,-,200,text/html, turnip
-Is Pageview – Desktop – locally cached
content,en.wikipedia,true,true,false,174.62.175.82,-,en.wikipedia.org,
/wiki/Horseshoe_crab,-,304,text/html, turnip
-Is Pageview – App -
Android,en.wikipedia,true,false,true,174.62.175.83,-,en.wikipedia.org,
/w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200,
application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google
Play
-is Pageview – App – iOS – old
version,en.wikipedia,true,false,true,174.62.175.83,-,en.wikipedia.org,
/w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200,
application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone)
-Is Pageview – App - iOS – new
version,en.wikipedia,true,false,true,174.62.175.83,-,en.wikipedia.org,
/w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200,
application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone)
-Is Pageview – Mobile
Web,en.wikipedia,true,true,false,174.62.175.84,-,en.m.wikipedia.org,/wiki/Bernard_Manning,-,200,text/html,rutabaga
-Is Pageview – Desktop - Serbian
sr-ec,sr.wikipedia,true,false,false,174.62.175.85,-,sr.wikipedia.org,/sr-ec/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger
salute
-Is Pageview – Desktop - Serbian
sr-el,sr.wikipedia,true,false,false,174.62.175.86,-,sr.wikipedia.org,/sr-el/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger
salute
-Is Pageview – Desktop - Chinese
zh-cn,zh.wikipedia,true,false,false,174.62.175.87,-,zh.wikipedia.org,/zh-cn/Wikipedia:首页,-,200,text/html,Five-test
plan
-Is Pageview – Desktop - Chinese
zh-hans,zh.wikipedia,true,false,false,174.62.175.88,-,zh.wikipedia.org,/zh-hans/Wikipedia:首页,-,200,text/html,Five-test
plan
-Is Pageview – Desktop - Chinese
zh-hant,zh.wikipedia,true,false,false,174.62.175.89,-,zh.wikipedia.org,/zh-hant/Wikipedia:首页,-,200,text/html,Five-test
plan
-Is Pageview – Desktop - Chinese
zh-hk,zh.wikipedia,true,false,false,174.62.175.90,-,zh.wikipedia.org,/zh-hk/Wikipedia:foo,-,200,text/html,Five-test
plan
-Is Pageview – Desktop - Chinese
zh-mo,zh.wikipedia,true,false,false,174.62.175.91,-,zh.wikipedia.org,/zh-mo/Wikipedia:首页,-,200,text/html,Five-test
plan
-Is Pageview – Desktop - Chinese
zh-my,zh.wikipedia,true,false,false,174.62.175.92,-,zh.wikipedia.org,/zh-my/Wikipedia:首页,-,200,text/html,Five-test
plan
-Is Pageview – Desktop - Chinese
zh-sg,zh.wikipedia,true,false,false,174.62.175.93,-,zh.wikipedia.org,/zh-sg/Wikipedia:首页,-,200,text/html,Five-test
plan
-Is Pageview – Desktop - Chinese
zh-tw,zh.wikipedia,true,false,false,174.62.175.94,-,zh.wikipedia.org,/zh-tw/Wikipedia:首页,-,200,text/html,Five-test
plan
-Is Pageview –
Wikidata,wikidata,true,true,false,174.62.175.94,-,www.wikidata.org,/wiki/Q5651758,-,200,text/html,Five-test
plan
-Is Pageview –
MediaWiki,mediawiki,true,true,false,174.62.175.94,-,www.mediawiki.org,/wiki/Gerrit/git-review,-,200,text/html,Five-test
plan
-Is Pageview – iOS
search,en.wikipedia,true,false,false,174.62.175.94,-,en.wikipedia.org,/,?search=afdfsdfsd,200,text/html,Five-test
plan
-Is Not Pageview - http_status !=
200,en.wikipedia,false,true,false,174.62.175.95,-, en.wikipedia.org,
/wiki/Noppperrrrs,-,400,text/html ,turnip
-Is Not Pageview - content_type does not
match,en.wikipedia,false,true,false,174.62.175.96,-, en.wikipedia.org,
/wiki/Noppperrrrs,-,200, image/png, turnip
-Is Not Pageview - API stupidity: it outputs a 200 status code and text/html as
a MIME type on certain classes of
error,en.wikipedia,false,false,false,174.62.175.97,-, en.wikipedia.org,
/w/api.php,-,200, text/html, turnip
-Is Not Pageview – App request for non-page
content,en.wikipedia,false,false,false,174.62.175.98,-,en.wikipedia.org,/w/api.php,?action=query&format=json&titles=Foo&prop=pageimages&piprop=thumbnail&pithumbsize=96&pilimit=1
,200, application/json, WikipediaApp/1.2.3
-Is Not Pageview – Non-App request for page content,en.wikipedia,false,
false,false,174.62.175.99,-, en.wikipedia.org,
/w/api.php,?action=mobileview§ions=0,200, application/json, TributeApp/1.2.3
-Is Not Pageview – edit
attempt,en.wikipedia,false,true,false,174.62.175.82,-,en.wikipedia.org,
/wiki/Horseshoe_crab,?action=edit,200,text/html, turnip
-Is not pageview – non-wikidata raw
domain,wikipedia,false,false,false,174.62.175.82,-,www.wikipedia.org,-,-,200,text/html,
turnip
-Is Not Pageview – App – Android – Refresh,en.wikipedia,false,
false,false,174.62.175.82,-,en.wikipedia.org,
/w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200,
application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google
Play
-"Is Pageview - mobile -
wikimediafoundation",wikimediafoundation,true,true,false,174.62.175.82,-,m.wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip
-"Is Pageview - Desktop -
wikimediafoundation",wikimediafoundation,true,false,false,174.62.175.82,-,wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip
-Is Not Pageview - Desktop,test2.wikipedia,
true,true,false,174.62.175.82,-,test2.wikipedia.org,
/wiki/Horseshoe_crab,-,200,text/html, turnip
-Is Not Pageview -
Desktop,test.wikimediafoundation,true,true,false,174.62.175.82,-,test.wikimediafoundation.org,
/wiki/Horseshoe_crab,-,200,text/html, turnip
\ No newline at end of file
+test_description,project,dialect,page_title,is_pageview,is_legacy_pageview,is_app_pageview,ip_address,x_forwarded_for,uri_host,uri_path,uri_query,http_status,content_type,user_agent
+Is Pageview - Desktop,en.wikipedia,default,Horseshoe
crab,true,true,false,174.62.175.82,-,en.wikipedia.org,
/wiki/Horseshoe%20crab#anchor,-,200,text/html, turnip/
+Is Pageview – Desktop – locally cached
content,en.wikipedia,default,Horseshoe/crab,true,true,false,174.62.175.82,-,en.wikipedia.org,/wiki/Horseshoe/crab,-,304,text/html,
turnip
+Is Pageview – App -
Android,en.wikipedia,-,Hachiko_–_Eine_wunderbare_Freundschaft,true,false,true,174.62.175.83,-,en.wikipedia.org,/w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200,
application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google
Play
+is Pageview – App – iOS – old
version,en.wikipedia,-,Hachiko_–_Eine_wunderbare_Freundschaft,true,false,true,174.62.175.83,-,en.wikipedia.org,
/w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200,
application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone)
+Is Pageview – App - iOS – new
version,en.wikipedia,-,Hachiko_–_Eine_wunderbare_Freundschaft,true,false,true,174.62.175.83,-,en.wikipedia.org,
/w/api.php,?action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200,
application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone)
+Is Pageview – Mobile
Web,en.wikipedia,default,Bernard_Manning,true,true,false,174.62.175.84,-,en.m.wikipedia.org,/wiki/Bernard_Manning,-,200,text/html,rutabaga
+Is Pageview – Desktop - Serbian
sr-ec,sr.wikipedia,sr-ec,Историја_Срба_пре_Немањића,true,false,false,174.62.175.85,-,sr.wikipedia.org,/sr-ec/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger
salute
+Is Pageview – Desktop - Serbian
sr-el,sr.wikipedia,sr-el,Историја_Срба_пре_Немањића,true,false,false,174.62.175.86,-,sr.wikipedia.org,/sr-el/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger
salute
+Is Pageview – Desktop - Chinese
zh-cn,zh.wikipedia,zh-cn,Wikipedia:首页,true,false,false,174.62.175.87,-,zh.wikipedia.org,/zh-cn/Wikipedia:首页,-,200,text/html,Five-test
plan
+Is Pageview – Desktop - Chinese
zh-hans,zh.wikipedia,zh-hans,Wikipedia:首页,true,false,false,174.62.175.88,-,zh.wikipedia.org,/zh-hans/Wikipedia:首页,-,200,text/html,Five-test
plan
+Is Pageview – Desktop - Chinese
zh-hant,zh.wikipedia,zh-hant,Wikipedia:首页,true,false,false,174.62.175.89,-,zh.wikipedia.org,/zh-hant/Wikipedia:首页,-,200,text/html,Five-test
plan
+Is Pageview – Desktop - Chinese
zh-hk,zh.wikipedia,zh-hk,Wikipedia:foo,true,false,false,174.62.175.90,-,zh.wikipedia.org,/zh-hk/Wikipedia:foo,-,200,text/html,Five-test
plan
+Is Pageview – Desktop - Chinese
zh-mo,zh.wikipedia,zh-mo,Wikipedia:首页,true,false,false,174.62.175.91,-,zh.wikipedia.org,/zh-mo/Wikipedia:首页,-,200,text/html,Five-test
plan
+Is Pageview – Desktop - Chinese
zh-my,zh.wikipedia,zh-my,Wikipedia:首页,true,false,false,174.62.175.92,-,zh.wikipedia.org,/zh-my/Wikipedia:首页,-,200,text/html,Five-test
plan
+Is Pageview – Desktop - Chinese
zh-sg,zh.wikipedia,zh-sg,Wikipedia:首页,true,false,false,174.62.175.93,-,zh.wikipedia.org,/zh-sg/Wikipedia:首页,-,200,text/html,Five-test
plan
+"Is Pageview – Desktop - Chinese
zh-tw",zh.wikipedia,default,Wikipedia:首页,true,false,false,174.62.175.94,-,zh.wikipedia.org,/zh/Wikipedia:首页,-,200,text/html,"Five-test
plan"
+Is Pageview –
Wikidata,wikidata,default,Q5651758,true,true,false,174.62.175.94,-,www.wikidata.org,/wiki/Q5651758,-,200,text/html,Five-test
plan
+Is Pageview –
MediaWiki,mediawiki,default,Gerrit/git-review,true,true,false,174.62.175.94,-,www.mediawiki.org,/wiki/Gerrit/git-review,-,200,text/html,Five-test
plan
+Is Pageview – iOS
search,en.wikipedia,default,-,true,false,false,174.62.175.94,-,en.wikipedia.org,/,?search=afdfsdfsd,200,text/html,Five-test
plan
+Is Not Pageview - http_status !=
200,en.wikipedia,-,-,false,true,false,174.62.175.95,-, en.wikipedia.org,
/wiki/Noppperrrrs,-,400,text/html ,turnip
+Is Not Pageview - content_type does not
match,en.wikipedia,-,-,false,true,false,174.62.175.96,-, en.wikipedia.org,
/wiki/Noppperrrrs,-,200, image/png, turnip
+Is Not Pageview - API stupidity: it outputs a 200 status code and text/html as
a MIME type on certain classes of
error,en.wikipedia,-,-,false,false,false,174.62.175.97,-, en.wikipedia.org,
/w/api.php,-,200, text/html, turnip
+Is Not Pageview – App request for non-page
content,en.wikipedia,-,-,false,false,false,174.62.175.98,-,en.wikipedia.org,/w/api.php,?action=query&format=json&titles=Foo&prop=pageimages&piprop=thumbnail&pithumbsize=96&pilimit=1
,200, application/json, WikipediaApp/1.2.3
+Is Not Pageview – Non-App request for page content,en.wikipedia,-,-,false,
false,false,174.62.175.99,-,en.wikipedia.org,
/w/api.php,?action=mobileview§ions=0,200, application/json, TributeApp/1.2.3
+Is Not Pageview – edit
attempt,en.wikipedia,-,-,false,true,false,174.62.175.82,-,en.wikipedia.org,
/wiki/Horseshoe_crab,?action=edit,200,text/html, turnip
+Is not pageview – non-wikidata raw
domain,wikipedia,-,-,false,false,false,174.62.175.82,-,www.wikipedia.org,-,-,200,text/html,
turnip
+Is Not Pageview – App – Android – Refresh,en.wikipedia,-,-,false,
false,false,174.62.175.82,-,en.wikipedia.org,
/w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200,
application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google
Play
+"Is Pageview - mobile -
wikimediafoundation",wikimediafoundation,default,Horseshoe_crab,true,true,false,174.62.175.82,-,m.wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip
+"Is Pageview - Desktop -
wikimediafoundation",wikimediafoundation,default,Horseshoe_crab,true,false,false,174.62.175.82,-,wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip
+Is Not Pageview -
Desktop,test2.wikipedia,default,Horseshoe_crab,true,true,false,174.62.175.82,-,test2.wikipedia.org,
/wiki/Horseshoe_crab,-,200,text/html, turnip
+Is Not Pageview -
Desktop,test.wikimediafoundation,default,Horseshoe_crab,true,true,false,174.62.175.82,-,test.wikimediafoundation.org,
/wiki/Horseshoe_crab,-,200,text/html, turnip
\ No newline at end of file
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
new file mode 100644
index 0000000..fce507c
--- /dev/null
+++
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java
@@ -0,0 +1,153 @@
+/**
+ * Copyright (C) 2014 Wikimedia Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.wikimedia.analytics.refinery.hive;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.wikimedia.analytics.refinery.core.PageviewDefinition;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * A Hive UDF to identify pageview data in a map
+ * Map fields are project, dialect, article.
+ * NOTE: this udf only works well if
+ * (uri_host, uri_path, uri_query) comes from
+ * a webrequest having is_pageview = true
+ *
+ * <p>
+ * Hive Usage:
+ * ADD JAR /path/to/refinery-hive.jar;
+ * CREATE TEMPORARY FUNCTION get_pageview_info AS
+ * 'org.wikimedia.analytics.refinery.hive.PageviewInfoUDF';
+ * SELECT
+ * get_pageview_info(uri_host, uri_path, uri_query) as pageview_data,
+ * count(*) as cnt
+ * FROM
+ * wmf_raw.webrequest
+ * WHERE
+ * webrequest_source = 'mobile'
+ * AND year=2014
+ * AND month=12
+ * AND day=7
+ * AND hour=12
+ * AND is_pageview(uri_host, uri_path, uri_query, http_status,
content_type, user_agent)
+ * GROUP BY
+ * get_pageview_info(uri_host, uri_path, uri_query)
+ * ORDER BY cnt desc
+ * LIMIT 10
+ * ;
+ */
+@UDFType(deterministic = true)
+@Description(name = "get_pageview_info",
+ value = "_FUNC_(uri_host, uri_path, uri_query) - Returns the pageview
information map "
+ + "(project, dialect, article) for the pageview request.",
+ extended = "")
+public class GetPageviewInfoUDF extends GenericUDF {
+
+ public static final String PROJECT_KEY = "project";
+ public static final String DIALECT_KEY = "dialect";
+ public static final String PAGE_TITLE_KEY = "page_title";
+
+
+ Map<String, String> result;
+ private StringObjectInspector[] inputsOI = new StringObjectInspector[3];
+ private PageviewDefinition pageviewDefinition;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments)
+ throws UDFArgumentException {
+ // We need exactly 3 parameters
+ if (arguments == null || arguments.length != 3) {
+ throw new UDFArgumentLengthException("The function "
+ + "GetPageviewInfoUDF expects exactly 3 parameter");
+ }
+
+ // ... and the parameters have to be strings
+ for (int i = 0; i < 3; i++) {
+ if (!(arguments[i] instanceof StringObjectInspector)) {
+ throw new UDFArgumentTypeException(0, "Parameter "
+ + Integer.toString(i) + " to GetPageviewInfoUDF "
+ + "has to be a string");
+ }
+ inputsOI[i] = (StringObjectInspector) arguments[i];
+ }
+
+ result = new HashMap<>(3);
+ pageviewDefinition = PageviewDefinition.getInstance();
+
+ return ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ assert arguments != null : "Method 'evaluate' of GetPageviewInfoUDF "
+ + "called with null arguments array";
+ assert arguments.length == 3 : "Method 'evaluate' of "
+ + "GetPageviewInfoUDF called arguments of length "
+ + arguments.length + " (instead of 3)";
+ // arguments is an array with exactly 3 entry.
+
+ assert result != null : "Result object has not yet been initialized, "
+ + "but evaluate called";
+ // result map has been initialized.
+ assert pageviewDefinition != null : "PageviewDefinition object has not
yet been initialized, "
+ + "but evaluate called";
+ // pageviewDefinition has been initialized.
+
+ result.clear();
+
+ String uriHost =
inputsOI[0].getPrimitiveJavaObject(arguments[0].get());
+ String uriPath =
inputsOI[1].getPrimitiveJavaObject(arguments[1].get());
+ String uriQuery =
inputsOI[2].getPrimitiveJavaObject(arguments[2].get());
+
+ result.put(PROJECT_KEY,
pageviewDefinition.getProjectFromHost(uriHost));
+ result.put(DIALECT_KEY,
pageviewDefinition.getDialectFromPath(uriPath));
+ result.put(PAGE_TITLE_KEY,
pageviewDefinition.getPageTitleFromUri(uriPath, uriQuery));
+
+ return result;
+
+ }
+
+ @Override
+ public String getDisplayString(String[] arguments) {
+ String argument;
+ if (arguments == null) {
+ argument = "<arguments == null>";
+ } else if (arguments.length == 3) {
+ argument = arguments[0] + ", "
+ + arguments[1] + ", "
+ + arguments[2];
+ } else {
+ argument = "<arguments of length " + arguments.length + ">";
+ }
+ return "get_pageview_info(" + argument +")";
+ }
+}
\ No newline at end of file
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java
deleted file mode 100644
index 0552a84..0000000
---
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Copyright (C) 2014 Wikimedia Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.wikimedia.analytics.refinery.hive;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDF;
-import org.wikimedia.analytics.refinery.core.PageviewDefinition;
-
-
-/**
- * A Hive UDF to identify a Wikimedia webrequest pageview project.
- * NOTE: this udf only works well if the uri_host comes from
- * a webrequest having is_pageview = true
- *
- * <p>
- * Hive Usage:
- * ADD JAR /path/to/refinery-hive.jar;
- * CREATE TEMPORARY FUNCTION get_project AS
- * 'org.wikimedia.analytics.refinery.hive.IdentifyProjectUDF';
- * SELECT
- * get_project(uri_host) as project_qualifier,
- * count(*) as cnt
- * FROM
- * wmf_raw.webrequest
- * WHERE
- * webrequest_source = 'mobile'
- * AND year=2014
- * AND month=12
- * AND day=7
- * AND hour=12
- * AND is_pageview(uri_host, uri_path, uri_query, http_status,
content_type, user_agent)
- * GROUP BY
- * get_project(uri_host)
- * ORDER BY cnt desc
- * LIMIT 10
- * ;
- */
-@Description(name = "get_project",
- value = "_FUNC_(uri_host) - Returns the project identifier for the
pageview request.",
- extended = "")
-public class GetProjectUDF extends UDF {
- public String evaluate(String uriHost) {
- PageviewDefinition pageviewDefinitionInstance =
PageviewDefinition.getInstance();
- return pageviewDefinitionInstance.getProjectFromHost(uriHost);
- }
-}
\ No newline at end of file
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java
new file mode 100644
index 0000000..135da7d
--- /dev/null
+++
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java
@@ -0,0 +1,127 @@
+/**
+ * Copyright (C) 2014 Wikimedia Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wikimedia.analytics.refinery.hive;
+
+import junitparams.FileParameters;
+import junitparams.JUnitParamsRunner;
+import junitparams.mappers.CsvWithHeaderMapper;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.wikimedia.analytics.refinery.core.PageviewDefinition;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+@RunWith(JUnitParamsRunner.class)
+public class TestGetPageviewInfoUDF {
+
+ @Test(expected = UDFArgumentLengthException.class)
+ public void testBadNumberOfArguments() throws HiveException {
+ ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] initArguments = new ObjectInspector[]{value1,
value2};
+ GetPageviewInfoUDF udf = new GetPageviewInfoUDF();
+ udf.initialize(initArguments);
+ }
+
+ @Test(expected = UDFArgumentTypeException.class)
+ public void testWrongTypeOfArguments() throws HiveException {
+ ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaIntObjectInspector;
+ ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector value3 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] initArguments = new ObjectInspector[]{value1,
value2, value3};
+ GetPageviewInfoUDF udf = new GetPageviewInfoUDF();
+ udf.initialize(initArguments);
+ }
+
+
+ @Test
+ @FileParameters(
+ value = "../refinery-core/src/test/resources/pageview_test_data.csv",
+ mapper = CsvWithHeaderMapper.class
+ )
+ public void testGetPageviewInfo(
+ String test_description,
+ String project,
+ String dialect,
+ String pageTitle,
+ boolean is_pageview,
+ boolean is_legacy_pageview,
+ boolean is_app_pageview,
+ String ip_address,
+ String x_forwarded_for,
+ String uri_host,
+ String uri_path,
+ String uri_query,
+ String http_status,
+ String content_type,
+ String user_agent
+ ) throws HiveException, IOException {
+ if (is_pageview) {
+ ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector value3 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] initArguments = new ObjectInspector[]{value1,
value2, value3};
+ GetPageviewInfoUDF udf = new GetPageviewInfoUDF();
+
+ udf.initialize(initArguments);
+
+ GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[]
{
+ new GenericUDF.DeferredJavaObject(uri_host),
+ new GenericUDF.DeferredJavaObject(uri_path),
+ new GenericUDF.DeferredJavaObject(uri_query)
+ };
+ Map<String, String> result = (Map<String,
String>)udf.evaluate(args);
+ udf.close();
+
+ assertEquals("Project check -" + test_description, project,
result.get(GetPageviewInfoUDF.PROJECT_KEY));
+ assertEquals("Dialect check -" + test_description, dialect,
result.get(GetPageviewInfoUDF.DIALECT_KEY));
+ assertEquals("Page_Title check - " + test_description, pageTitle,
result.get(GetPageviewInfoUDF.PAGE_TITLE_KEY));
+ }
+ }
+
+ @Test
+ public void testGetPageviewInfoNull() throws HiveException, IOException {
+ ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector value3 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] initArguments = new ObjectInspector[]{value1,
value2, value3};
+ GetPageviewInfoUDF udf = new GetPageviewInfoUDF();
+
+ udf.initialize(initArguments);
+
+ GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {
+ new GenericUDF.DeferredJavaObject(null),
+ new GenericUDF.DeferredJavaObject(null),
+ new GenericUDF.DeferredJavaObject(null)
+ };
+ Map<String, String> result = (Map<String, String>)udf.evaluate(args);
+ udf.close();
+
+ assertEquals("Empty project",
PageviewDefinition.UNKNOWN_PROJECT_VALUE,
result.get(GetPageviewInfoUDF.PROJECT_KEY));
+ assertEquals("Empty dialect",
PageviewDefinition.UNKNOWN_DIALECT_VALUE,
result.get(GetPageviewInfoUDF.DIALECT_KEY));
+ assertEquals("Empty page_title",
PageviewDefinition.UNKNOWN_PAGE_TITLE_VALUE,
result.get(GetPageviewInfoUDF.PAGE_TITLE_KEY));
+ }
+
+}
\ No newline at end of file
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java
deleted file mode 100644
index 951921a..0000000
---
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Copyright (C) 2014 Wikimedia Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.wikimedia.analytics.refinery.hive;
-
-import junitparams.FileParameters;
-import junitparams.JUnitParamsRunner;
-import junitparams.mappers.CsvWithHeaderMapper;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import static org.junit.Assert.assertEquals;
-
-@RunWith(JUnitParamsRunner.class)
-public class TestGetProjectUDF {
-
-
- @Test
- @FileParameters(
- value = "../refinery-core/src/test/resources/pageview_test_data.csv",
- mapper = CsvWithHeaderMapper.class
- )
- public void testGetProject(
- String test_description,
- String project,
- boolean is_pageview,
- boolean is_legacy_pageview,
- boolean is_app_pageview,
- String ip_address,
- String x_forwarded_for,
- String uri_host,
- String uri_path,
- String uri_query,
- String http_status,
- String content_type,
- String user_agent
- ) {
- GetProjectUDF udf = new GetProjectUDF();
-
- assertEquals(
- test_description,
- project,
- udf.evaluate(uri_host)
- );
- }
-
- @Test
- public void testGetProjectNull() {
- GetProjectUDF udf = new GetProjectUDF();
-
- assertEquals(
- "Test null input to getProject",
- "-",
- udf.evaluate(null)
- );
- }
-
-}
\ No newline at end of file
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
index f36ead9..3b93b0b 100644
---
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
+++
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
@@ -34,6 +34,8 @@
public void testIsAppPageview(
String test_description,
String project,
+ String dialect,
+ String pageTitle,
boolean is_pageview,
boolean is_legacy_pageview,
boolean is_app_pageview,
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java
index 5dbdb8a..2629038 100644
---
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java
+++
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java
@@ -35,6 +35,8 @@
public void testIsPageview(
String test_description,
String project,
+ String dialect,
+ String pageTitle,
boolean is_pageview,
boolean is_legacy_pageview,
boolean is_app_pageview,
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
index 172b85f..bcefdcc 100644
---
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
+++
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
@@ -35,6 +35,8 @@
public void testIsPageview(
String test_description,
String project,
+ String dialect,
+ String pageTitle,
boolean is_pageview,
boolean is_legacy_pageview,
boolean is_app_pageview,
--
To view, visit https://gerrit.wikimedia.org/r/214349
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ieed48b6c520c09e62d3bc05085e67f11ed3a96b7
Gerrit-PatchSet: 6
Gerrit-Project: analytics/refinery/source
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Madhuvishy <[email protected]>
Gerrit-Reviewer: OliverKeyes <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
