Ottomata has submitted this change and it was merged. Change subject: Add UAParserUDF from kraken ......................................................................
Add UAParserUDF from kraken This was reviewed over at https://gerrit.wikimedia.org/r/#/c/96738 Added parametized test for most popular user agents Change-Id: I4af37b982809415d3d27fc53b965802e6e6b5b3d --- M pom.xml A refinery-hive/pom.xml A refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/UAParserUDF.java A refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFArgumentHandling.java A refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentMostPopular.java A refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentRecognition.java 6 files changed, 720 insertions(+), 0 deletions(-) Approvals: Ottomata: Verified; Looks good to me, approved diff --git a/pom.xml b/pom.xml index 9cbbceb..f93546b 100644 --- a/pom.xml +++ b/pom.xml @@ -14,6 +14,7 @@ <modules> <module>refinery-tools</module> + <module>refinery-hive</module> </modules> <scm> @@ -60,6 +61,10 @@ <id>wmf-mirrored</id> <url>http://archiva.wikimedia.org/repository/mirrored/</url> </repository> + <repository> + <id>wmf-releases</id> + <url>http://archiva.wikimedia.org/repository/releases/</url> + </repository> </repositories> <pluginRepositories> diff --git a/refinery-hive/pom.xml b/refinery-hive/pom.xml new file mode 100644 index 0000000..0fd69d2 --- /dev/null +++ b/refinery-hive/pom.xml @@ -0,0 +1,72 @@ +<project xmlns = "http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation = "http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.wikimedia.analytics.refinery</groupId> + <artifactId>refinery</artifactId> + <version>0.0.1</version> + </parent> + + <groupId>org.wikimedia.analytics.refinery.hive</groupId> + <artifactId>refinery-hive</artifactId> + <name>Wikimedia Analytics Refinery Hive</name> + <packaging>jar</packaging> + + <dependencies> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>0.12.0-cdh5.0.2</version> + </dependency> + + <dependency> + <groupId>ua_parser</groupId> + <artifactId>ua-parser</artifactId> + <version>1.3.0-wmf1</version> + </dependency> + <dependency> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> + <version>1.1.1</version> + </dependency> + + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.0</version> + <configuration> + <shadedArtifactAttached>false</shadedArtifactAttached> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>false</createDependencyReducedPom> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/UAParserUDF.java b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/UAParserUDF.java new file mode 100644 index 0000000..7cf97ae --- /dev/null +++ b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/UAParserUDF.java @@ -0,0 +1,215 @@ +package org.wikimedia.analytics.refinery.hive; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.log4j.Logger; +import ua_parser.*; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * UDF that parses user agent strings using UA parser and returns a hashmap in + * this form: + * { + * "device_family":"Other", + * "browser_major":"11", + * "os_family":"Windows", + * "os_major":"-", + * "browser_family":"IE", + * "os_minor":"-" + * } + * <p/> + * Records are processed one by one. + * + * NOTE: This UDF was original coded as part of the Kraken repository, and may have not + * received as high of qualitiy code review as we would like. However, it works as is, + * and we want to make this functionality available for use. Please also note that there + * is currently not a process for ensuring that the dependent ua_parser package is up to date + * with the latest user agent classification regexes. + */ + +@UDFType(deterministic = true) +@Description(name = "ua_parser", value = "_FUNC_(UA) - " + + "Returns a map with browser_name, browser_major, device, os_name, os_minor, os_major keys and " + + "the appropriate values for each of them") +public class UAParserUDF extends GenericUDF { + Map<String, String> result = new HashMap<String, String>(); + public CachingParser cachingParser; + private ObjectInspector argumentOI; + + // TODO figure out why not everything is logged to hive.log and some logging + // TODO through log4j stays on the hadoop logs + static final Logger Log = Logger.getLogger(UAParserUDF.class.getName()); + + /** + * The initialize method is called only once during the lifetime of the UDF. + * <p/> + * Method checks for the validity (number, type, etc) + * of the arguments being passed to the UDF. + * It also sets the return type of the result of the UDF, + * in this case the ObjectInspector equivalent of + * Map<String,String> + * + * @param arguments + * @return ObjectInspector Map<String,String> + * @throws UDFArgumentException + */ + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + + if (arguments.length != 1) { + throw new UDFArgumentLengthException( + "The UAParserUDF takes an array with only 1 element as argument"); + } + + //we are expecting the parameter to be of String type. + ObjectInspector arg = arguments[0]; + int argIndex = 0; + + if (arg.getCategory() != Category.PRIMITIVE) { + throw new UDFArgumentTypeException(argIndex, + "A string argument was expected but an argument of type " + arg.getTypeName() + + " was given."); + + } + + // Now that we have made sure that the argument is of primitive type, we can get the primitive + // category + PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) arg) + .getPrimitiveCategory(); + + if (primitiveCategory != PrimitiveCategory.STRING) { + throw new UDFArgumentTypeException(argIndex, + "A string argument was expected but an argument of type " + arg.getTypeName() + + " was given."); + + } + + + try { + cachingParser = new CachingParser(); + } catch (IOException e) { + // no recovery should be possible, log and rethrow + // runtime exception will be logged to stdout by default + Log.error(e.getMessage(), e); + throw new RuntimeException("Failed to instantiate CachingParser"); + } + + argumentOI = arg; + return ObjectInspectorFactory.getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaStringObjectInspector); + } + + private final String NA = "-"; + + private String replaceNA(String str) { + final String ret; + if (str == null || str.isEmpty() || str.equals("-")) { + ret = NA; + } else { + ret = str; + } + return ret; + } + + /** + * Takes the actual arguments and returns the result. + * Gets passed the input, does whatever it wants to it, + * and then returns the output. + * <p/> + * The input is accessed using the ObjectInspectors that + * were saved into global variables in the call to initialize() + * <p/> + * This method is called once for every row of data being processed. + * UDFs are called during the map phase of the MapReduce job. + * This means that we have no control over the order in which the + * records get sent to the UDF. + * + * @param arguments + * @return + * @throws HiveException + */ + @SuppressWarnings("unchecked") + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + result.clear(); + + UserAgent browser = null; + Device device = null; + OS os = null; + + try { + if (arguments.length == 1 + && argumentOI != null && arguments[0] != null) { + Client c; + String pattern = ((StringObjectInspector) argumentOI) + .getPrimitiveJavaObject(arguments[0].get()); + + c = cachingParser.parse(pattern); + if (c != null) { + browser = c.userAgent; + device = c.device; + os = c.os; + } + } + } catch (Exception e) { + // catch it all to make sure job does not halt if one record is faulty + // TODO find out why this gets logged to hadoop but not to hive.log + Log.error(e.getMessage(), e); + } + + if (browser != null) { + result.put("browser_family", replaceNA(browser.family)); + result.put("browser_major", replaceNA(browser.major)); + } else { + result.put("browser_family", NA); + result.put("browser_major", NA); + } + + if (device != null) { + result.put("device_family", replaceNA(device.family)); + } else { + result.put("device_family", NA); + } + + if (os != null) { + result.put("os_family", replaceNA(os.family)); + result.put("os_major", replaceNA(os.major)); + result.put("os_minor", replaceNA(os.minor)); + } else { + result.put("os_family", NA); + result.put("os_major", NA); + result.put("os_minor", NA); + } + + return result; + } + + /** + * Returns string representation of the function. + * Will be printed when an error happens. + * * + */ + @Override + public String getDisplayString(String[] arguments) { + assert (arguments.length == 1); + return "user_agent_parser(" + arguments[0] + ")"; + + } +} diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFArgumentHandling.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFArgumentHandling.java new file mode 100644 index 0000000..9574094 --- /dev/null +++ b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFArgumentHandling.java @@ -0,0 +1,40 @@ +package org.wikimedia.analytics.refinery.hive; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Test; + +import java.util.HashMap; + +public class TestUAParserUDFArgumentHandling { + + + @Test(expected = UDFArgumentLengthException.class) + public void testBadNumberOfArguments() throws HiveException { + + ObjectInspector value1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector value2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] initArguments = new ObjectInspector[]{value1, value2}; + UAParserUDF uaParserUDF = new UAParserUDF(); + uaParserUDF.initialize(initArguments); + + } + + @Test(expected = UDFArgumentTypeException.class) + public void testWrongTypeOfArguments() throws HiveException { + + ObjectInspector value1 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector[] initArguments = new ObjectInspector[]{value1}; + UAParserUDF uaParserUDF = new UAParserUDF(); + uaParserUDF.initialize(initArguments); + + } + + +} + + + diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentMostPopular.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentMostPopular.java new file mode 100644 index 0000000..8e9e6df --- /dev/null +++ b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentMostPopular.java @@ -0,0 +1,217 @@ +package org.wikimedia.analytics.refinery.hive; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Collection; +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Before; +import org.junit.After; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runner.RunWith; +import org.json.simple.parser.JSONParser; +import org.json.simple.JSONObject; +import org.json.simple.parser.ParseException; + + +/** + * We test the most popular browser + device combos (from sampled logs) + * and the ua parser reporting on on those. + * <p/> + * Test failing will indicate than the newer version of ua parser + * is significantly different from the prior one. + */ +@RunWith(Parameterized.class) +public class TestUAParserUDFUserAgentMostPopular { + ObjectInspector[] initArguments = null; + UAParserUDF uaParserUDF = null; + JSONParser parser = null; + + @Before + public void setUp() throws HiveException { + ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + initArguments = new ObjectInspector[]{valueOI}; + uaParserUDF = new UAParserUDF(); + uaParserUDF.initialize(initArguments); + parser = new JSONParser(); + + } + + @After + public void tearDown() throws HiveException { + + try { + uaParserUDF.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + @Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][]{ + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"8\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"GMozilla/5.0 (Linux; Android 4.4.2; GT-I9505 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.102 Mobile Safari/537.36", "{\"os_minor\": \"4\", \"os_major\": \"4\", \"device_family\": \"Samsung GT-I9505\", \"os_family\": \"Android\", \"browser_major\": \"38\", \"browser_family\": \"Chrome Mobile\"}"}, + {"Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Spider\", \"os_family\": \"Other\", \"browser_major\": \"-\", \"browser_family\": \"Other\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/600.1.17 (KHTML, like Gecko) Version/6.2 Safari/537.85.10", "{\"os_minor\": \"8\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"6\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"36\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) CriOS/38.0.2125.59 Mobile/11D257 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"38\", \"browser_family\": \"Chrome Mobile iOS\"}"}, + {"Opera/9.80 (Windows NT 5.1) Presto/2.12.388 Version/12.17", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows XP\", \"browser_major\": \"12\", \"browser_family\": \"Opera\"}"}, + {"Mozilla/5.0 (Linux; Android 4.4.4; Nexus 5 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.102 Mobile Safari/537.36", "{\"os_minor\": \"4\", \"os_major\": \"4\", \"device_family\": \"Nexus 5\", \"os_family\": \"Android\", \"browser_major\": \"38\", \"browser_family\": \"Chrome Mobile\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53", "{\"os_minor\": \"0\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36 OPR/25.0.1614.50", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"25\", \"browser_family\": \"Opera\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 6_1_3 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25", "{\"os_minor\": \"1\", \"os_major\": \"6\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"6\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"7\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"35\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 8_0_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) CriOS/38.0.2125.59 Mobile/12A405 Safari/600.1.4", "{\"os_minor\": \"0\", \"os_major\": \"8\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"38\", \"browser_family\": \"Chrome Mobile iOS\"}"}, + {"Mozilla/5.0 (Android; Mobile; rv:33.0) Gecko/33.0 Firefox/33.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Android\", \"browser_major\": \"33\", \"browser_family\": \"Firefox Mobile\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"AppleDictionaryService/208", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Other\", \"browser_major\": \"-\", \"browser_family\": \"Other\"}"}, + {"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows Vista\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 5_1_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B206 Safari/7534.48.3", "{\"os_minor\": \"1\", \"os_major\": \"5\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"5\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Other\", \"browser_major\": \"9\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 YaBrowser/14.8.1985.12084 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"14\", \"browser_family\": \"Yandex Browser\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A365 Safari/600.1.4", "{\"os_minor\": \"0\", \"os_major\": \"8\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"8\", \"browser_family\": \"Mobile Safari\"}"}, + {"MediaWiki/1.25wmf3", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Other\", \"browser_major\": \"-\", \"browser_family\": \"Other\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"11\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"6\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Opera/9.80 (Windows NT 6.1; WOW64) Presto/2.12.388 Version/12.17", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"12\", \"browser_family\": \"Opera\"}"}, + {"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Ubuntu\", \"browser_major\": \"33\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4", "{\"os_minor\": \"1\", \"os_major\": \"8\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"8\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 7_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D167 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36 OPR/25.0.1614.50", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"25\", \"browser_family\": \"Opera\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_6 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B651 Safari/9537.53", "{\"os_minor\": \"0\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "{\"os_minor\": \"0\", \"os_major\": \"6\", \"device_family\": \"Spider\", \"os_family\": \"iOS\", \"browser_major\": \"2\", \"browser_family\": \"Googlebot\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"31\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"9\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.78.2 (KHTML, like Gecko) Version/6.1.6 Safari/537.78.2", "{\"os_minor\": \"7\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"6\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_3 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25", "{\"os_minor\": \"1\", \"os_major\": \"6\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"6\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8\", \"browser_major\": \"10\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 7_0_4 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B554a Safari/9537.53", "{\"os_minor\": \"0\", \"os_major\": \"7\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 YaBrowser/14.8.1985.12084 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"14\", \"browser_family\": \"Yandex Browser\"}"}, + {"Mozilla/5.0 (Windows NT 6.0; rv:32.0) Gecko/20100101 Firefox/32.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows Vista\", \"browser_major\": \"32\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 8_0_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) GSA/4.2.2.38484 Mobile/12A405 Safari/9537.53", "{\"os_minor\": \"0\", \"os_major\": \"8\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"8\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.77.4 (KHTML, like Gecko) Version/7.0.5 Safari/537.77.4", "{\"os_minor\": \"9\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"7\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0", "{\"os_minor\": \"9\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"32\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.78.2 (KHTML, like Gecko) Version/7.0.6 Safari/537.78.2", "{\"os_minor\": \"9\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"7\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) GSA/4.2.2.38484 Mobile/11D257 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows Vista\", \"browser_major\": \"9\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8\", \"browser_major\": \"32\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Spider\", \"os_family\": \"Other\", \"browser_major\": \"2\", \"browser_family\": \"Googlebot\"}"}, + {"NativeHost", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Other\", \"browser_major\": \"-\", \"browser_family\": \"Other\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.78.2 (KHTML, like Gecko) Version/7.0.6 Safari/537.78.2", "{\"os_minor\": \"9\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"7\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36 OPR/25.0.1614.50", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"25\", \"browser_family\": \"Opera\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"33\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 7_1_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D201 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.59.10 (KHTML, like Gecko) Version/5.1.9 Safari/534.59.10", "{\"os_minor\": \"6\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"5\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"10\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"10\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D167 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows XP\", \"browser_major\": \"33\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows Vista\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Other\", \"browser_major\": \"1\", \"browser_family\": \"FacebookBot\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; rv:33.0) Gecko/20100101 Firefox/33.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"33\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A365 Safari/600.1.4", "{\"os_minor\": \"0\", \"os_major\": \"8\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"8\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_4 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B554a Safari/9537.53", "{\"os_minor\": \"0\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4", "{\"os_minor\": \"1\", \"os_major\": \"8\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"8\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"9\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"10\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows XP\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"-", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Other\", \"browser_major\": \"-\", \"browser_family\": \"Other\"}"}, + {"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Spider\", \"os_family\": \"Other\", \"browser_major\": \"2\", \"browser_family\": \"bingbot\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"11\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25", "{\"os_minor\": \"10\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"8\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"32\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"9\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D201 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.1.17 (KHTML, like Gecko) Version/7.1 Safari/537.85.10", "{\"os_minor\": \"9\", \"os_major\": \"10\", \"device_family\": \"Other\", \"os_family\": \"Mac OS X\", \"browser_major\": \"7\", \"browser_family\": \"Safari\"}"}, + {"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"9\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 8_0_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A405 Safari/600.1.4", "{\"os_minor\": \"0\", \"os_major\": \"8\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"8\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (iPad; CPU OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPad\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 5.1; rv:32.0) Gecko/20100101 Firefox/32.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows XP\", \"browser_major\": \"32\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; rv:32.0) Gecko/20100101 Firefox/32.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"32\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"33\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"11\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows XP\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"37\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 8_0_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A405 Safari/600.1.4", "{\"os_minor\": \"0\", \"os_major\": \"8\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"8\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 8.1\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53", "{\"os_minor\": \"1\", \"os_major\": \"7\", \"device_family\": \"iPhone\", \"os_family\": \"iOS\", \"browser_major\": \"7\", \"browser_family\": \"Mobile Safari\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"11\", \"browser_family\": \"IE\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"32\", \"browser_family\": \"Firefox\"}"}, + {"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36", "{\"os_minor\": \"-\", \"os_major\": \"-\", \"device_family\": \"Other\", \"os_family\": \"Windows 7\", \"browser_major\": \"38\", \"browser_family\": \"Chrome\"}"}, + + }); + } + + private String fInput; + + private String fExpected; + + + public TestUAParserUDFUserAgentMostPopular(String input, String expected) { + fInput = input; + fExpected = expected; + } + + + @Test + public void testMatchingOfMostPopularUA() throws HiveException, ParseException { + + DeferredJavaObject ua = new DeferredJavaObject(fInput); + + // decode expected output and turn it into an object + Object obj = parser.parse(fExpected); + JSONObject expected_ua = (JSONObject) obj; + + DeferredObject[] args1 = {ua}; + HashMap<String, String> computed_ua = (HashMap<String, String>) uaParserUDF + .evaluate(args1); + + assertEquals("OS name check", expected_ua.get("os_family"), + computed_ua.get("os_family")); + + assertEquals("OS major version check", expected_ua.get("os_major"), + computed_ua.get("os_major")); + + assertEquals("OS minor version check", expected_ua.get("os_minor"), + computed_ua.get("os_minor")); + + assertEquals("browser check", expected_ua.get("browser_family"), + computed_ua.get("browser_family")); + + assertEquals("browser major version check", expected_ua.get("browser_major"), + computed_ua.get("browser_major")); + + assertEquals("device check", expected_ua.get("device_family"), + computed_ua.get("device_family")); + + } + +} + + + diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentRecognition.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentRecognition.java new file mode 100644 index 0000000..6002b07 --- /dev/null +++ b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestUAParserUDFUserAgentRecognition.java @@ -0,0 +1,171 @@ +package org.wikimedia.analytics.refinery.hive; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.HashMap; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Before; +import org.junit.After; +import org.junit.Test; + +public class TestUAParserUDFUserAgentRecognition { + ObjectInspector[] initArguments = null; + UAParserUDF uaParserUDF = null; + + @Before + public void setUp() throws HiveException { + ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + initArguments = new ObjectInspector[]{valueOI}; + uaParserUDF = new UAParserUDF(); + uaParserUDF.initialize(initArguments); + + } + + @After + public void tearDown() throws HiveException { + + try { + uaParserUDF.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + @Test + public void testHappyCase() throws HiveException { + + DeferredJavaObject ua1 = new DeferredJavaObject( + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:19.0) Gecko/20100101 Firefox/19.0"); + DeferredJavaObject ua2 = new DeferredJavaObject( + "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); + DeferredJavaObject ua3 = new DeferredJavaObject( + "Mozilla/5.0 (iPad; CPU OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53"); + + + DeferredObject[] args1 = { ua1 }; + HashMap<String, String> evaled = (HashMap<String, String>) uaParserUDF + .evaluate(args1); + assertEquals("OS name check", (new String("Ubuntu")), + evaled.get("os_family").toString()); + assertEquals("Browser name check", (new String("Firefox")), + evaled.get("browser_family").toString()); + + + DeferredObject[] args2 = { ua2 }; + evaled = (HashMap<String, String>) uaParserUDF + .evaluate(args2); + assertEquals("OS name check", (new String("iOS")), + evaled.get("os_family").toString()); + + assertEquals("Browser name check", (new String("Mobile Safari")), + evaled.get("browser_family").toString()); + + + DeferredObject[] args3 = { ua3 }; + evaled = (HashMap<String, String>) uaParserUDF + .evaluate(args3); + assertEquals("OS name check", (new String("iOS")), + evaled.get("os_family").toString()); + assertEquals("Browser name check", (new String("Mobile Safari")), + evaled.get("browser_family").toString()); + + + + + + } + + /** + * Tests what we return when browser is empty. + * + * UA parser will return this as "browser obj" for an empty user agent string: + * { + * user_agent: {family: "Other", major: null, minor: null, patch: null}, + * os: {family: "Other", major: null, minor: null, patch: null, patch_minor: null}, + * device: {family: "Other"} + * } + * UDFs returns something like the following: + * { + * "device_family":"Other", + * "browser_major":"-", + * "os_family":"Other", + * "os_major":"-", + * "browser_family":"Other", + * "os_minor":"-" + * } + **/ + @Test + public void testEmptyUA() throws HiveException { + DeferredJavaObject ua1 = new DeferredJavaObject(""); + + DeferredObject[] args = { ua1 }; + + HashMap<String, String> evaled = (HashMap<String, String>) uaParserUDF + .evaluate(args); + + String resultOSName = evaled.get("os_family"); + String resultBrowserName = evaled.get("browser_family"); + String resultOsMinor = evaled.get("os_minor"); + assertEquals("OS name check", (new String("Other")), + resultOSName.toString()); + assertEquals("Browser name check", (new String("Other")), + resultBrowserName.toString()); + + assertEquals("OS minor", (new String("-")), + resultOsMinor.toString()); + } + + /** + * Tests what we return when browser is null + * Is this right? Maybe we should return just null + * + * UA parser will return this as "browser obj" for an empty user agent string: + * { + * user_agent: {family: "Other", major: null, minor: null, patch: null}, + * os: {family: "Other", major: null, minor: null, patch: null, patch_minor: null}, + * device: {family: "Other"} + * } + * UDFs returns something like the following: + * { + * "device_family":"Other", + * "browser_major":"-", + * "os_family":"Other", + * "os_major":"-", + * "browser_family":"Other", + * "os_minor":"-" + * } + **/ + public void testHandlingOfNulls() throws HiveException { + + DeferredJavaObject ua1 = new DeferredJavaObject(null); + + DeferredObject[] args = { ua1 }; + + HashMap<String, String> evaled = (HashMap<String, String>) uaParserUDF + .evaluate(args); + + String resultOSName = evaled.get("os_family"); + String resultBrowserName = evaled.get("browser_family"); + String resultOsMinor = evaled.get("os_minor"); + assertEquals("OS name check", (new String("Other")), + resultOSName.toString()); + assertEquals("Browser name check", (new String("Other")), + resultBrowserName.toString()); + + assertEquals("OS minor", (new String("-")), + resultOsMinor.toString()); + + } + +} + + + -- To view, visit https://gerrit.wikimedia.org/r/166142 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I4af37b982809415d3d27fc53b965802e6e6b5b3d Gerrit-PatchSet: 14 Gerrit-Project: analytics/refinery/source Gerrit-Branch: master Gerrit-Owner: Ottomata <[email protected]> Gerrit-Reviewer: Nuria <[email protected]> Gerrit-Reviewer: Ottomata <[email protected]> Gerrit-Reviewer: QChris <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
