Diederik has submitted this change and it was merged. Change subject: Updating Kraken to cope with libdclass 2.2.2 ......................................................................
Updating Kraken to cope with libdclass 2.2.2 Although libdclass 2.2.2 can now be used, we still rely on the dtree file from libdclass-data 2.0.14 (i.e.: the dtree file in version 1.13), due significant changes in the patterns of newer dtree files. Change-Id: I3621ed24a83fe8cfdc431e74c3867e7b92b5603c --- M README.md M kraken-dclass/pom.xml A kraken-dclass/src/main/java/dclass/dClass.java D kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DclassWrapper.java M kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java D kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassLoaderTest.java A kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java A kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java 8 files changed, 229 insertions(+), 181 deletions(-) Approvals: Diederik: Verified; Looks good to me, approved diff --git a/README.md b/README.md index ddde037..b2f04e4 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ ### Debian-like systems -If your an a Debian system, you can probably just grab the most recent .deb files of libdclass0 and libdclassjni0 for your architecture from [garage-coding](http://garage-coding.com/releases/libdclass-dev/) and install them using dpkg and can proceed to the next step. +If your an a Debian system, you can probably just grab the most recent .deb files of libdclass0 and libdclass-jni for your architecture from [Wikimedia's apt repository](http://apt.wikimedia.org/wikimedia/pool/main/d/dclass/) and install them using dpkg and can proceed to the next step. ### Non-Debian-like systems @@ -46,7 +46,7 @@ #### Getting files by hand -For an amd64 system with the most recent version being 2.0.14, +For an amd64 system with the most recent version being 2.2.0, that would be: mkdir dclass @@ -54,19 +54,19 @@ mkdir libdclass0 cd libdclass0 - wget http://garage-coding.com/releases/libdclass-dev/libdclass0_2.0.14_amd64.deb - ar x libdclass0_2.0.14_amd64.deb + wget http://apt.wikimedia.org/wikimedia/pool/main/d/dclass/libdclass0_2.2.2-1_amd64.deb + ar x libdclass0_2.2.2-1_amd64.deb tar -xzvf data.tar.gz - sudo mv usr/lib/libdclass.so.0.0.0 /usr/lib/libdclass.so.0 + sudo mv usr/lib/x86_64-linux-gnu/libdclass.so.0.0.0 /usr/lib/libdclass.so.0 sudo chmod +x /usr/lib/libdclass.so.0 cd .. - mkdir libdclassjni0 - cd libdclassjni0 - wget http://garage-coding.com/releases/libdclass-dev/libdclassjni0_2.0.14_amd64.deb - ar x libdclassjni0_2.0.14_amd64.deb + mkdir libdclass-jni + cd libdclass-jni + wget http://apt.wikimedia.org/wikimedia/pool/main/d/dclass/libdclass-jni_2.2.2-1_amd64.deb + ar x libdclass-jni_2.2.2-1_amd64.deb tar -xzvf data.tar.gz - sudo mv usr/lib/libdclassjni.so.0.0.0 /usr/lib/libdclassjni.so + sudo mv usr/lib/x86_64-linux-gnu/jni/libdclassjni.so.0.0.0 /usr/lib/libdclassjni.so sudo chmod +x /usr/lib/libdclassjni.so cd .. @@ -104,9 +104,9 @@ ## openddr.dtree file -Get a openddr.dtree file from an [Analytics Team member](http://www.mediawiki.org/wiki/Analytics), and store it as +Get a openddr.dtree file (version 1.13) from an [Analytics Team member](http://www.mediawiki.org/wiki/Analytics), and store it as - /usr/share/libdclass/dtrees/openddr.dtree + /usr/share/libdclass/openddr.dtree ## GeoIP files diff --git a/kraken-dclass/pom.xml b/kraken-dclass/pom.xml index f91dab5..cf755d8 100644 --- a/kraken-dclass/pom.xml +++ b/kraken-dclass/pom.xml @@ -17,13 +17,6 @@ </parent> <dependencies> - <!-- - <dependency> - <groupId>org.apache.devicemap</groupId> - <artifactId>openddr-java</artifactId> - <version>0.9.9-SNAPSHOT</version> - </dependency> - --> </dependencies> <properties> diff --git a/kraken-dclass/src/main/java/dclass/dClass.java b/kraken-dclass/src/main/java/dclass/dClass.java new file mode 100644 index 0000000..bee86b7 --- /dev/null +++ b/kraken-dclass/src/main/java/dclass/dClass.java @@ -0,0 +1,144 @@ +/** + * Copyright (C) 2012-2013 Wikimedia Foundation + + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package dclass; + + + +import java.io.IOException; +import java.util.Map; +import java.util.HashMap; + + +public class dClass { + private String path; + + /* + * Note: + * + * the dclass_index property of this class is being set by + * the shared object native method setindex() + * + * which is being called from native method init() + * + */ + private long dclass_index; + private boolean isDClassInitialized = false; + + + private native int init(String file); + private native void free(long index); + + private native long classify(long index,String s); + + private native int kvlength(long kv); + private native String kvgetid(long kv); + private native String kvgetkey(long kv,int pos); + private native String kvgetvalue(long kv,int pos); + + + private long getDclassIndex() { + return dclass_index; + }; + + + private static final String OS = System.getProperty("os.name").toLowerCase(); + + public dClass(String s) + { + path=s; + dclass_index=0; + isDClassInitialized = ( init(s) >= 0); + } + + public Map<String,String> classify(String s) + { + Map<String,String> ret=new HashMap<String,String>(); + + + + if(isDClassInitialized && s!=null) + { + long kv; + + + kv=classify(dclass_index,s); + + if(kv!=0) + { + int len=kvlength(kv); + String id=kvgetid(kv); + for(int i=0;i<len;i++) + { + String key=kvgetkey(kv,i); + String value=kvgetvalue(kv,i); + ret.put(key,value); + } + ret.put("id",id); + } + } + + return ret; + } + + + + protected void finalize() { + if (isDClassInitialized) + free(this.getDclassIndex()); + isDClassInitialized = false; + + } + + public static void loadDclassSharedObject() throws IOException { + try { + if (OS.contains("mac")) { + System.load("/usr/local/lib/libdclassjni.0.dylib"); + } else if (OS.contains("nix") + || OS.contains("nux") + || OS.contains("aix")) { + + try { + System.loadLibrary("dclassjni"); + } catch(Exception e) { + System.err.println("Error: " + e); + System.exit(-1); + }; + + } else { + System.err.println("OS not supported."); + System.exit(-1); + } + } catch (UnsatisfiedLinkError e) { + System.err.println("Native code library failed to load.\n" + e); + e.printStackTrace(); + } + } + + static { + try { + loadDclassSharedObject(); + } catch (IOException e) { + System.err.println("Error loading dClass Shared Object: "+e.getMessage()); + e.printStackTrace(); + } + } + + + +} + diff --git a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DclassWrapper.java b/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DclassWrapper.java deleted file mode 100644 index 0a2bd61..0000000 --- a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DclassWrapper.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright (C) 2012-2013 Wikimedia Foundation - - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.wikimedia.analytics.dclassjni; - - -import java.io.File; -import java.io.IOException; -import java.util.Map; - -public class DclassWrapper { - /** - * allocate and load up dtrees in memory. - */ - public native void initUA(); - - /** - * deallocate dtrees(sort-of a destructor). - */ - public native void destroyUA(); - - /** - * the good stuff (what we actually want to do, classify UAs). - * - * @param ua the useragent device strings - * @return the @Map<String, String> - */ - public native Map<String, String> classifyUA(String ua); - - /** - * Pointer to dclass_index structure. - */ - public long pointer_di; - - /** - * Set_pointer_di. - * - * @param i the i - * @return the long - */ - public long set_pointer_di(long i) { - pointer_di = i; - return pointer_di; - } - - /** - * Gets the _pointer_di. - * - * @return the _pointer_di - */ - public final long get_pointer_di() { - return pointer_di; - } - - - private static final String OS = System.getProperty("os.name").toLowerCase(); - - public static void loadDclassSharedObject() throws IOException { - try { - if (OS.contains("mac")) { - System.load("/usr/local/lib/libdclassjni.0.dylib"); - } else if (OS.contains("nix") - || OS.contains("nux") - || OS.contains("aix")) { - System.load("/usr/lib/libdclassjni.so"); - } else { - System.err.println("OS not supported."); - System.exit(-1); - } - } catch (UnsatisfiedLinkError e) { - File f = new File("/usr/lib/libdclassjni.so"); - System.err.println("/usr/lib/libdclassjni.so exists: " + f.exists()); - System.err.println("/usr/lib/libdclassjni.so is readable: " + f.canRead()); - System.err.println("Native code library failed to load.\n" + e); - e.printStackTrace(); - } - } - - static { - try { - loadDclassSharedObject(); - } catch (IOException e) { - System.err.println("Error loading dClass Shared Object: "+e.getMessage()); - e.printStackTrace(); - } - } - - - static String userAgentSample = "Mozilla/5.0 (Linux; U; Android 2.2; en; HTC Aria A6380 Build/ERE27) AppleWebKit/540.13+ (KHTML, like Gecko) Version/3.1 Mobile Safari/524.15.0"; - public static void main(final String[] args){ - DeviceClassification deviceClassification = new DeviceClassification(); - deviceClassification.classifyUseragent(userAgentSample); - } - - -} - diff --git a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java b/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java index dae5ebf..1854ba2 100644 --- a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java +++ b/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java @@ -21,6 +21,9 @@ import java.util.Map; +import dclass.dClass; + + /** * The class dClassParsedResult. This class is a mapping of the results from * dclass_keyvalue data structure to a Java data structure :) @@ -40,12 +43,12 @@ * */ public class DeviceClassification { - + public final static String dtree_path = "/usr/share/libdclass/openddr.dtree"; /** * Only initialize the dClass JNI wrapper on-demand (and only once), * as doing so loads 2+ MB of dtree data into memory from disk. */ - private static boolean isDClassInitialized = false; + /** * Wrapper subclass to release the dtree data when the singleton is @@ -53,24 +56,11 @@ * happen without imminent JVM termination, such as in IOC containers, * like Spring). */ - final private static DclassWrapper dClass = new DclassWrapper() { - @Override - protected void finalize() { - if (isDClassInitialized) destroyUA(); - } - }; + final private static dClass dClass = new dClass(dtree_path); - /** - * Initializing the dClass JNI wrapper loads the dtree data (~2MB) into memory. - * By using a static wrapper we avoid reloading the db, but we must synchronize - * access to settle a race during classification. - */ synchronized private static Map classifyUA(String ua) { - if (!isDClassInitialized) { - dClass.initUA(); - isDClassInitialized = true; - } - return dClass.classifyUA(ua); + + return dClass.classify(ua); } diff --git a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassLoaderTest.java b/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassLoaderTest.java deleted file mode 100644 index 71d94f0..0000000 --- a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassLoaderTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - Copyright (C) 2012 Wikimedia Foundation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.wikimedia.analytics.dclassjni; - -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; - - -public class DclassLoaderTest { - @Test - public void testDclassLoader() { - DclassWrapper dw = new DclassWrapper(); - dw.initUA(); - assertNotNull(dw); - } - -} diff --git a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java b/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java new file mode 100644 index 0000000..7a402f7 --- /dev/null +++ b/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java @@ -0,0 +1,45 @@ +/** + Copyright (C) 2012 Wikimedia Foundation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.wikimedia.analytics.dclassjni; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.junit.Test; + +import dclass.dClass; + + + +public class DclassWrapperTest { + + public final String dtree_path = "/usr/share/libdclass/openddr.dtree"; + @Test + public void testDclassWrapper() { + String userAgentSample = "Mozilla/5.0 (Linux; U; Android 2.2; en; HTC Aria A6380 Build/ERE27) AppleWebKit/540.13+ (KHTML, like Gecko) Version/3.1 Mobile Safari/524.15.0"; + dClass dw = new dClass(dtree_path); + Map<String, String> c = dw.classify(userAgentSample); + assertEquals("Vendor as expected", "HTC" , c.get("vendor")); + assertEquals("Model as expected" , "A6380", c.get("model")); + assertEquals("device_os as expected","Android",c.get("device_os")); + assertEquals("is_tablet as expected","false",c.get("is_tablet")); + + } + +} diff --git a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java b/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java new file mode 100644 index 0000000..bd84f9a --- /dev/null +++ b/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java @@ -0,0 +1,20 @@ +package org.wikimedia.analytics.dclassjni; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import org.wikimedia.analytics.dclassjni.DeviceClassification; + +public class DeviceClassificationTest { + + @Test + public void testDeviceClassification() { + String userAgentSample = "Mozilla/5.0 (Linux; U; Android 2.2; en; HTC Aria A6380 Build/ERE27) AppleWebKit/540.13+ (KHTML, like Gecko) Version/3.1 Mobile Safari/524.15.0"; + DeviceClassification d = new DeviceClassification(); + d.classifyUseragent(userAgentSample); + + assertEquals("Vendor as expected", "HTC" , d.getVendor()); + assertEquals("Model as expected" , "A6380", d.getModel()); + } +} -- To view, visit https://gerrit.wikimedia.org/r/75349 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I3621ed24a83fe8cfdc431e74c3867e7b92b5603c Gerrit-PatchSet: 11 Gerrit-Project: analytics/kraken Gerrit-Branch: master Gerrit-Owner: Stefan.petrea <[email protected]> Gerrit-Reviewer: Diederik <[email protected]> Gerrit-Reviewer: QChris <[email protected]> Gerrit-Reviewer: Stefan.petrea <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
