Diederik has submitted this change and it was merged.
Change subject: Allow countryCode lookups for IPv6 addresses
......................................................................
Allow countryCode lookups for IPv6 addresses
We are lacking GeoIP city databases for IPv6 addresses, so our lookups
of location information for IPv6 addresses failed [1]. As some scripts
rely on such lookups to obtain countryCodes, and countryCodes can be
fed from the GeoIPv6.dat GeoIP database, we now at least fill country
information when trying get the location of IPv6 addresses.
[1] Pig logs showed
IO Exception while seting up segments
errors.
Change-Id: I2f350e1201dfc1af4f96d3176cfa15e4edf9500e
---
M
kraken-generic/src/main/java/org/wikimedia/analytics/kraken/geo/GeoIpLookup.java
A
kraken-generic/src/test/java/org/wikimedia/analytics/kraken/geo/GeoIpLookupTest.java
M
kraken-pig/src/test/java/org/wikimedia/analytics/kraken/pig/GeoIpLookupTest.java
3 files changed, 92 insertions(+), 2 deletions(-)
Approvals:
Diederik: Verified; Looks good to me, approved
diff --git
a/kraken-generic/src/main/java/org/wikimedia/analytics/kraken/geo/GeoIpLookup.java
b/kraken-generic/src/main/java/org/wikimedia/analytics/kraken/geo/GeoIpLookup.java
index c5e7571..e2fa502 100644
---
a/kraken-generic/src/main/java/org/wikimedia/analytics/kraken/geo/GeoIpLookup.java
+++
b/kraken-generic/src/main/java/org/wikimedia/analytics/kraken/geo/GeoIpLookup.java
@@ -209,7 +209,9 @@
}
/**
- *
+ * Gets the location for an Ip address.
+ * <p>
+ * For IPv6 addresses, only countryCode and countryName are set.
* @param ip
* @return location object
*/
@@ -221,7 +223,24 @@
location = ip4Lookup.getLocation(ip);
break;
case 6:
- location = ip6Lookup.getLocationV6(ip);
+ // Since we do not have access to a GeoIPCityv6.dat, we cannot
+ // determine the Location for IPv6 addresses.
+ // Trying to call
+ //
+ // location = ip6Lookup.getLocation(ip);
+ //
+ // with only GeoIPv6.dat results in
+ // IO Exception while seting up segments
+ // errors. However, we have access to GeoIPv6.dat which allows
+ // to at least get the country information. As some pig scripts
+ // rely on doGeoLookup (e.g.: pig/pageviews.pig), to arrive at
+ // a countryCode, we do the best we can to get transparent
+ // handling of that use case, and at least fill the
+ // country fields in the returned Location instance.
+ location = new Location();
+ com.maxmind.geoip.Country country = ip6Lookup.getCountryV6(ip);
+ location.countryCode = country.getCode();
+ location.countryName = country.getName();
break;
default:
break;
diff --git
a/kraken-generic/src/test/java/org/wikimedia/analytics/kraken/geo/GeoIpLookupTest.java
b/kraken-generic/src/test/java/org/wikimedia/analytics/kraken/geo/GeoIpLookupTest.java
new file mode 100644
index 0000000..579b780
--- /dev/null
+++
b/kraken-generic/src/test/java/org/wikimedia/analytics/kraken/geo/GeoIpLookupTest.java
@@ -0,0 +1,57 @@
+/**
+ * Copyright (C) 2013 Wikimedia Foundation
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
USA.
+ */
+package org.wikimedia.analytics.kraken.geo;
+
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import com.maxmind.geoip.Location;
+
+
+public class GeoIpLookupTest extends TestCase {
+
+ public void testDoGeoLookupIpv6US() throws IOException {
+ GeoIpLookup lookup = new GeoIpLookup("countryCode", "GeoIPCity",
+ "LOCAL");
+
+ // Arbitrary IPv6 address in the US
+ String ip = "2600:1011:b103:9999:6864:ac5e:1686:c20f";
+
+ Location location = lookup.doGeoLookup(ip);
+
+ assertNotNull("location is null", location);
+ assertEquals("US", location.countryCode);
+ assertEquals("United States", location.countryName);
+ }
+
+ public void testDoGeoLookupIpv6Europe() throws IOException {
+ GeoIpLookup lookup = new GeoIpLookup("countryCode", "GeoIPCity",
+ "LOCAL");
+
+ // Arbitrary IPv6 address in Germany
+ String ip = "2001:aa8:abcd:1234::2222";
+
+ Location location = lookup.doGeoLookup(ip);
+
+ assertNotNull("location is null", location);
+ assertEquals("DE", location.countryCode);
+ assertEquals("Germany", location.countryName);
+ }
+}
diff --git
a/kraken-pig/src/test/java/org/wikimedia/analytics/kraken/pig/GeoIpLookupTest.java
b/kraken-pig/src/test/java/org/wikimedia/analytics/kraken/pig/GeoIpLookupTest.java
index 62e884e..fc431e8 100644
---
a/kraken-pig/src/test/java/org/wikimedia/analytics/kraken/pig/GeoIpLookupTest.java
+++
b/kraken-pig/src/test/java/org/wikimedia/analytics/kraken/pig/GeoIpLookupTest.java
@@ -143,4 +143,18 @@
String countryCode = (String) output.get(0);
assertEquals("IN", countryCode);
}
+
+ /**
+ * Test country code for IPv6 address
+ * @throws IOException
+ */
+ @Test
+ public void testIpv6CountryCode() throws IOException {
+ GeoIpLookupEvalFunc geo = new GeoIpLookupEvalFunc("countryCode",
"GeoIPCity", "LOCAL");
+ this.input.set(0, "2001:ab0:abcd:1234::2222"); // Arbitrary IPv6
address in Germany
+ Tuple output = geo.exec(this.input);
+ assertNotNull("Returned tuple is null", output);
+ assertEquals("Returned tuple element cound does not contain match", 1,
output.size());
+ assertEquals("Returned tuple's value does not match", "DE",
output.get(0));
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/74595
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2f350e1201dfc1af4f96d3176cfa15e4edf9500e
Gerrit-PatchSet: 1
Gerrit-Project: analytics/kraken
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
Gerrit-Reviewer: Diederik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
