Author: lewismc
Date: Sat Jan 10 23:24:58 2015
New Revision: 1650829
URL: http://svn.apache.org/r1650829
Log:
NUTCH-1660 Index filter for Page's latitude and longitudex
Added:
nutch/trunk/src/plugin/index-geoip/
nutch/trunk/src/plugin/index-geoip/build.xml
nutch/trunk/src/plugin/index-geoip/ivy.xml
nutch/trunk/src/plugin/index-geoip/plugin.xml
nutch/trunk/src/plugin/index-geoip/src/
nutch/trunk/src/plugin/index-geoip/src/java/
nutch/trunk/src/plugin/index-geoip/src/java/org/
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/package-info.java
nutch/trunk/src/plugin/index-geoip/src/test/
nutch/trunk/src/plugin/index-geoip/src/test/org/
nutch/trunk/src/plugin/index-geoip/src/test/org/apache/
nutch/trunk/src/plugin/index-geoip/src/test/org/apache/nutch/
nutch/trunk/src/plugin/index-geoip/src/test/org/apache/nutch/indexer/
nutch/trunk/src/plugin/index-geoip/src/test/org/apache/nutch/indexer/geoip/
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/build.xml
nutch/trunk/conf/nutch-default.xml
nutch/trunk/conf/schema-solr4.xml
nutch/trunk/conf/schema.xml
nutch/trunk/conf/solrindex-mapping.xml
nutch/trunk/default.properties
nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
nutch/trunk/src/plugin/build.xml
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sat Jan 10 23:24:58 2015
@@ -2,6 +2,8 @@ Nutch Change Log
Nutch Current Development 1.10-SNAPSHOT
+* NUTCH-1660 Index filter for Page's latitude and longitude (Yasin Kılınç,
lewismc)
+
* NUTCH-1140 index-more plugin, resetTitle creates multiple values in title
field (Joe Liedtke, kaveh minooie via snagel)
* NUTCH-1904 Schema for Solr4 doesn't include _version_ field (mattmann)
Modified: nutch/trunk/build.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/build.xml?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/build.xml (original)
+++ nutch/trunk/build.xml Sat Jan 10 23:24:58 2015
@@ -176,6 +176,7 @@
<packageset dir="${plugins.dir}/index-basic/src/java"/>
<packageset dir="${plugins.dir}/index-metadata/src/java"/>
<packageset dir="${plugins.dir}/index-more/src/java"/>
+ <packageset dir="${plugins.dir}/index-geoip/src/java"/>
<packageset dir="${plugins.dir}/index-static/src/java"/>
<packageset dir="${plugins.dir}/indexer-dummy/src/java"/>
<packageset dir="${plugins.dir}/indexer-elastic/src/java/" />
@@ -579,6 +580,7 @@
<packageset dir="${plugins.dir}/headings/src/java"/>
<packageset dir="${plugins.dir}/index-anchor/src/java"/>
<packageset dir="${plugins.dir}/index-basic/src/java"/>
+ <packageset dir="${plugins.dir}/index-geoip/src/java"/>
<packageset dir="${plugins.dir}/index-metadata/src/java"/>
<packageset dir="${plugins.dir}/index-more/src/java"/>
<packageset dir="${plugins.dir}/index-static/src/java"/>
@@ -958,6 +960,8 @@
<source path="${plugins.dir}/index-anchor/src/test/" />
<source path="${plugins.dir}/index-basic/src/java/" />
<source path="${plugins.dir}/index-basic/src/test/" />
+ <source path="${plugins.dir}/index-geoip/src/java/" />
+ <source path="${plugins.dir}/index-geoip/src/test/" />
<source path="${plugins.dir}/indexer-dummy/src/java/" />
<source path="${plugins.dir}/indexer-solr/src/java/" />
<source path="${plugins.dir}/indexer-elastic/src/java/" />
Modified: nutch/trunk/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Sat Jan 10 23:24:58 2015
@@ -1373,6 +1373,36 @@
</description>
</property>
+<!-- index-geoip plugin properties -->
+<property>
+ <name>index.geoip.usage</name>
+ <value>insightsService</value>
+ <description>
+ A string representing the information source to be used for GeoIP information
+ association. Either enter 'cityDatabase', 'connectionTypeDatabase',
+ 'domainDatabase', 'ispDatabase' or 'insightsService'. If you wish to use any
one of the
+ Database options, you should make one of GeoIP2-City.mmdb,
GeoIP2-Connection-Type.mmdb,
+ GeoIP2-Domain.mmdb or GeoIP2-ISP.mmdb files respectively available on the
classpath and
+ available at runtime.
+ </description>
+</property>
+
+<property>
+ <name>index.geoip.userid</name>
+ <value></value>
+ <description>
+ The userId associated with the GeoIP2 Precision Services account.
+ </description>
+</property>
+
+<property>
+ <name>index.geoip.licensekey</name>
+ <value></value>
+ <description>
+ The license key associated with the GeoIP2 Precision Services account.
+ </description>
+</property>
+
<!-- parse-metatags plugin properties -->
<property>
<name>metatags.names</name>
Modified: nutch/trunk/conf/schema-solr4.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/schema-solr4.xml?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/conf/schema-solr4.xml (original)
+++ nutch/trunk/conf/schema-solr4.xml Sat Jan 10 23:24:58 2015
@@ -79,7 +79,9 @@
Note: For faster range queries, consider the tdate type
-->
<fieldType name="date" class="solr.TrieDateField" omitNorms="true"
precisionStep="0" positionIncrementGap="0"/>
-
+
+ <fieldType name="location" class="solr.LatLonType"
subFieldSuffix="_coordinate"/>
+
<!-- A Trie based date field for faster date range queries and date
faceting. -->
<fieldType name="tdate" class="solr.TrieDateField" omitNorms="true"
precisionStep="6" positionIncrementGap="0"/>
@@ -298,6 +300,25 @@
any data added to them will be ignored outright. -->
<fieldtype name="ignored" stored="false" indexed="false"
multiValued="true" class="solr.StrField" />
+ <!-- boolean type: "true" or "false" -->
+ <fieldType name="boolean" class="solr.BoolField"
sortMissingLast="true"/>
+
+ <!-- sortMissingLast and sortMissingFirst attributes are optional
attributes are
+ currently supported on types that are sorted internally as strings
+ and on numeric types.
+ This includes "string","boolean", and, as of 3.5 (and 4.x),
+ int, float, long, date, double, including the "Trie" variants.
+ - If sortMissingLast="true", then a sort on this field will cause
documents
+ without the field to come after documents with the field,
+ regardless of the requested sort order (asc or desc).
+ - If sortMissingFirst="true", then a sort on this field will cause
documents
+ without the field to come before documents with the field,
+ regardless of the requested sort order.
+ - If sortMissingLast="false" and sortMissingFirst="false" (the default),
+ then default lucene sorting will be used which places docs without the
+ field first in an ascending sort and last in a descending sort.
+ -->
+
</types>
<fields>
@@ -318,6 +339,41 @@
<field name="cache" type="string" stored="true" indexed="false"/>
<field name="tstamp" type="date" stored="true" indexed="false"/>
+ <!-- fields for index-geoip plugin -->
+ <field name="ip" type="string" stored="true" indexed="true" />
+ <field name="cityName" type="string" stored="true" indexed="true" />
+ <field name="cityConfidence" type="int" stored="true" indexed="true" />
+ <field name="cityGeoNameId" type="int" stored="true" indexed="true" />
+ <field name="continentCode" type="string" stored="true" indexed="true" />
+ <field name="continentGeoNameId" type="int" stored="true" indexed="true" />
+ <field name="contentName" type="string" stored="true" indexed="true" />
+ <field name="countryIsoCode" type="string" stored="true" indexed="true"/>
+ <field name="countryName" type="string" stored="true" indexed="true" />
+ <field name="countryConfidence" type="int" stored="true" indexed="true"/>
+ <field name="countryGeoNameId" type="int" stored="true" indexed="true"/>
+ <field name="latLon" type="string" stored="true" indexed="true"/>
+ <field name="accRadius" type="int" stored="true" indexed="true"/>
+ <field name="timeZone" type="string" stored="true" indexed="true"/>
+ <field name="metroCode" type="int" stored="true" indexed="true" />
+ <field name="postalCode" type="string" stored="true" indexed="true" />
+ <field name="postalConfidence" type="int" stored="true" indexed="true" />
+ <field name="countryType" type="string" stored="true" indexed="true" />
+ <field name="subDivName" type="string" stored="true" indexed="true" />
+ <field name="subDivIsoCode" type="string" stored="true" indexed="true" />
+ <field name="subDivConfidence" type="int" stored="true" indexed="true" />
+ <field name="subDivGeoNameId" type="int" stored="true" indexed="true" />
+ <field name="autonSystemNum" type="int" stored="true" indexed="true" />
+ <field name="autonSystemOrg" type="string" stored="true" indexed="true" />
+ <field name="domain" type="string" stored="true" indexed="true" />
+ <field name="isp" type="string" stored="true" indexed="true" />
+ <field name="org" type="string" stored="true" indexed="true" />
+ <field name="userType" type="string" stored="true" indexed="true" />
+ <field name="isAnonProxy" type="boolean" stored="true" indexed="true" />
+ <field name="isSatelitteProv" type="boolean" stored="true" indexed="true"
/>
+ <field name="connType" type="string" stored="true" indexed="true" />
+
+ <dynamicField name="*_coordinate" type="tdouble" indexed="true"
stored="false"/>
+
<!-- catch-all field -->
<field name="text" type="text_general" stored="false" indexed="true"
multiValued="true"/>
@@ -363,5 +419,5 @@
<copyField source="title" dest="text"/>
<copyField source="anchor" dest="text"/>
<copyField source="author" dest="text"/>
-
+ <copyField source="latLon" dest="location"/>
</schema>
Modified: nutch/trunk/conf/schema.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/schema.xml?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/conf/schema.xml (original)
+++ nutch/trunk/conf/schema.xml Sat Jan 10 23:24:58 2015
@@ -38,6 +38,7 @@
omitNorms="true" positionIncrementGap="0"/>
<fieldType name="date" class="solr.TrieDateField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="location" class="solr.LatLonType"
subFieldSuffix="_coordinate"/>
<fieldType name="text" class="solr.TextField"
positionIncrementGap="100">
@@ -64,6 +65,24 @@
generateWordParts="1" generateNumberParts="1"/>
</analyzer>
</fieldType>
+ <!-- boolean type: "true" or "false" -->
+ <fieldType name="boolean" class="solr.BoolField"
sortMissingLast="true"/>
+
+ <!-- sortMissingLast and sortMissingFirst attributes are optional
attributes are
+ currently supported on types that are sorted internally as strings
+ and on numeric types.
+ This includes "string","boolean", and, as of 3.5 (and 4.x),
+ int, float, long, date, double, including the "Trie" variants.
+ - If sortMissingLast="true", then a sort on this field will cause
documents
+ without the field to come after documents with the field,
+ regardless of the requested sort order (asc or desc).
+ - If sortMissingFirst="true", then a sort on this field will cause
documents
+ without the field to come before documents with the field,
+ regardless of the requested sort order.
+ - If sortMissingLast="false" and sortMissingFirst="false" (the default),
+ then default lucene sorting will be used which places docs without the
+ field first in an ascending sort and last in a descending sort.
+ -->
</types>
<fields>
<field name="id" type="string" stored="true" indexed="true"
@@ -83,6 +102,43 @@
<field name="title" type="text" stored="true" indexed="true"/>
<field name="cache" type="string" stored="true" indexed="false"/>
<field name="tstamp" type="date" stored="true" indexed="false"/>
+
+ <!-- fields for index-geoip plugin -->
+ <field name="ip" type="string" stored="true" indexed="true" />
+ <field name="cityName" type="string" stored="true" indexed="true" />
+ <field name="cityConfidence" type="int" stored="true" indexed="true" />
+ <field name="cityGeoNameId" type="int" stored="true" indexed="true" />
+ <field name="continentCode" type="string" stored="true" indexed="true"
/>
+ <field name="continentGeoNameId" type="int" stored="true"
indexed="true" />
+ <field name="contentName" type="string" stored="true" indexed="true" />
+ <field name="countryIsoCode" type="string" stored="true"
indexed="true"/>
+ <field name="countryName" type="string" stored="true" indexed="true" />
+ <field name="countryConfidence" type="int" stored="true"
indexed="true"/>
+ <field name="countryGeoNameId" type="int" stored="true"
indexed="true"/>
+ <field name="latLon" type="string" stored="true" indexed="true"/>
+ <field name="accRadius" type="int" stored="true" indexed="true"/>
+ <field name="timeZone" type="string" stored="true" indexed="true"/>
+ <field name="metroCode" type="int" stored="true" indexed="true" />
+ <field name="postalCode" type="string" stored="true" indexed="true" />
+ <field name="postalConfidence" type="int" stored="true" indexed="true"
/>
+ <field name="countryType" type="string" stored="true" indexed="true" />
+ <field name="subDivName" type="string" stored="true" indexed="true" />
+ <field name="subDivIsoCode" type="string" stored="true" indexed="true"
/>
+ <field name="subDivConfidence" type="int" stored="true" indexed="true"
/>
+ <field name="subDivGeoNameId" type="int" stored="true" indexed="true"
/>
+ <field name="autonSystemNum" type="int" stored="true" indexed="true" />
+ <field name="autonSystemOrg" type="string" stored="true"
indexed="true" />
+ <field name="domain" type="string" stored="true" indexed="true" />
+ <field name="isp" type="string" stored="true" indexed="true" />
+ <field name="org" type="string" stored="true" indexed="true" />
+ <field name="userType" type="string" stored="true" indexed="true" />
+ <field name="isAnonProxy" type="boolean" stored="true" indexed="true"
/>
+ <field name="isSatelitteProv" type="boolean" stored="true"
indexed="true" />
+ <field name="connType" type="string" stored="true" indexed="true" />
+
+
+
+ <dynamicField name="*_coordinate" type="tdouble" indexed="true"
stored="false"/>
<!-- fields for index-anchor plugin -->
<field name="anchor" type="string" stored="true" indexed="true"
@@ -137,5 +193,6 @@
<copyField source="title" dest="text"/>
<copyField source="anchor" dest="text"/>
<copyField source="author" dest="text"/>
+ <copyField source="latLon" dest="location"/>
</schema>
Modified: nutch/trunk/conf/solrindex-mapping.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/solrindex-mapping.xml?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/conf/solrindex-mapping.xml (original)
+++ nutch/trunk/conf/solrindex-mapping.xml Sat Jan 10 23:24:58 2015
@@ -17,8 +17,8 @@
-->
<mapping>
- <!-- Simple mapping of fields created by Nutch IndexingFilters
- to fields defined (and expected) in Solr schema.xml.
+ <!-- Simple mapping of fields created by Nutch IndexingFilters
+ to fields defined (and expected) in Solr schema.xml.
Any fields in NutchDocument that match a name defined
in field/@source will be renamed to the corresponding
@@ -30,14 +30,14 @@
uniqueKey has the same meaning as in Solr schema.xml
and defaults to "id" if not defined.
-->
- <fields>
- <field dest="content" source="content"/>
- <field dest="title" source="title"/>
- <field dest="host" source="host"/>
- <field dest="segment" source="segment"/>
- <field dest="boost" source="boost"/>
- <field dest="digest" source="digest"/>
- <field dest="tstamp" source="tstamp"/>
- </fields>
- <uniqueKey>id</uniqueKey>
+ <fields>
+ <field dest="content" source="content"/>
+ <field dest="title" source="title"/>
+ <field dest="host" source="host"/>
+ <field dest="segment" source="segment"/>
+ <field dest="boost" source="boost"/>
+ <field dest="digest" source="digest"/>
+ <field dest="tstamp" source="tstamp"/>
+ </fields>
+ <uniqueKey>id</uniqueKey>
</mapping>
Modified: nutch/trunk/default.properties
URL:
http://svn.apache.org/viewvc/nutch/trunk/default.properties?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/default.properties (original)
+++ nutch/trunk/default.properties Sat Jan 10 23:24:58 2015
@@ -148,6 +148,7 @@ plugins.index=\
org.apache.nutch.indexer.anchor*:\
org.apache.nutch.indexer.basic*:\
org.apache.nutch.indexer.feed*:\
+ org.apache.nutch.indexer.geoip*:\
org.apache.nutch.indexer.metadata*:\
org.apache.nutch.indexer.more*:\
org.apache.nutch.indexer.static*:\
Modified:
nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
(original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
Sat Jan 10 23:24:58 2015
@@ -17,7 +17,6 @@
package org.apache.nutch.indexer;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
Modified: nutch/trunk/src/plugin/build.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/build.xml?rev=1650829&r1=1650828&r2=1650829&view=diff
==============================================================================
--- nutch/trunk/src/plugin/build.xml (original)
+++ nutch/trunk/src/plugin/build.xml Sat Jan 10 23:24:58 2015
@@ -31,6 +31,7 @@
<ant dir="headings" target="deploy"/>
<ant dir="index-basic" target="deploy"/>
<ant dir="index-anchor" target="deploy"/>
+ <ant dir="index-geoip" target="deploy"/>
<ant dir="index-more" target="deploy"/>
<ant dir="index-static" target="deploy"/>
<ant dir="index-metadata" target="deploy"/>
@@ -83,6 +84,7 @@
<ant dir="creativecommons" target="test"/>
<ant dir="index-basic" target="test"/>
<ant dir="index-anchor" target="test"/>
+ <ant dir="index-geoip" target="test"/>
<ant dir="index-more" target="test"/>
<ant dir="index-static" target="test"/>
<ant dir="language-identifier" target="test"/>
@@ -122,6 +124,7 @@
<ant dir="headings" target="clean"/>
<ant dir="index-basic" target="clean"/>
<ant dir="index-anchor" target="clean"/>
+ <ant dir="index-geoip" target="clean"/>
<ant dir="index-more" target="clean"/>
<ant dir="index-static" target="clean"/>
<ant dir="index-metadata" target="clean"/>
Added: nutch/trunk/src/plugin/index-geoip/build.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-geoip/build.xml?rev=1650829&view=auto
==============================================================================
--- nutch/trunk/src/plugin/index-geoip/build.xml (added)
+++ nutch/trunk/src/plugin/index-geoip/build.xml Sat Jan 10 23:24:58 2015
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="index-geoip" default="jar-core">
+
+ <import file="../build-plugin.xml"/>
+ <target name="init-plugin">
+ <echo>Copying MaxMind GeoIP .mmdb files to build</echo>
+ <copy todir="${build.classes}">
+ <fileset dir="${src.dir}" includes="**/*.mmdb" />
+ </copy>
+ </target>
+</project>
Added: nutch/trunk/src/plugin/index-geoip/ivy.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-geoip/ivy.xml?rev=1650829&view=auto
==============================================================================
--- nutch/trunk/src/plugin/index-geoip/ivy.xml (added)
+++ nutch/trunk/src/plugin/index-geoip/ivy.xml Sat Jan 10 23:24:58 2015
@@ -0,0 +1,42 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../..//ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ <dependency org="com.maxmind.geoip2" name="geoip2" rev="2.1.0" />
+ </dependencies>
+
+</ivy-module>
Added: nutch/trunk/src/plugin/index-geoip/plugin.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-geoip/plugin.xml?rev=1650829&view=auto
==============================================================================
--- nutch/trunk/src/plugin/index-geoip/plugin.xml (added)
+++ nutch/trunk/src/plugin/index-geoip/plugin.xml Sat Jan 10 23:24:58 2015
@@ -0,0 +1,43 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<plugin
+ id="index-geoip"
+ name="GeoIP2 Indexing Filter"
+ version="1.0.0"
+ provider-name="nutch.org">
+
+
+ <runtime>
+ <library name="index-geoip.jar">
+ <export name="*"/>
+ </library>
+ <library name="geoip2-2.1.0.jar"/>
+ </runtime>
+
+ <requires>
+ <import plugin="nutch-extensionpoints"/>
+ </requires>
+
+ <extension id="org.apache.nutch.indexer.geoip"
+ name="Nutch GeoIP2 Indexing Filter"
+ point="org.apache.nutch.indexer.IndexingFilter">
+ <implementation id="GeoIPIndexingFilter"
+
class="org.apache.nutch.indexer.geoip.GeoIPIndexingFilter"/>
+ </extension>
+
+</plugin>
+
Added:
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java?rev=1650829&view=auto
==============================================================================
---
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
(added)
+++
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java
Sat Jan 10 23:24:58 2015
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer.geoip;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.apache.nutch.indexer.NutchDocument;
+
+import com.maxmind.geoip2.DatabaseReader;
+import com.maxmind.geoip2.WebServiceClient;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.maxmind.geoip2.model.InsightsResponse;
+import com.maxmind.geoip2.model.CityResponse;
+import com.maxmind.geoip2.model.ConnectionTypeResponse;
+import com.maxmind.geoip2.model.CountryResponse;
+import com.maxmind.geoip2.model.DomainResponse;
+import com.maxmind.geoip2.model.IspResponse;
+import com.maxmind.geoip2.record.City;
+import com.maxmind.geoip2.record.Continent;
+import com.maxmind.geoip2.record.Country;
+import com.maxmind.geoip2.record.Location;
+import com.maxmind.geoip2.record.Postal;
+import com.maxmind.geoip2.record.RepresentedCountry;
+import com.maxmind.geoip2.record.Subdivision;
+import com.maxmind.geoip2.record.Traits;
+
+/**
+ * <p>Simple utility class which enables efficient, structured
+ * {@link org.apache.nutch.indexer.NutchDocument} building based on input
+ * from {@link GeoIPIndexingFilter}, where configuration is also read.</p>
+ * <p>Based on the nature of the input, this class wraps factory type
+ * implementations for populating {@link
org.apache.nutch.indexer.NutchDocument}'s
+ * with the correct {@link org.apache.nutch.indexer.NutchField} information.
+ *
+ */
+public class GeoIPDocumentCreator {
+
+ /**
+ * Default constructor.
+ */
+ public GeoIPDocumentCreator() {
+ }
+
+ public static NutchDocument createDocFromInsightsService(String serverIp,
+ NutchDocument doc, WebServiceClient client) throws UnknownHostException,
IOException, GeoIp2Exception {
+ doc.add("ip", serverIp);
+ InsightsResponse response =
client.insights(InetAddress.getByName(serverIp));
+ //CityResponse response = client.city(InetAddress.getByName(serverIp));
+
+ City city = response.getCity();
+ doc.add("cityName", city.getName()); // 'Minneapolis'
+ doc.add("cityConfidence", city.getConfidence()); // 50
+ doc.add("cityGeoNameId", city.getGeoNameId());
+
+ Continent continent = response.getContinent();
+ doc.add("continentCode", continent.getCode());
+ doc.add("continentGeoNameId", continent.getGeoNameId());
+ doc.add("continentName", continent.getName());
+
+ Country country = response.getCountry();
+ doc.add("countryIsoCode", country.getIsoCode()); // 'US'
+ doc.add("countryName", country.getName()); // 'United States'
+ doc.add("countryConfidence", country.getConfidence()); // 99
+ doc.add("countryGeoName", country.getGeoNameId());
+
+ Location location = response.getLocation();
+ doc.add("latLon", location.getLatitude() + "," + location.getLongitude());
// 44.9733, -93.2323
+ doc.add("accRadius", location.getAccuracyRadius()); // 3
+ doc.add("timeZone", location.getTimeZone()); // 'America/Chicago'
+ doc.add("metroCode", location.getMetroCode());
+
+ Postal postal = response.getPostal();
+ doc.add("postalCode", postal.getCode()); // '55455'
+ doc.add("postalConfidence", postal.getConfidence()); // 40
+
+ RepresentedCountry rCountry = response.getRepresentedCountry();
+ doc.add("countryType", rCountry.getType());
+
+ Subdivision subdivision = response.getMostSpecificSubdivision();
+ doc.add("subDivName", subdivision.getName()); // 'Minnesota'
+ doc.add("subDivIdoCode", subdivision.getIsoCode()); // 'MN'
+ doc.add("subDivConfidence", subdivision.getConfidence()); // 90
+ doc.add("subDivGeoNameId", subdivision.getGeoNameId());
+
+ Traits traits = response.getTraits();
+ doc.add("autonSystemNum", traits.getAutonomousSystemNumber());
+ doc.add("autonSystemOrg", traits.getAutonomousSystemOrganization());
+ doc.add("domain", traits.getDomain());
+ doc.add("isp", traits.getIsp());
+ doc.add("org", traits.getOrganization());
+ doc.add("userType", traits.getUserType());
+ doc.add("isAnonProxy", traits.isAnonymousProxy());
+ doc.add("isSatelliteProv", traits.isSatelliteProvider());
+ return doc;
+ }
+
+ @SuppressWarnings("unused")
+ public static NutchDocument createDocFromCityService(String serverIp,
+ NutchDocument doc, WebServiceClient client) throws UnknownHostException,
IOException, GeoIp2Exception {
+ CityResponse response = client.city(InetAddress.getByName(serverIp));
+ return doc;
+ }
+
+ @SuppressWarnings("unused")
+ public static NutchDocument createDocFromCountryService(String serverIp,
+ NutchDocument doc, WebServiceClient client) throws UnknownHostException,
IOException, GeoIp2Exception {
+ CountryResponse response =
client.country(InetAddress.getByName(serverIp));
+ return doc;
+ }
+
+ public static NutchDocument createDocFromIspDb(String serverIp,
NutchDocument doc,
+ DatabaseReader reader) throws UnknownHostException, IOException,
GeoIp2Exception {
+ IspResponse response = reader.isp(InetAddress.getByName(serverIp));
+ doc.add("ip", serverIp);
+ doc.add("autonSystemNum", response.getAutonomousSystemNumber());
+ doc.add("autonSystemOrg", response.getAutonomousSystemOrganization());
+ doc.add("isp", response.getIsp());
+ doc.add("org", response.getOrganization());
+ return doc;
+ }
+
+ public static NutchDocument createDocFromDomainDb(String serverIp,
NutchDocument doc,
+ DatabaseReader reader) throws UnknownHostException, IOException,
GeoIp2Exception {
+ DomainResponse response = reader.domain(InetAddress.getByName(serverIp));
+ doc.add("ip", serverIp);
+ doc.add("domain", response.getDomain());
+ return doc;
+ }
+
+ public static NutchDocument createDocFromConnectionDb(String serverIp,
+ NutchDocument doc, DatabaseReader reader) throws UnknownHostException,
IOException, GeoIp2Exception {
+ ConnectionTypeResponse response =
reader.connectionType(InetAddress.getByName(serverIp));
+ doc.add("ip", serverIp);
+ doc.add("connType", response.getConnectionType().toString());
+ return doc;
+ }
+
+ public static NutchDocument createDocFromCityDb(String serverIp,
NutchDocument doc,
+ DatabaseReader reader) throws UnknownHostException, IOException,
GeoIp2Exception {
+ doc.add("ip", serverIp);
+ CityResponse response = reader.city(InetAddress.getByName(serverIp));
+
+ City city = response.getCity();
+ doc.add("cityName", city.getName()); // 'Minneapolis'
+ doc.add("cityConfidence", city.getConfidence()); // 50
+ doc.add("cityGeoNameId", city.getGeoNameId());
+
+ Continent continent = response.getContinent();
+ doc.add("continentCode", continent.getCode());
+ doc.add("continentGeoNameId", continent.getGeoNameId());
+ doc.add("continentName", continent.getName());
+
+ Country country = response.getCountry();
+ doc.add("countryIsoCode", country.getIsoCode()); // 'US'
+ doc.add("countryName", country.getName()); // 'United States'
+ doc.add("countryConfidence", country.getConfidence()); // 99
+ doc.add("countryGeoName", country.getGeoNameId());
+
+ Location location = response.getLocation();
+ doc.add("latLon", location.getLatitude() + "," + location.getLongitude());
// 44.9733, -93.2323
+ doc.add("accRadius", location.getAccuracyRadius()); // 3
+ doc.add("timeZone", location.getTimeZone()); // 'America/Chicago'
+ doc.add("metroCode", location.getMetroCode());
+
+ Postal postal = response.getPostal();
+ doc.add("postalCode", postal.getCode()); // '55455'
+ doc.add("postalConfidence", postal.getConfidence()); // 40
+
+ RepresentedCountry rCountry = response.getRepresentedCountry();
+ doc.add("countryType", rCountry.getType());
+
+ Subdivision subdivision = response.getMostSpecificSubdivision();
+ doc.add("subDivName", subdivision.getName()); // 'Minnesota'
+ doc.add("subDivIdoCode", subdivision.getIsoCode()); // 'MN'
+ doc.add("subDivConfidence", subdivision.getConfidence()); // 90
+ doc.add("subDivGeoNameId", subdivision.getGeoNameId());
+ return doc;
+ }
+
+}
Added:
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java?rev=1650829&view=auto
==============================================================================
---
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java
(added)
+++
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java
Sat Jan 10 23:24:58 2015
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer.geoip;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.Inlinks;
+import org.apache.nutch.indexer.IndexingException;
+import org.apache.nutch.indexer.IndexingFilter;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.maxmind.geoip2.DatabaseReader;
+import com.maxmind.geoip2.WebServiceClient;
+
+/**
+ * <p>This plugin implements an indexing filter which takes
+ * advantage of the
+ * <a href="https://github.com/maxmind/GeoIP2-java";>GeoIP2-java API</a>.</p>
+ * <p>The third party library distribution provides an API for the GeoIP2
+ * <a href="http://dev.maxmind.com/geoip/geoip2/web-services";>Precision web
services</a>
+ * and <a
href="http://dev.maxmind.com/geoip/geoip2/downloadable";>databases</a>.
+ * The API also works with the free
+ * <a href="http://dev.maxmind.com/geoip/geoip2/geolite2/";>GeoLite2
databases</a>.</p>
+ * <p>Depending on the service level agreement, you have with the GeoIP
service provider,
+ * the plugin can add a number of the following fields to the index data model:
+ * <ol>
+ * <li>Continent</li>
+ * <li>Country</li>
+ * <li>Regional Subdivision</li>
+ * <li>City</li>
+ * <li>Postal Code</li>
+ * <li>Latitude/Longitude</li>
+ * <li>ISP/Organization</li>
+ * <li>AS Number</li>
+ * <li>Confidence Factors</li>
+ * <li>Radius</li>
+ * <li>User Type</li>
+ * </ol></p>
+ *
+ * <p>Some of the services are documented at the
+ * <a href="https://www.maxmind.com/en/geoip2-precision-services";>GeoIP2
Precision Services</a>
+ * webpage where more information can be obtained.</p>
+ *
+ * <p>You should also consult the following three properties in
<code>nutch-site.xml</code></p>
+ * <pre>
+ * {@code
+ *<!-- index-geoip plugin properties -->
+<property>
+ <name>index.geoip.usage</name>
+ <value>insightsService</value>
+ <description>
+ A string representing the information source to be used for GeoIP information
+ association. Either enter 'cityDatabase', 'connectionTypeDatabase',
+ 'domainDatabase', 'ispDatabase' or 'insightsService'. If you wish to use any
one of the
+ Database options, you should make one of GeoIP2-City.mmdb,
GeoIP2-Connection-Type.mmdb,
+ GeoIP2-Domain.mmdb or GeoIP2-ISP.mmdb files respectively available on the
Hadoop classpath
+ and available at runtime. This can be achieved by adding it to
$NUTCH_HOME/conf
+ </description>
+</property>
+
+<property>
+ <name>index.geoip.userid</name>
+ <value></value>
+ <description>
+ The userId associated with the GeoIP2 Precision Services account.
+ </description>
+</property>
+
+<property>
+ <name>index.geoip.licensekey</name>
+ <value></value>
+ <description>
+ The license key associated with the GeoIP2 Precision Services account.
+ </description>
+</property>
+}
+ * </pre>
+ *
+ */
+public class GeoIPIndexingFilter implements IndexingFilter {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(GeoIPIndexingFilter.class);
+
+ private Configuration conf;
+
+ private String usage = null;
+
+ private File geoDb = null;
+
+ WebServiceClient client = null;
+
+ DatabaseReader reader = null;
+
+ //private AbstractResponse response = null;
+
+ /**
+ * Default constructor for this plugin
+ */
+ public GeoIPIndexingFilter() {
+ }
+
+ /**
+ * @see org.apache.hadoop.conf.Configurable#getConf()
+ */
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * @see
org.apache.hadoop.conf.Configurable#setConf(org.apache.hadoop.conf.Configuration)
+ */
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ String use = conf.get("index.geoip.usage", "insightsService");
+ LOG.debug("GeoIP usage medium set to: {}", use);
+ if (use.equalsIgnoreCase("cityDatabase")) {
+ try {
+ geoDb = new File(conf.getResource("GeoIP2-City.mmdb").getFile());
+ buildDb();
+ } catch (Exception e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ } else if (use.equalsIgnoreCase("connectionTypeDatabase")) {
+ try {
+ geoDb = new
File(conf.getResource("GeoIP2-Connection-Type.mmdb").getFile());
+ buildDb();
+ } catch (Exception e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ } else if (use.equalsIgnoreCase("domainDatabase")) {
+ try {
+ geoDb = new File(conf.getResource("GeoIP2-Domain.mmdb").getFile());
+ buildDb();
+ } catch (Exception e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ } else if (use.equalsIgnoreCase("ispDatabase")) {
+ try {
+ geoDb = new File(conf.getResource("GeoIP2-ISP.mmdb").getFile());
+ buildDb();
+ } catch (Exception e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ } else if (use.equalsIgnoreCase("insightsService")) {
+ client = new WebServiceClient.Builder(
+ conf.getInt("index.geoip.userid", 12345),
conf.get("index.geoip.licensekey")).build();
+ }
+ usage = use;
+ }
+
+ private void buildDb() {
+ try {
+ reader = new DatabaseReader.Builder(geoDb).build();
+ } catch (IOException e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
+
+ /**
+ *
+ * @see
org.apache.nutch.indexer.IndexingFilter#filter(org.apache.nutch.indexer.NutchDocument,
org.apache.nutch.parse.Parse, org.apache.hadoop.io.Text,
org.apache.nutch.crawl.CrawlDatum, org.apache.nutch.crawl.Inlinks)
+ */
+ @Override
+ public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
+ CrawlDatum datum, Inlinks inlinks) throws IndexingException {
+ return addServerGeo(doc, parse.getData(), url.toString());
+ }
+
+ private NutchDocument addServerGeo(NutchDocument doc, ParseData data, String
url) {
+
+ if (conf.getBoolean("store.ip.address", false) == true) {
+ try {
+ String serverIp = data.getContentMeta().get("_ip_");
+ if (serverIp != null) {
+ if (usage.equalsIgnoreCase("cityDatabase")) {
+ doc = GeoIPDocumentCreator.createDocFromCityDb(serverIp, doc,
reader);
+ } else if (usage.equalsIgnoreCase("connectionTypeDatabase")) {
+ doc = GeoIPDocumentCreator.createDocFromConnectionDb(serverIp,
doc, reader);
+ } else if (usage.equalsIgnoreCase("domainDatabase")) {
+ doc = GeoIPDocumentCreator.createDocFromDomainDb(serverIp, doc,
reader);
+ } else if (usage.equalsIgnoreCase("ispDatabase")) {
+ doc = GeoIPDocumentCreator.createDocFromIspDb(serverIp, doc,
reader);
+ } else if (usage.equalsIgnoreCase("insightsService")) {
+ doc = GeoIPDocumentCreator.createDocFromInsightsService(serverIp,
doc, client);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error(e.getMessage());
+ e.printStackTrace();
+ }
+ }
+ return doc;
+ }
+
+}
Added:
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/package-info.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/package-info.java?rev=1650829&view=auto
==============================================================================
---
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/package-info.java
(added)
+++
nutch/trunk/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/package-info.java
Sat Jan 10 23:24:58 2015
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * <p>This plugin implements an indexing filter which takes
+ * advantage of the
+ * <a href="https://github.com/maxmind/GeoIP2-java";>GeoIP2-java API</a>.</p>
+ * <p>The third party library distribution provides an API for the GeoIP2
+ * <a href="http://dev.maxmind.com/geoip/geoip2/web-services";>Precision web
services</a>
+ * and <a
href="http://dev.maxmind.com/geoip/geoip2/downloadable";>databases</a>.
+ * The API also works with the free
+ * <a href="http://dev.maxmind.com/geoip/geoip2/geolite2/";>GeoLite2
databases</a>.
+ *
+ */
+package org.apache.nutch.indexer.geoip;
\ No newline at end of file
