Author: markg
Date: Sun Oct 20 20:04:41 2013
New Revision: 1533959
URL: http://svn.apache.org/r1533959
Log:
OPENNLP-579
GeoEntityLinkerImpl: Implemented better scoring using Dice coefficient of
bigram, as well as highly improved scoring based on country context. Created an
NgramGenerator class and a FuzzyStringMatching class, assuming they would be
useful for other linker impls. Implemented Regex based discovery of
countrycontext, which enabled proximity based analysis of doctext
Multiple other small efficiencies in the GeoEntityLinker
Added:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazEntry.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazEntry.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/domain/BaseLink.java
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java?rev=1533959&r1=1533958&r2=1533959&view=diff
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java
(original)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java
Sun Oct 20 20:04:41 2013
@@ -21,23 +21,42 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
- *Finds instances of country mentions in a String, typically a document text.
+ * Finds instances of country mentions in a String, typically a document text.
* Used to boost or degrade scoring of linked geo entities
-
+ *
*/
public class CountryContext {
private Connection con;
private List<CountryContextEntry> countrydata;
+ private Map<String, Set<String>> nameCodesMap = new HashMap<String,
Set<String>>();
+
+ public Map<String, Set<String>> getNameCodesMap() {
+ return nameCodesMap;
+ }
+
+ public void setNameCodesMap(Map<String, Set<String>> nameCodesMap) {
+ this.nameCodesMap = nameCodesMap;
+ }
public CountryContext() {
}
+ /**
+ * use regexFind
+ */
+ @Deprecated
public List<CountryContextHit> find(String docText, EntityLinkerProperties
properties) {
List<CountryContextHit> hits = new ArrayList<CountryContextHit>();
try {
@@ -51,7 +70,7 @@ public class CountryContext {
if (docText.contains(entry.getFull_name_nd_ro())) {
System.out.println("\tFound Country indicator: " +
entry.getFull_name_nd_ro());
- CountryContextHit hit = new CountryContextHit(entry.getCc1(),
docText.indexOf(entry.getFull_name_nd_ro()),
docText.indexOf(entry.getFull_name_nd_ro()+
entry.getFull_name_nd_ro().length()));
+ CountryContextHit hit = new CountryContextHit(entry.getCc1(),
docText.indexOf(entry.getFull_name_nd_ro()),
docText.indexOf(entry.getFull_name_nd_ro() +
entry.getFull_name_nd_ro().length()));
hits.add(hit);
}
}
@@ -60,6 +79,81 @@ public class CountryContext {
Logger.getLogger(CountryContext.class.getName()).log(Level.SEVERE,
null, ex);
}
return hits;
+
+ }
+/**
+ * Finds mentions of countries based on a list from MySQL stored procedure
called getCountryList. This method finds country mentions in documents,
+ * which is an essential element of the scoring that is done for geo
linkedspans. Lazily loads the list from the database.
+ * @param docText the full text of the document
+ * @param properties EntityLinkerProperties for getting database connection
+ * @return
+ */
+ public Map<String, Set<Integer>> regexfind(String docText,
EntityLinkerProperties properties) {
+ Map<String, Set<Integer>> hits = new HashMap<String, Set<Integer>>();
+ try {
+ if (con == null) {
+ con = getMySqlConnection(properties);
+ }
+ if (countrydata == null) {
+ countrydata = getCountryData(properties);
+ }
+ for (CountryContextEntry entry : countrydata) {
+ Pattern regex = Pattern.compile(entry.getFull_name_nd_ro(),
Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ Matcher rs = regex.matcher(docText);
+ String code = entry.getCc1().toLowerCase();
+ while (rs.find()) {
+ Integer start = rs.start();
+ String hit = rs.group().toLowerCase();
+ if (hits.containsKey(code)) {
+ hits.get(code).add(start);
+ } else {
+ Set<Integer> newset = new HashSet<Integer>();
+ newset.add(start);
+ hits.put(code, newset);
+ }
+ if (!hit.equals("")) {
+ if (this.nameCodesMap.containsKey(hit)) {
+ nameCodesMap.get(hit).add(code);
+ } else {
+ HashSet<String> newset = new HashSet<String>();
+ newset.add(code);
+ nameCodesMap.put(hit, newset);
+ }
+ }
+ }
+
+ }
+
+ } catch (Exception ex) {
+ Logger.getLogger(CountryContext.class.getName()).log(Level.SEVERE, null,
ex);
+ }
+
+ //System.out.println(hits);
+ return hits;
+ }
+/**
+ * returns a unique list of country codes
+ * @param hits the hits discovered
+ * @return
+ */
+ public static Set<String> getCountryCodes(List<CountryContextHit> hits) {
+ Set<String> ccs = new HashSet<String>();
+ for (CountryContextHit hit : hits) {
+ ccs.add(hit.getCountryCode().toLowerCase());
+ }
+ return ccs;
+ }
+
+ public static String getCountryCodeCSV(Set<String> hits) {
+ String csv = "";
+ if (hits.isEmpty()) {
+ return csv;
+ }
+
+ for (String code : hits) {
+ csv += "," + code;
+ }
+ return csv.substring(1);
}
private Connection getMySqlConnection(EntityLinkerProperties properties) throws Exception {
@@ -73,7 +167,12 @@ public class CountryContext {
Connection conn = DriverManager.getConnection(url, username, password);
return conn;
}
-
+/**
+ * reads the list from the database by calling a stored procedure
getCountryList
+ * @param properties
+ * @return
+ * @throws SQLException
+ */
private List<CountryContextEntry> getCountryData(EntityLinkerProperties
properties) throws SQLException {
List<CountryContextEntry> entries = new ArrayList<CountryContextEntry>();
try {
Added:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java?rev=1533959&view=auto
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java
(added)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java
Sun Oct 20 20:04:41 2013
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.entitylinker;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import opennlp.tools.ngram.NGramGenerator;
+
+/**
+ *
+ *Generates scores for string comparisons.
+ */
+public class FuzzyStringMatcher {
+/**
+ * Generates a score based on an overlap of nGrams between two strings using
the DiceCoefficient technique.
+ *
+ * @param s1 first string
+ * @param s2 second string
+ * @param nGrams number of chars in each gram
+ * @return
+ */
+ public static double getDiceCoefficient(String s1, String s2, int nGrams) {
+ if (s1.equals("") || s1.equals("")) {
+ return 0d;
+ }
+ List<String> s1Grams = NGramGenerator.generate(s1.toCharArray(), nGrams,
"");
+ List<String> s2Grams = NGramGenerator.generate(s2.toCharArray(), nGrams,
"");
+
+ Set<String> overlap = new HashSet<String>(s1Grams);
+ overlap.retainAll(s2Grams);
+ double totcombigrams = overlap.size();
+
+ return (2 * totcombigrams) / (s1Grams.size() + s2Grams.size());
+ }
+}
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java?rev=1533959&r1=1533958&r2=1533959&view=diff
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java
(original)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java
Sun Oct 20 20:04:41 2013
@@ -19,6 +19,8 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import opennlp.tools.entitylinker.domain.BaseLink;
@@ -26,17 +28,24 @@ import opennlp.tools.entitylinker.domain
import opennlp.tools.util.Span;
/**
- * Links location entities to gazatteers.
+ * Links location entities to gazatteers. Currently supports gazateers in a
+ * MySql database (NGA and USGS)
*
*
*/
public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
+ GeoEntityScorer scorer = new GeoEntityScorer();
private MySQLGeoNamesGazLinkable geoNamesGaz;// = new
MySQLGeoNamesGazLinkable();
private MySQLUSGSGazLinkable usgsGaz;//= new MySQLUSGSGazLinkable();
private CountryContext countryContext;
- private List<CountryContextHit> hits;
- private EntityLinkerProperties props;
+ private Map<String, Set<Integer>> countryMentions;
+ private EntityLinkerProperties linkerProperties;
+ /**
+ * Flag for deciding whether to search gaz only for toponyms within countries
+ * that are mentioned in the document
+ */
+ private Boolean filterCountryContext=true;
public GeoEntityLinker() {
if (geoNamesGaz == null || usgsGaz == null) {
@@ -50,25 +59,44 @@ public class GeoEntityLinker implements
public List<LinkedSpan> find(String text, Span[] sentences, String[]
tokens, Span[] names) {
ArrayList<LinkedSpan> spans = new ArrayList<LinkedSpan>();
try {
- if (props == null) {
- props = new EntityLinkerProperties(new
File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
+ if (linkerProperties == null) {
+ linkerProperties = new EntityLinkerProperties(new
File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
}
- if (hits == null) {
- System.out.println("getting country context");
- hits = countryContext.find(text, props);
- }
-
+
+ countryMentions = countryContext.regexfind(text, linkerProperties);
+
+ //prioritize query
+ filterCountryContext =
Boolean.valueOf(linkerProperties.getProperty("geoentitylinker.filter_by_country_context",
"true"));
String[] matches = Span.spansToStrings(names, tokens);
for (int i = 0; i < matches.length; i++) {
- System.out.println("processing match " + i + " of " + matches.length);
- ArrayList<BaseLink> geoNamesEntries = geoNamesGaz.find(matches[i],
names[i], hits, props);
- ArrayList<BaseLink> usgsEntries = usgsGaz.find(matches[i], names[i],
hits, props);
- LinkedSpan<BaseLink> geoSpans = new
LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
- geoSpans.getLinkedEntries().addAll(usgsEntries);
- geoSpans.setSearchTerm(matches[i]);
- spans.add(geoSpans);
+
+//nga gazateer is for other than US placenames, don't use it unless US is a
mention in the document
+ ArrayList<BaseLink> geoNamesEntries = new ArrayList<BaseLink>();
+ if (!(countryMentions.keySet().contains("us") &&
countryMentions.keySet().size() == 1) || countryMentions.keySet().size() > 1) {
+ geoNamesEntries = geoNamesGaz.find(matches[i], names[i],
countryMentions, linkerProperties);
+ }
+ ArrayList<BaseLink> usgsEntries = new ArrayList<BaseLink>();
+ if (countryMentions.keySet().contains("us")) {
+ usgsEntries = usgsGaz.find(matches[i], names[i], countryMentions,
linkerProperties);
+ }
+ LinkedSpan<BaseLink> geoSpan = new
LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
+
+ if (!usgsEntries.isEmpty()) {
+ geoSpan.getLinkedEntries().addAll(usgsEntries);
+ geoSpan.setSearchTerm(matches[i]);
+ }
+
+ if (!geoSpan.getLinkedEntries().isEmpty()) {
+ geoSpan.setSearchTerm(matches[i]);
+ spans.add(geoSpan);
+ }
+
}
- return spans;
+ //score the spans
+
+ scorer.score(spans, countryMentions, countryContext.getNameCodesMap(),
text, sentences, 1000);
+
+ // return spans;
} catch (IOException ex) {
Logger.getLogger(GeoEntityLinker.class.getName()).log(Level.SEVERE,
null, ex);
}
@@ -78,12 +106,14 @@ public class GeoEntityLinker implements
public List<LinkedSpan> find(String text, Span[] sentences, Span[] tokens,
Span[] names) {
ArrayList<LinkedSpan> spans = new ArrayList<LinkedSpan>();
try {
-
-
- if (props == null) {
- props = new EntityLinkerProperties(new
File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
+ if (linkerProperties == null) {
+ linkerProperties = new EntityLinkerProperties(new
File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
}
- List<CountryContextHit> hits = countryContext.find(text, props);
+
+ // System.out.println("getting country context");
+ //hits = countryContext.find(text, linkerProperties);
+ countryMentions = countryContext.regexfind(text, linkerProperties);
+
//get the sentence text....must assume some index
Span s = sentences[0];
String sentence = text.substring(s.getStart(), s.getEnd());
@@ -92,17 +122,32 @@ public class GeoEntityLinker implements
//get the names based on the tokens
String[] matches = Span.spansToStrings(names, stringtokens);
for (int i = 0; i < matches.length; i++) {
- ArrayList<BaseLink> geoNamesEntries = geoNamesGaz.find(matches[i],
names[i], hits, props);
- ArrayList<BaseLink> usgsEntries = usgsGaz.find(matches[i], names[i],
hits, props);
- LinkedSpan<BaseLink> geoSpans = new
LinkedSpan<BaseLink>(geoNamesEntries, names[i], 0);
- geoSpans.getLinkedEntries().addAll(usgsEntries);
- geoSpans.setSearchTerm(matches[i]);
- spans.add(geoSpans);
+ //nga gazateer is for other than US placenames, don't use it unless US
is a mention in the document
+ ArrayList<BaseLink> geoNamesEntries = new ArrayList<BaseLink>();
+ if (!(countryMentions.keySet().contains("us") &&
countryMentions.keySet().size() == 1) || countryMentions.keySet().size() > 1) {
+ geoNamesEntries = geoNamesGaz.find(matches[i], names[i],
countryMentions, linkerProperties);
+ }
+ ArrayList<BaseLink> usgsEntries = new ArrayList<BaseLink>();
+ if (countryMentions.keySet().contains("us")) {
+ usgsEntries = usgsGaz.find(matches[i], names[i], countryMentions,
linkerProperties);
+ }
+ LinkedSpan<BaseLink> geoSpan = new
LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
+
+ if (!usgsEntries.isEmpty()) {
+ geoSpan.getLinkedEntries().addAll(usgsEntries);
+ geoSpan.setSearchTerm(matches[i]);
+ }
+
+ if (!geoSpan.getLinkedEntries().isEmpty()) {
+ geoSpan.setSearchTerm(matches[i]);
+ spans.add(geoSpan);
+ }
}
- return spans;
+
} catch (IOException ex) {
Logger.getLogger(GeoEntityLinker.class.getName()).log(Level.SEVERE,
null, ex);
}
+ scorer.score(spans, countryMentions, countryContext.getNameCodesMap(),
text, sentences, 1000);
return spans;
}
@@ -110,10 +155,11 @@ public class GeoEntityLinker implements
ArrayList<LinkedSpan> spans = new ArrayList<LinkedSpan>();
try {
- if (props == null) {
- props = new EntityLinkerProperties(new
File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
+ if (linkerProperties == null) {
+ linkerProperties = new EntityLinkerProperties(new
File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
}
- List<CountryContextHit> hits = countryContext.find(text, props);
+
+ countryMentions = countryContext.regexfind(text, linkerProperties);
Span s = sentences[sentenceIndex];
String sentence = text.substring(s.getStart(), s.getEnd());
@@ -123,15 +169,29 @@ public class GeoEntityLinker implements
String[] matches = Span.spansToStrings(names, stringtokens);
for (int i = 0; i < matches.length; i++) {
- ArrayList<BaseLink> geoNamesEntries = geoNamesGaz.find(matches[i],
names[i], hits, props);
- ArrayList<BaseLink> usgsEntries = usgsGaz.find(matches[i], names[i],
hits, props);
- LinkedSpan<BaseLink> geoSpans = new
LinkedSpan<BaseLink>(geoNamesEntries, names[i], 0);
- geoSpans.getLinkedEntries().addAll(usgsEntries);
- geoSpans.setSearchTerm(matches[i]);
- geoSpans.setSentenceid(sentenceIndex);
- spans.add(geoSpans);
+//nga gazateer is for other than US placenames, don't use it unless US is a
mention in the document
+ ArrayList<BaseLink> geoNamesEntries = new ArrayList<BaseLink>();
+ if (!(countryMentions.keySet().contains("us") &&
countryMentions.keySet().size() == 1) || countryMentions.keySet().size() > 1) {
+ geoNamesEntries = geoNamesGaz.find(matches[i], names[i],
countryMentions, linkerProperties);
+ }
+ ArrayList<BaseLink> usgsEntries = new ArrayList<BaseLink>();
+ if (countryMentions.keySet().contains("us")) {
+ usgsEntries = usgsGaz.find(matches[i], names[i], countryMentions,
linkerProperties);
+ }
+ LinkedSpan<BaseLink> geoSpan = new
LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
+
+ if (!usgsEntries.isEmpty()) {
+ geoSpan.getLinkedEntries().addAll(usgsEntries);
+ geoSpan.setSearchTerm(matches[i]);
+ }
+
+ if (!geoSpan.getLinkedEntries().isEmpty()) {
+ geoSpan.setSearchTerm(matches[i]);
+ geoSpan.setSentenceid(sentenceIndex);
+ spans.add(geoSpan);
+ }
}
-
+ scorer.score(spans, countryMentions, countryContext.getNameCodesMap(),
text, sentences, 2000);
} catch (IOException ex) {
Logger.getLogger(GeoEntityLinker.class.getName()).log(Level.SEVERE,
null, ex);
}
@@ -139,6 +199,6 @@ public class GeoEntityLinker implements
}
public void setEntityLinkerProperties(EntityLinkerProperties properties) {
- this.props = properties;
+ this.linkerProperties = properties;
}
}
Added:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java?rev=1533959&view=auto
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java
(added)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java
Sun Oct 20 20:04:41 2013
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.entitylinker;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import opennlp.tools.entitylinker.domain.BaseLink;
+import opennlp.tools.entitylinker.domain.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Scores toponyms based on country context as well as fuzzy string matching
+ */
+public class GeoEntityScorer {
+
+ private Map<String, Set<String>> nameCodesMap;
+ String dominantCode = "";
+
+ /**
+ * Assigns a score to each BaseLink in each linkedSpan's set of N best
+ * matches. Currently the scoring indicates the probability that the toponym
+ * is correct based on the country context in the document and fuzzy string
matching
+ *
+ * @param linkedData the linked spans, holds the Namefinder results, and
+ * the list of BaseLink for each
+ * @param countryHits all the country mentions in the document
+ * @param nameCodesMap maps a country indicator name to a country code.
Used
+ * to determine if the namefinder found the same exact
+ * toponym the country context did. If so the score is
+ * boosted due to the high probability that the
+ * NameFinder actually "rediscovered" a country
+ * @param docText the full text of the document...not used in this
+ * default implementation
+ * @param sentences the sentences that correspond to the doc text.
+ * @param maxAllowedDist a constant that is used to determine which country
+ * mentions, based on proximity within the text, should
+ * be used to score the Named Entity.
+ * @return
+ */
+ public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>>
countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer
maxAllowedDist) {
+ this.nameCodesMap = nameCodesMap;
+ setDominantCode(countryHits);
+ for (LinkedSpan<BaseLink> linkedspan : linkedData) {
+
+ for (BaseLink link : linkedspan.getLinkedEntries()) {
+ Double dice = FuzzyStringMatcher.getDiceCoefficient(linkedspan.getSearchTerm().toLowerCase().replace("
", ""), link.getItemName().toLowerCase().replace(" ", ""), 2);
+ /**
+ * Since MySQL is using "boolean mode" this score will always be very
+ * high. To allow more recall, change mysql to "natural language mode",
+ * and this score will become more significant
+ */
+ link.setFuzzyStringMatchingScore(dice);
+
+ }
+ linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan,
maxAllowedDist);
+ }
+ return linkedData;
+ }
+/**
+ * sets class level variable to a code based on the number of mentions
+ * @param countryHits
+ */
+ private void setDominantCode(Map<String, Set<Integer>> countryHits) {
+ int hits = -1;
+ for (String code : countryHits.keySet()) {
+ if (countryHits.get(code).size() > hits) {
+ hits = countryHits.get(code).size();
+ dominantCode = code;
+ }
+ }
+ }
+
+ /**
+ * Generates distances from each country mention to the span's location in
the
+ * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
+ * are resolved to the correct country and coordinate.
+ *
+ * @param sentences
+ * @param countryHits
+ * @param span
+ * @return
+ */
+ private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String,
Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
+ //get the index of the actual span, begining of sentence
+ //should generate tokens from sentence and create a char offset...
+ //could have large sentences due to poor sentence detection or wonky doc
text
+ int sentenceIdx = span.getSentenceid();
+ int sentIndexInDoc = sentences[sentenceIdx].getStart();
+ /**
+ * create a map of all the span's proximal country mentions in the document
+ * Map< countrycode, set of <distances from this NamedEntity>>
+ */
+ Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<String,
Set<Integer>>();
+ //map = Map<countrycode, Set <of distances this span is from all the mentions of
the code>>
+ for (String cCode : countryHits.keySet()) {
+//iterate over all the regex start values and calculate an offset
+ for (Integer cHit : countryHits.get(cCode)) {
+ Integer absDist = Math.abs(sentIndexInDoc - cHit);
+ //only include near mentions based on a heuristic
+ //TODO make this a property
+ // if (absDist < maxAllowedDistance) {
+ if (distancesFromCodeMap.containsKey(cCode)) {
+ distancesFromCodeMap.get(cCode).add(absDist);
+ } else {
+ HashSet<Integer> newset = new HashSet<Integer>();
+ newset.add(absDist);
+ distancesFromCodeMap.put(cCode, newset);
+ }
+ }
+
+ //}
+ }
+ //we now know how far this named entity is from every country mention in
the document
+
+ /**
+ * the gaz matches that have a country code that have mentions in the doc
+ * that are closest to the Named Entity should return the best score
Analyze
+ * map generates a likelihood score that the toponym from the gaz is
+ * referring to one of the countries Map<countrycode, prob that this span
is
+ * referring to the toponym form this code key>
+ */
+ Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences,
span);
+ for (BaseLink link : span.getLinkedEntries()) {
+ //getItemParentId is the country code
+ String spanCountryCode = link.getItemParentID();
+ if (scoreMap.containsKey(spanCountryCode)) {
+ link.setScore(scoreMap.get(spanCountryCode));
+ ///does the name extracted match a country name?
+ if (nameCodesMap.containsKey(link.getItemName().toLowerCase())) {
+ //if so, is it the correct country code for that name
+ if
(nameCodesMap.get(link.getItemName().toLowerCase()).contains(link.getItemParentID()))
{
+ //boost the score becuase it is likely that this is the location
in the text, so add 50% to the score or set to 1
+ //TODO: make this multiplier configurable
+ //TODO: improve this with a geographic/geometry based clustering
(linear binning to be more precise) of points returned from the gaz
+ Double score = (link.getScore() + .75) > 1.0 ? 1d :
(link.getScore() + .75);
+ //boost the score if the hit is from the dominant country context
+
+ if(link.getItemParentID().equals(dominantCode)){
+ score = (score + .25) > 1.0 ? 1d : (score + .25);
+ }
+ link.setScore(score);
+
+ }
+
+ }
+ }
+ }
+ return span;
+ }
+
+ /**
+ * takes a map of distances from the NE to each country mention and generates
+ * a map of scores for each country code. The map is then correlated to teh
+ * correlated to the code of the BaseLink parentid for retrieval. Then the
+ * score is added to the overall.
+ *
+ * @param distanceMap
+ * @param sentences
+ * @param span
+ * @return
+ */
+ private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap,
Span[] sentences, LinkedSpan<BaseLink> span) {
+
+ Map<String, Double> scoreMap = new HashMap<String, Double>();
+ TreeSet<Integer> all = new TreeSet<Integer>();
+ for (String key : distanceMap.keySet()) {
+ all.addAll(distanceMap.get(key));
+ }
+ //get min max for normalization, this could be more efficient
+ Integer min = all.first();
+ Integer max = all.last();
+ for (String key : distanceMap.keySet()) {
+
+ TreeSet<Double> normalizedDistances = new TreeSet<Double>();
+ for (Integer i : distanceMap.get(key)) {
+ Double norm = normalize(i, min, max);
+ //reverse the normed distance so low numbers (closer) are better
+ //this could be improved with a "decaying " function using an
imcreaseing negative exponent
+ Double reverse = Math.abs(norm - 1);
+ normalizedDistances.add(reverse);
+ }
+
+
+ List<Double> doubles = new ArrayList<Double>(normalizedDistances);
+ scoreMap.put(key, slidingDistanceAverage(doubles));
+ }
+ return scoreMap;
+ }
+
+ /**
+ * this method is an attempt to make closer clusters of mentions group
+ * together to smooth out the average, so one distant outlier does not kill
+ * the score for an obviously good hit. More elegant solution is possible
+ * using Math.pow, and making the score decay with distance by using an
+ * increasing negative exponent
+ *
+ * @param normDis the normalized and sorted set of distances as a list
+ * @return
+ */
+ private Double slidingDistanceAverage(List<Double> normDis) {
+ List<Double> windowOfAverages = new ArrayList<Double>();
+
+ if (normDis.size() < 3) {
+ windowOfAverages.addAll(normDis);
+ } else {
+
+ for (int i = 0; i < normDis.size() - 1; i++) {
+ double a = normDis.get(i);
+ double b = normDis.get(i + 1);
+ windowOfAverages.add((a + b) / 2);
+
+ }
+ }
+ double sum = 0d;
+ for (double d : windowOfAverages) {
+ sum += d;
+ }
+ double result = sum / windowOfAverages.size();
+ //TODO: ++ prob when large amounts of mentions for a code
+ //System.out.println("avg of window:" + result);
+ return result;
+ }
+
+ /**
+ * transposes a value within one range to a relative value in a different
+ * range. Used to normalize distances in this class.
+ *
+ * @param valueToNormalize the value to place within the new range
+ * @param minimum the min of the set to be transposed
+ * @param maximum the max of the set to be transposed
+ * @return
+ */
+ private Double normalize(int valueToNormalize, int minimum, int maximum) {
+ Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum -
minimum) + 0;
+ d = d == null ? 0d : d;
+ return d;
+ }
+}
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazEntry.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazEntry.java?rev=1533959&r1=1533958&r2=1533959&view=diff
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazEntry.java
(original)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazEntry.java
Sun Oct 20 20:04:41 2013
@@ -18,59 +18,31 @@ package opennlp.tools.entitylinker;
import opennlp.tools.entitylinker.domain.BaseLink;
/**
- *
+ *Stores an entry from the NGA Geonames gazateer
*/
public class MySQLGeoNamesGazEntry extends BaseLink
{
- ////actual fields returned
-//ufi,
-//latitude,
-//longitude,
-//cc1,
-//adm1,
-//dsg,
-//SHORT_FORM ,
-// SORT_NAME_RO ,
-// FULL_NAME_RO ,
-// FULL_NAME_ND_RO ,
-// SORT_NAME_RG ,
-// FULL_NAME_RG ,
-// FULL_NAME_ND_RG ,
-//match(`SHORT_FORM` ,`SORT_NAME_RO`,`FULL_NAME_RO`,`FULL_NAME_ND_RO`
,`SORT_NAME_RG` ,`FULL_NAME_RG` ,`FULL_NAME_ND_RG`)
-//against(pSearch in natural language mode) as rank
-
- ///////
-
- // private String RC;// VARCHAR(150) NULL DEFAULT NULL,
+
private String UFI;
- //private String UNI;
+
private Double LATITUDE; //DOUBLE NULL DEFAULT NULL,
private Double LONGITUDE;// DOUBLE NULL DEFAULT NULL,
- // private String DMS_LAT;// VARCHAR(150) NULL DEFAULT NULL,
- // private String DMS_LONG;// VARCHAR(150) NULL DEFAULT NULL,
- // private String MGRS;// VARCHAR(150) NULL DEFAULT NULL,
-// private String JOG;// VARCHAR(150) NULL DEFAULT NULL,
- // private String FC;// VARCHAR(150) NULL DEFAULT NULL,
+
private String DSG;// VARCHAR(150) NULL DEFAULT NULL,
- // private String PC;// VARCHAR(150) NULL DEFAULT NULL,
+
private String CC1;//` VARCHAR(150) NULL DEFAULT NULL,
private String ADM1;// VARCHAR(150) NULL DEFAULT NULL,
- // private String POP;// VARCHAR(150) NULL DEFAULT NULL,
- //private String ELEV;//VARCHAR(150) NULL DEFAULT NULL,
-// private String CC2;// VARCHAR(150) NULL DEFAULT NULL,
- // private String NT;//VARCHAR(150) NULL DEFAULT NULL,
- // private String LC;// VARCHAR(150) NULL DEFAULT NULL,
+
private String SHORT_FORM;// VARCHAR(500) NULL DEFAULT NULL,
- // private String GENERIC;// VARCHAR(150) NULL DEFAULT NULL,
+
private String SORT_NAME_RO;//VARCHAR(500) NULL DEFAULT NULL,
private String FULL_NAME_RO;// VARCHAR(500) NULL DEFAULT NULL,
private String FULL_NAME_ND_RO;// VARCHAR(500) NULL DEFAULT NULL,
private String SORT_NAME_RG;// VARCHAR(500) NULL DEFAULT NULL,
private String FULL_NAME_RG;// VARCHAR(500) NULL DEFAULT NULL,
private String FULL_NAME_ND_RG;// VARCHAR(500) NULL DEFAULT NULL,
-// private String NOTE;//VARCHAR(500) NULL DEFAULT NULL,
- // private String MODIFY_DATE;// VARCHAR(150) NULL DEFAULT NULL,
+
private Double rank;
public String getUFI()
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java?rev=1533959&r1=1533958&r2=1533959&view=diff
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java
(original)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java
Sun Oct 20 20:04:41 2013
@@ -1,17 +1,13 @@
package opennlp.tools.entitylinker;
-/**
- *
- * @author Owner
- */
+
import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -20,7 +16,7 @@ import opennlp.tools.util.Span;
/**
*
- *
+ *Links names to the NGA gazateer
*/
public final class MySQLGeoNamesGazLinkable {
@@ -30,7 +26,7 @@ public final class MySQLGeoNamesGazLinka
public MySQLGeoNamesGazLinkable() {
}
- public ArrayList<BaseLink> find(String locationText, Span span, List<CountryContextHit> countryHits, EntityLinkerProperties properties) {
+ public ArrayList<BaseLink> find(String locationText, Span span, Map<String,
Set<Integer>> countryHits, EntityLinkerProperties properties) {
ArrayList<BaseLink> returnlocs = new ArrayList<BaseLink>();
try {
@@ -40,13 +36,13 @@ public final class MySQLGeoNamesGazLinka
// pull from config to utilize country context filtering
filterCountryContext =
Boolean.valueOf(properties.getProperty("geoentitylinker.filter_by_country_context",
"false"));
- Set<String> countrycodes = getCountryCodes(countryHits);
+
String thresh = properties.getProperty("mysqlusgsgazscorethresh", "25");
int threshhold = -1;
if (!thresh.matches("[azAZ]")) {
threshhold = Integer.valueOf(thresh);
}
- returnlocs.addAll(this.searchGaz(locationText, threshhold, countrycodes,
properties));
+ returnlocs.addAll(this.searchGaz(locationText, threshhold,
countryHits.keySet(), properties));
} catch (Exception ex) {
@@ -56,7 +52,7 @@ public final class MySQLGeoNamesGazLinka
}
protected Connection getMySqlConnection(EntityLinkerProperties property) throws Exception {
- // EntityLinkerProperties property = new EntityLinkerProperties(new
File("c:\\temp\\opennlpmodels\\entitylinker.properties"));
+ // EntityLinkerProperties property = new EntityLinkerProperties(new
File("c:\\temp\\opennlpmodels\\entitylinker.properties"));
String driver = property.getProperty("mysql.driver",
"org.gjt.mm.mysql.Driver");
String url = property.getProperty("mysql.url",
"jdbc:mysql://localhost:3306/world");
String username = property.getProperty("mysql.username", "root");
@@ -73,16 +69,23 @@ public final class MySQLGeoNamesGazLinka
con = getMySqlConnection(properties);
}
CallableStatement cs;
- cs = con.prepareCall("CALL `search_geonames`(?, ?)");
+ cs = con.prepareCall("CALL `search_geonames`(?, ?, ?)");
cs.setString(1, this.format(searchString));
cs.setInt(2, matchthresh);
- ArrayList<MySQLGeoNamesGazEntry> retLocs = new
ArrayList<MySQLGeoNamesGazEntry>();
+ if (filterCountryContext) {
+ cs.setString(3,CountryContext.getCountryCodeCSV(countryCodes));
+ } else {
+ //database stored procedure handles empty string
+ cs.setString(3, "");
+ }
+
+ ArrayList<MySQLGeoNamesGazEntry> toponyms = new
ArrayList<MySQLGeoNamesGazEntry>();
ResultSet rs;
try {
rs = cs.executeQuery();
if (rs == null) {
- return retLocs;
+ return toponyms;
}
while (rs.next()) {
@@ -117,17 +120,13 @@ public final class MySQLGeoNamesGazLinka
s.setRank(rs.getDouble(14));
- if (filterCountryContext) {
- if (countryCodes.contains(s.getCC1().toLowerCase())) {
- // System.out.println(searchString +" GeoNames qualified on: " +
s.getCC1());
- s.setRank(s.getRank() + 1.0);
- } else {
- // System.out.println(s.getFULL_NAME_ND_RO() + ", with CC1 of "+ s.getCC1()+
", is not within countries discovered in the document. The Country list used to discover
countries can be modified in mysql procedure getCountryList()");
- continue;
- }
- }
-
- retLocs.add(s);
+ //set the base link data
+ s.setItemName(s.getFULL_NAME_ND_RO().toLowerCase().trim());
+ s.setItemID(s.getUFI());
+ s.setItemType(s.getDSG());
+ s.setItemParentID(s.getCC1().toLowerCase());
+
+ toponyms.add(s);
}
} catch (SQLException ex) {
@@ -138,16 +137,10 @@ public final class MySQLGeoNamesGazLinka
con.close();
}
- return retLocs;
+ return toponyms;
}
- private Set<String> getCountryCodes(List<CountryContextHit> hits) {
- Set<String> ccs = new HashSet<String>();
- for (CountryContextHit hit : hits) {
- ccs.add(hit.getCountryCode().toLowerCase());
- }
- return ccs;
- }
+
public String format(String entity) {
return "\"" + entity + "\"";
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazEntry.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazEntry.java?rev=1533959&r1=1533958&r2=1533959&view=diff
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazEntry.java
(original)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazEntry.java
Sun Oct 20 20:04:41 2013
@@ -18,7 +18,7 @@ package opennlp.tools.entitylinker;
import opennlp.tools.entitylinker.domain.BaseLink;
/**
- *
+ *Stores an entry from the USGS gazateer
*/
public class MySQLUSGSGazEntry extends BaseLink
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java?rev=1533959&r1=1533958&r2=1533959&view=diff
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java
(original)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java
Sun Oct 20 20:04:41 2013
@@ -23,6 +23,7 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -30,8 +31,7 @@ import opennlp.tools.entitylinker.domain
import opennlp.tools.util.Span;
/**
- *
- * @author opennlp
+ * Links names to the USGS gazateer
*/
public class MySQLUSGSGazLinkable {
@@ -41,12 +41,12 @@ public class MySQLUSGSGazLinkable {
public MySQLUSGSGazLinkable() {
}
- public ArrayList<BaseLink> find(String locationText, Span span, List<CountryContextHit> countryHits, EntityLinkerProperties properties) {
+ public ArrayList<BaseLink> find(String locationText, Span span, Map<String,
Set<Integer>> countryHits, EntityLinkerProperties properties) {
ArrayList<BaseLink> returnlocs = new ArrayList<BaseLink>();
try {
filterCountryContext =
Boolean.valueOf(properties.getProperty("geoentitylinker.filter_by_country_context",
"false"));
//the usgs gazateer only has us geonames, so only use it if the user
doesn't care about country isolation or the hits contain us
- if (getCountryCodes(countryHits).contains("us") ||
!filterCountryContext) {
+ if (countryHits.keySet().contains("us") || !filterCountryContext) {
if (con == null) {
con = getMySqlConnection(properties);
@@ -56,7 +56,7 @@ public class MySQLUSGSGazLinkable {
if (!thresh.matches("[azAZ]")) {
threshhold = Integer.valueOf(thresh);
}
- returnlocs.addAll(this.searchGaz(locationText, threshhold,
getCountryCodes(countryHits), properties));
+ returnlocs.addAll(this.searchGaz(locationText, threshhold,
countryHits.keySet(), properties));
}
} catch (Exception ex) {
Logger.getLogger(MySQLUSGSGazLinkable.class.getName()).log(Level.SEVERE, null,
ex);
@@ -84,13 +84,13 @@ public class MySQLUSGSGazLinkable {
cs = con.prepareCall("CALL `search_gaz`(?, ?)");
cs.setString(1, this.format(searchString));
cs.setInt(2, matchthresh);
- ArrayList<MySQLUSGSGazEntry> retUrls = new ArrayList<MySQLUSGSGazEntry>();
+ ArrayList<MySQLUSGSGazEntry> toponyms = new ArrayList<MySQLUSGSGazEntry>();
ResultSet rs;
try {
rs = cs.executeQuery();
if (rs == null) {
- return retUrls;
+ return toponyms;
}
while (rs.next()) {
@@ -99,21 +99,20 @@ public class MySQLUSGSGazLinkable {
s.setFeatureid(String.valueOf(rs.getLong(2)));
s.setFeaturename(rs.getString(3));
+
s.setFeatureclass(rs.getString(4));
s.setStatealpha(rs.getString(5));
s.setPrimarylatitudeDEC(rs.getDouble(6));
s.setPrimarylongitudeDEC(rs.getDouble(7));
s.setMapname(rs.getString(8));
- if (countryCodes.contains("us")) {
- s.setRank(s.getRank() + (s.getRank() * .5));
- // System.out.println(searchString +"USGS qualified on: " +
s.getFeaturename());
- } else {
- s.setRank(s.getRank() * .5);
- if(filterCountryContext){
- continue;
- }
- }
- retUrls.add(s);
+
+ //set the base link data
+ s.setItemName(s.getFeaturename().toLowerCase().trim());
+ s.setItemID(s.getFeatureid());
+ s.setItemType(s.getFeatureclass());
+ s.setItemParentID("us");
+
+ toponyms.add(s);
}
} catch (SQLException ex) {
@@ -124,7 +123,7 @@ public class MySQLUSGSGazLinkable {
con.close();
}
- return retUrls;
+ return toponyms;
}
private Set<String> getCountryCodes(List<CountryContextHit> hits) {
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/domain/BaseLink.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/domain/BaseLink.java?rev=1533959&r1=1533958&r2=1533959&view=diff
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/domain/BaseLink.java
(original)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/domain/BaseLink.java
Sun Oct 20 20:04:41 2013
@@ -13,29 +13,48 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package opennlp.tools.entitylinker.domain;
/**
* Stores a minimal tuple of information. Intended to be used with LinkedSpan
*
-
+ *
*/
public abstract class BaseLink {
+ private String itemParentID;
private String itemID;
private String itemName;
private String itemType;
+ private Double score;
+ private Double fuzzyStringMatchingScore;
public BaseLink() {
}
- public BaseLink(String itemID, String itemName, String itemType) {
+ public BaseLink(String itemParentID, String itemID, String itemName, String
itemType) {
+ this.itemParentID = itemParentID;
this.itemID = itemID;
this.itemName = itemName;
this.itemType = itemType;
}
+ public Double getScore() {
+ return score;
+ }
+
+ public void setScore(Double score) {
+ this.score = score;
+ }
+
+ public String getItemParentID() {
+ return itemParentID;
+ }
+
+ public void setItemParentID(String itemParentID) {
+ this.itemParentID = itemParentID;
+ }
+
/**
* returns the itemid
*
@@ -93,10 +112,16 @@ public abstract class BaseLink {
this.itemType = itemType;
}
-
-
@Override
public String toString() {
return "BaseLink{" + "itemID=" + itemID + ", itemName=" + itemName + ",
itemType=" + itemType + '}';
}
+
+ public Double getFuzzyStringMatchingScore() {
+ return fuzzyStringMatchingScore;
+ }
+
+ public void setFuzzyStringMatchingScore(Double fuzzyStringMatchingScore) {
+ this.fuzzyStringMatchingScore = fuzzyStringMatchingScore;
+ }
}
\ No newline at end of file
Added:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
URL:
http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java?rev=1533959&view=auto
==============================================================================
---
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
(added)
+++
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
Sun Oct 20 20:04:41 2013
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.ngram;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Generates an nGram, with optional separator, and returns the grams as a list
+ * of strings
+ */
+public class NGramGenerator {
+
+
+ /**
+ * Creates an ngram separated
+ * by the separator param value i.e. a,b,c,d with n = 3 and separator = "-"
+ * would return a-b-c,b-c-d
+ *
+ * @param input the input tokens the output ngrams will be derived from
+ * @param n the number of tokens as the sliding window
+ * @param separator each string in each gram will be separated by this value
if desired. Pass in empty string if no separator is desired
+ * @return
+ */
+ public static List<String> generate(List<String> input, int n, String
separator) {
+
+ List<String> outGrams = new ArrayList<String>();
+ for (int i = 0; i < input.size() - (n - 2); i++) {
+ String gram = "";
+ if ((i + n) <= input.size()) {
+ for (int x = i; x < (n + i); x++) {
+ gram += input.get(x) + separator;
+ }
+ gram = gram.substring(0, gram.lastIndexOf(separator));
+ outGrams.add(gram);
+ }
+ }
+ return outGrams;
+ }
+/**
+ *Generates an nGram based on a char[] input
+ * @param input the array of chars to convert to nGram
+ * @param n The number of grams (chars) that each output gram will consist of
+ * @param separator each char in each gram will be separated by this value if
desired. Pass in empty string if no separator is desired
+ * @return
+ */
+ public static List<String> generate(char[] input, int n, String separator) {
+
+ List<String> outGrams = new ArrayList<String>();
+ for (int i = 0; i < input.length - (n - 2); i++) {
+ String gram = "";
+ if ((i + n) <= input.length) {
+ for (int x = i; x < (n + i); x++) {
+ gram += input[x] + separator;
+ }
+ gram = gram.substring(0, gram.lastIndexOf(separator));
+ outGrams.add(gram);
+ }
+ }
+ return outGrams;
+ }
+}
