Author: rwesten
Date: Sat Nov 3 08:17:44 2012
New Revision: 1405306
URL: http://svn.apache.org/viewvc?rev=1405306&view=rev
Log:
STANBOL-792: implementation of the Custom NER model extraction Engine
Added:
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/CustomNERModelEnhancementEngine.java
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/LICENSE
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/README.md
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/bionlp2004-DNA-en.bin
(with props)
Modified:
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/resources/OSGI-INF/metatype/metatype.properties
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java
Added:
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/CustomNERModelEnhancementEngine.java
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/CustomNERModelEnhancementEngine.java?rev=1405306&view=auto
==============================================================================
---
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/CustomNERModelEnhancementEngine.java
(added)
+++
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/CustomNERModelEnhancementEngine.java
Sat Nov 3 08:17:44 2012
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.opennlp.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import opennlp.tools.namefind.TokenNameFinderModel;
+
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.ConfigurationPolicy;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.ReferencePolicy;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.opennlp.OpenNLP;
+import
org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileListener;
+import
org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileTracker;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
+import org.osgi.framework.Constants;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Apache Stanbol Enhancer Named Entity Recognition enhancement engine based
on opennlp's Maximum Entropy
+ * models. In contrast to the {@link NamedEntityExtractionEnhancementEngine}
this
+ * engine is intended to be used for custom build models.
+ */
+@Component(
+ metatype = true,
+ immediate = true,
+ inherit = true,
+ configurationFactory = true,
+ policy = ConfigurationPolicy.REQUIRE,
+ specVersion = "1.1",
+ label = "%stanbol.CustomNERModelEnhancementEngine.name",
+ description = "%stanbol.CustomNERModelEnhancementEngine.description")
+@Service
[email protected](value={
+ @Property(name=EnhancementEngine.PROPERTY_NAME,value="changeme"),
+ @Property(name=CustomNERModelEnhancementEngine.NAME_FINDER_MODELS,
cardinality=Integer.MAX_VALUE,
+ value={"openNlp-namefinder-model-name.bin"}),
+ @Property(name=CustomNERModelEnhancementEngine.NAMED_ENTITY_TYPE_MAPPINGS,
cardinality=Integer.MAX_VALUE,
+ value={"person > http://dbpedia.org/ontology/Person";,
+ "organization > http://dbpedia.org/ontology/Organisation";,
+ "location > http://dbpedia.org/ontology/Place"}),
+ //set the ranking of the default config to a negative value
(ConfigurationPolicy.OPTIONAL)
+ @Property(name=Constants.SERVICE_RANKING,intValue=-100)
+})
+@Reference(name="openNLP",referenceInterface=OpenNLP.class,
+ cardinality=ReferenceCardinality.MANDATORY_UNARY,
+ policy=ReferencePolicy.STATIC)
+public class CustomNERModelEnhancementEngine
+ extends NEREngineCore
+ implements EnhancementEngine, ServiceProperties {
+
+ protected final Logger log =
LoggerFactory.getLogger(CustomNERModelEnhancementEngine.class);
+
+ /**
+ * Do hold the named entity type to dc:type value mappings as used for
+ * created fise:TextAnnotations. If a named entity type is not mapped than
+ * created fise:TextAnnotations will not have any dc:type values
+ */
+ public static final String NAMED_ENTITY_TYPE_MAPPINGS =
"stanbol.engines.opennlp-ner.typeMappings";
+ /**
+ * Allows to define the list of custom NER models
+ */
+ public static final String NAME_FINDER_MODELS =
"stanbol.engines.opennlp-ner.nameFinderModels";
+
+ /**
+ * The default value for the Execution of this Engine. Currently set to
+ * {@link ServiceProperties#ORDERING_CONTENT_EXTRACTION}
+ */
+ public static final Integer defaultOrder = ORDERING_CONTENT_EXTRACTION;
+ /**
+ * Bind method of {@link NEREngineCore#openNLP}
+ * @param openNlp
+ */
+ protected void bindOpenNLP(OpenNLP openNlp){
+ this.openNLP = openNlp;
+ }
+ /**
+ * Unbind method of {@link NEREngineCore#openNLP}
+ * @param openNLP
+ */
+ protected void unbindOpenNLP(OpenNLP openNLP){
+ this.openNLP = null;
+ }
+
+ @Reference
+ private DataFileTracker dataFileTracker;
+
+ private DataFileListener modelFileListener;
+
+ protected void activate(ComponentContext ctx) throws IOException,
ConfigurationException {
+ super.activate(ctx);
+ config = new NEREngineConfig();
+ config.getDefaultModelTypes().clear(); //this engine does not use
default models
+ Object value = ctx.getProperties().get(NAMED_ENTITY_TYPE_MAPPINGS);
+ if(value instanceof String[]){ //support array
+ value = Arrays.asList((String[])value);
+ } else if(value instanceof String) { //single value
+ value = Collections.singleton(value);
+ }
+ if(value instanceof Collection<?>){ //and collection
+ log.info("Init Named Entity Type Mappings");
+ configs :
+ for(Object o : (Iterable<?>)value){
+ if(o != null){
+ StringBuilder usage = new StringBuilder("useage: ");
+ usage.append("'{namedEntityType} > {dc-type-uri}'");
+ String[] config = o.toString().split(">");
+ String namedEntityType = config[0].trim();
+ if(namedEntityType.isEmpty()){
+ log.warn("Invalid Type Mapping Config '{}': Missing
namedEntityType ({}) -> ignore this config",
+ o,usage);
+ continue configs;
+ }
+ if(config.length < 2 || config[1].isEmpty()){
+ log.warn("Invalid Type Mapping Config '{}': Missing
dc:type URI '{}' ({}) -> ignore this config",
+ o,usage);
+ continue configs;
+ }
+ String dcTypeUri = config[1].trim();
+ if(config.length > 2){
+ log.warn("Configuration after 2nd '>' gets ignored.
Will use mapping '{} > {}' from config {}",
+ new Object[]{namedEntityType,dcTypeUri,o});
+ }
+ //TODO support short names (ns:localName)
+ try { //validate
+ new URI(dcTypeUri);
+ } catch (URISyntaxException e) {
+ log.warn("Invalid URI '{}' in Type Mapping Config '{}'
-> ignore this config",
+ dcTypeUri,o);
+ continue configs;
+ }
+ this.config.setMappedType(namedEntityType,new
UriRef(dcTypeUri));
+ log.info(" add mapping {} >
{}",namedEntityType,dcTypeUri);
+ }
+ }
+ } else {
+ log.debug("No Type mappings configured");
+ }
+ value = ctx.getProperties().get(NAME_FINDER_MODELS);
+ Set<String> nameFinderModelNames = new HashSet<String>();
+ if(value instanceof String[]){
+ nameFinderModelNames.addAll(Arrays.asList((String[]) value));
+ nameFinderModelNames.remove(null); //remove null
+ nameFinderModelNames.remove(""); //remove empty
+ } else if (value instanceof Collection<?>){
+ for(Object o : ((Collection<?>)value)){
+ if(o != null){
+ nameFinderModelNames.add(o.toString());
+ }
+ }
+ nameFinderModelNames.remove(""); //remove empty
+ } else if(value != null && !value.toString().isEmpty()){
+ //if a single String is parsed we support ',' as seperator
+ String[] languageArray = value.toString().split(",");
+ nameFinderModelNames.addAll(Arrays.asList(languageArray));
+ nameFinderModelNames.remove(null); //remove null
+ nameFinderModelNames.remove(""); //remove empty
+ } else {//no configuration
+ throw new ConfigurationException(NAME_FINDER_MODELS,
"Configurations for the "
+ + getClass().getSimpleName() +" MUST HAVE at least a
single custom "
+ + "OpenNLP NameFinder model configured! Supported are
comma separated "
+ + "Strings, Arrays and Collections. Values are the file
names of the "
+ + "Modles. Models are Loaded via the Apache Stanbol
DataFileProvider "
+ + "Infrastructure (usually user wants to copy modles in
the 'datafile' "
+ + "directory under the {stanbol.home} directory -
{working.dir}/stanbol"
+ + "/datafiles).");
+ }
+ //register the configured models with the DataFileTracker
+ modelFileListener = new NamedModelFileListener();
+ Map<String,String> modelProperties = new HashMap<String,String>();
+ modelProperties.put("Description",
+ String.format("Statistical NameFinder (NER) model for OpenNLP as
configured "
+ +"for the %s (name:
%s)",getClass().getSimpleName(),getName()));
+ modelProperties.put("Model Type",
TokenNameFinderModel.class.getSimpleName());
+ for(String modelName : nameFinderModelNames){
+ dataFileTracker.add(modelFileListener, modelName, modelProperties);
+ }
+ }
+
+ protected void deactivate(ComponentContext ctx) {
+ dataFileTracker.removeAll(modelFileListener); //remove all tracked
files
+ config = null;
+ super.deactivate(ctx);
+ }
+
+ @Override
+ public Map<String,Object> getServiceProperties() {
+ return
Collections.unmodifiableMap(Collections.singletonMap(ENHANCEMENT_ENGINE_ORDERING,
+ (Object) defaultOrder));
+ }
+
+ private class NamedModelFileListener implements DataFileListener {
+
+ private Map<String,String> registeredModels =
Collections.synchronizedMap(
+ new HashMap<String,String>());
+
+ @Override
+ public boolean available(String resourceName, InputStream is) {
+ TokenNameFinderModel model;
+ try {
+ model = openNLP.getModel(TokenNameFinderModel.class,
resourceName, null);
+ //register the new model to the configuration
+ String modelLang = model.getLanguage().toLowerCase();
+ log.info("register custom NameFinderModel from resource: {}
for language: {} to {} (name:{})",
+ new
Object[]{resourceName,model.getLanguage(),getClass().getSimpleName(),getName()});
+ String currentLang = registeredModels.remove(resourceName);
+ if(currentLang != null && !modelLang.equals(currentLang)){
+ config.removeCustomNameFinderModel(currentLang,
resourceName);
+ }
+ config.addCustomNameFinderModel(modelLang, resourceName);
+ registeredModels.put(resourceName, modelLang);
+ } catch (IOException e) {
+ log.warn("Error while loading custom TokenNameFinderModel
model from resource " +
+ " resourceName. This model will NOT be
available for the "+
+ getClass().getSimpleName()+" (name:"+getName()+")",e);
+ } catch (RuntimeException e){
+ log.warn("Error while loading custom TokenNameFinderModel
model from resource " +
+ " resourceName. This model will NOT be available for
the "+
+ getClass().getSimpleName()+" (name:"+getName()+")",e);
+ }
+ return false; //keep tracking
+ }
+
+ @Override
+ public boolean unavailable(String resource) {
+ String language = registeredModels.remove(resource);
+ if(language != null){
+ log.info("unregister custom NameFinderModel for resource: {}
for language: {} to {} (name:{})" +
+ "because the resource is no longer available
via the DataFileProvider infrastructure.",
+ new
Object[]{resource,language,getClass().getSimpleName(),getName()});
+ config.removeCustomNameFinderModel(language, resource);
+ }
+ return false; //keep tracking
+ }
+
+ }
+
+}
\ No newline at end of file
Modified:
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java?rev=1405306&r1=1405305&r2=1405306&view=diff
==============================================================================
---
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
(original)
+++
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
Sat Nov 3 08:17:44 2012
@@ -8,6 +8,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.stanbol.commons.opennlp.OpenNLP;
@@ -49,19 +50,34 @@ public class NEREngineConfig {
private String defaultLanguage;
- public void addNerModel(String lang, String modelFileName){
+ public synchronized void addCustomNameFinderModel(String lang, String
modelFileName){
if(lang == null || lang.isEmpty()){
throw new IllegalArgumentException("The parsed lanaguage MUST NOT
be NULL or empty!");
}
if(modelFileName == null || modelFileName.isEmpty()){
throw new IllegalArgumentException("The parsed NER model name MUST
NOT be NULL or empty!");
}
- Collection<String> langModels = additionalNerModels.get(lang);
- if(langModels == null){
- langModels = new ArrayList<String>();
- additionalNerModels.put(lang, langModels);
+ Collection<String> currentModels = additionalNerModels.get(lang);
+ if(currentModels == null){
+ currentModels = new CopyOnWriteArrayList<String>();
+ additionalNerModels.put(lang, currentModels);
+ }
+ currentModels.add(modelFileName);
+ }
+
+ public synchronized void removeCustomNameFinderModel(String lang, String
modelFileName){
+ if(lang == null || lang.isEmpty()){
+ throw new IllegalArgumentException("The parsed lanaguage MUST NOT
be NULL or empty!");
+ }
+ if(modelFileName == null || modelFileName.isEmpty()){
+ throw new IllegalArgumentException("The parsed NER model name MUST
NOT be NULL or empty!");
+ }
+ Collection<String> currentModels = additionalNerModels.get(lang);
+ if(currentModels != null && //if models for that language are present
+ currentModels.remove(modelFileName) && // and the model was
actually remove
+ currentModels.isEmpty()){ //no other models present for this
language
+ additionalNerModels.remove(lang);
}
- langModels.add(modelFileName);
}
public Set<String> getProcessedLanguages() {
@@ -89,11 +105,7 @@ public class NEREngineConfig {
public String getDefaultLanguage() {
return defaultLanguage;
}
-
- public void setDefaultModelTypes(Set<String> defaultModelTypes) {
- this.defaultModelTypes = defaultModelTypes;
- }
-
+
public Set<String> getDefaultModelTypes() {
return defaultModelTypes;
}
@@ -107,4 +119,15 @@ public class NEREngineConfig {
public UriRef getMappedType(String namedEntityType){
return typeMappings.get(namedEntityType);
}
+ public void setMappedType(String namedEntityType,UriRef dcType){
+ if(namedEntityType != null && !namedEntityType.isEmpty()){
+ if(dcType == null){
+ typeMappings.remove(namedEntityType);
+ } else {
+ typeMappings.put(namedEntityType, dcType);
+ }
+ } else {
+ throw new IllegalArgumentException("The parsed NamedEntity type
MUST NOT be NULL nor empty!");
+ }
+ }
}
Modified:
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java?rev=1405306&r1=1405305&r2=1405306&view=diff
==============================================================================
---
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java
(original)
+++
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java
Sat Nov 3 08:17:44 2012
@@ -400,10 +400,10 @@ public abstract class NEREngineCore
String[] tokens = Span.spansToStrings(tokenSpans, sentence);
Span[] nameSpans = finder.find(tokens);
double[] probs = finder.probs();
- String[] names = Span.spansToStrings(nameSpans, tokens);
//int lastStartPosition = 0;
- for (int j = 0; j < names.length; j++) {
- String name = names[j];
+ for (int j = 0; j < nameSpans.length; j++) {
+ String name =
sentence.substring(tokenSpans[nameSpans[j].getStart()].getStart(),
+ tokenSpans[nameSpans[j].getEnd()-1].getEnd());
Double confidence = 1.0;
for (int k = nameSpans[j].getStart(); k <
nameSpans[j].getEnd(); k++) {
confidence *= probs[k];
Modified:
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/resources/OSGI-INF/metatype/metatype.properties
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1405306&r1=1405305&r2=1405306&view=diff
==============================================================================
---
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/resources/OSGI-INF/metatype/metatype.properties
(original)
+++
stanbol/trunk/enhancer/engines/opennlp-ner/src/main/resources/OSGI-INF/metatype/metatype.properties
Sat Nov 3 08:17:44 2012
@@ -43,4 +43,16 @@ An empty text indicates that all languag
(e.g. 'en,de' to enhance only English and German texts). \
NOTE: This porperty can be used to configure multiple instances of this engine
that \
process only documents with specific languages. This might e.g. be useful to \
-enable/disable NER for specific languages.
\ No newline at end of file
+enable/disable NER for specific languages.
+
+stanbol.CustomNERModelEnhancementEngine.name=Apache Stanbol Enhancer Engine:
Custom NER Model
+stanbol.CustomNERModelEnhancementEngine.description=NER Engine that allows to
configure custom \
+OpenNLP NameFinder modles for arbitrary Named Entity types
+stanbol.engines.opennlp-ner.typeMappings.name=Type Mappings
+stanbol.engines.opennlp-ner.typeMappings.description="{named-entity-type} >
{uri}" mappings \
+for the Named Entity Types recognized by any of the configured NER models to
the URIs used \
+as values for the dc:type property for the generated fise:TextAnnotations.
NOTE: that \
+TextAnnotations for unmapped Named Entity Types will have no dc:type
information.
+stanbol.engines.opennlp-ner.nameFinderModels.name=Name Finder Models
+stanbol.engines.opennlp-ner.nameFinderModels.description=The list of NER -
OpenNLP \
+TokenNameFinderModel's
\ No newline at end of file
Modified:
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java?rev=1405306&r1=1405305&r2=1405306&view=diff
==============================================================================
---
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java
(original)
+++
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java
Sat Nov 3 08:17:44 2012
@@ -32,6 +32,8 @@ import org.apache.clerezza.rdf.core.Reso
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
import org.apache.clerezza.rdf.core.impl.TripleImpl;
+import org.apache.stanbol.commons.opennlp.OpenNLP;
+import
org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider;
import
org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
@@ -39,6 +41,7 @@ import org.apache.stanbol.enhancer.servi
import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -57,14 +60,28 @@ public class TestNamedEntityExtractionEn
+ " without any name.\n"
+ "A new paragraph is being written. This paragraph has two
sentences.";
+
+ public static final String EHEALTH = "Whereas activation of the HIV-1
enhancer following T-cell "
+ + "stimulation is mediated largely through binding of the
transcription factor NF-kappa "
+ + "B to two adjacent kappa B sites in the HIV-1 long terminal
repeat, activation of the "
+ + "HIV-2 enhancer in monocytes and T cells is dependent on four
cis-acting elements : a "
+ + "single kappa B site, two purine-rich binding sites , PuB1 and
PuB2 , and a pets site .";
+
private static ContentItemFactory ciFactory =
InMemoryContentItemFactory.getInstance();
- static NEREngineCore nerEngine;
+ private NEREngineCore nerEngine;
public static final String FAKE_BUNDLE_SYMBOLIC_NAME =
"FAKE_BUNDLE_SYMBOLIC_NAME";
-
+ public static OpenNLP openNLP;
+
@BeforeClass
- public static void setUpServices() throws IOException {
- nerEngine = new NEREngineCore(new
ClasspathDataFileProvider(FAKE_BUNDLE_SYMBOLIC_NAME),
+ public static void initDataFileProvicer(){
+ DataFileProvider dataFileProvider = new
ClasspathDataFileProvider(FAKE_BUNDLE_SYMBOLIC_NAME);
+ openNLP = new OpenNLP(dataFileProvider);
+ }
+
+ @Before
+ public void setUpServices() throws IOException {
+ nerEngine = new NEREngineCore(openNLP,
new NEREngineConfig()){};
}
@@ -141,5 +158,26 @@ public class TestNamedEntityExtractionEn
int textAnnotationCount =
validateAllTextAnnotations(g,SINGLE_SENTENCE,expectedValues);
assertEquals(3, textAnnotationCount);
}
+ @Test
+ public void testCustomModel() throws EngineException, IOException {
+ ContentItem ci =
wrapAsContentItem("urn:test:content-item:single:sentence", EHEALTH,"en");
+ //this test does not use default models
+ nerEngine.config.getDefaultModelTypes().clear();
+ //but instead a custom model provided by the test data
+ nerEngine.config.addCustomNameFinderModel("en",
"bionlp2004-DNA-en.bin");
+ nerEngine.config.setMappedType("DNA", new
UriRef("http://www.bootstrep.eu/ontology/GRO#DNA";));
+ nerEngine.computeEnhancements(ci);
+ Map<UriRef,Resource> expectedValues = new HashMap<UriRef,Resource>();
+ expectedValues.put(Properties.ENHANCER_EXTRACTED_FROM, ci.getUri());
+ expectedValues.put(Properties.DC_CREATOR,
LiteralFactory.getInstance().createTypedLiteral(nerEngine.getClass().getName()));
+ //adding null as expected for confidence makes it a required property
+ expectedValues.put(Properties.ENHANCER_CONFIDENCE, null);
+ //and dc:type values MUST be the URI set as mapped type
+ expectedValues.put(Properties.DC_TYPE, new
UriRef("http://www.bootstrep.eu/ontology/GRO#DNA";));
+ MGraph g = ci.getMetadata();
+ int textAnnotationCount =
validateAllTextAnnotations(g,EHEALTH,expectedValues);
+ assertEquals(6, textAnnotationCount);
+ }
+
}
\ No newline at end of file
Added:
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/LICENSE
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/LICENSE?rev=1405306&view=auto
==============================================================================
---
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/LICENSE
(added)
+++
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/LICENSE
Sat Nov 3 08:17:44 2012
@@ -0,0 +1,90 @@
+GENIA Project License for Annotated Corpora
+
+1. Copyright of abstracts
+
+Any abstracts contained in this corpus are from PubMed(R), a database
+of the U.S. National Library of Medicine (NLM).
+
+NLM data are produced by a U.S. Government agency and include works of
+the United States Government that are not protected by U.S. copyright
+law but may be protected by non-US copyright law, as well as abstracts
+originating from publications that may be protected by U.S. copyright
+law.
+
+NLM assumes no responsibility or liability associated with use of
+copyrighted material, including transmitting, reproducing,
+redistributing, or making commercial use of the data. NLM does not
+provide legal advice regarding copyright, fair use, or other aspects
+of intellectual property rights. Persons contemplating any type of
+transmission or reproduction of copyrighted material such as abstracts
+are advised to consult legal counsel.
+
+2. Copyright of full texts
+
+Any full texts contained in this corpus are from the PMC Open Access
+Subset of PubMed Central (PMC), the U.S. National Institutes of Health
+(NIH) free digital archive of biomedical and life sciences journal
+literature.
+
+Articles in the PMC Open Access Subset are protected by copyright, but
+are made available under a Creative Commons or similar license that
+generally allows more liberal redistribution and reuse than a
+traditional copyrighted work. Please refer to the license of each
+article for specific license terms.
+
+3. Copyright of annotations
+
+The copyrights of annotations created in the GENIA Project of Tsujii
+Laboratory, University of Tokyo, belong in their entirety to the GENIA
+Project.
+
+4. Licence terms
+
+Use and distribution of abstracts drawn from PubMed is subject to the
+PubMed(R) license terms as stated in Clause 1.
+
+Use and distribution of full texts is subject to the license terms
+applying to each publication.
+
+Annotations created by the GENIA Project are licensed under the
+Creative Commons Attribution 3.0 Unported License. To view a copy of
+this license, visit http://creativecommons.org/licenses/by/3.0/ or
+send a letter to Creative Commons, 444 Castro Street, Suite 900,
+Mountain View, California, 94041, USA.
+
+Annotations created by the GENIA Project must be attributed as
+detailed in Clause 5.
+
+5. Attribution
+
+The GENIA Project was founded and led by prof. Jun'ichi Tsujii and
+the project and its annotation efforts have been coordinated in part
+by Nigel Collier, Yuka Tateisi, Sang-Zoo Lee, Tomoko Ohta, Jin-Dong
+Kim, and Sampo Pyysalo.
+
+For a complete list of the GENIA Project members and contributors,
+please refer to http://www.geniaproject.org.
+
+The GENIA Project has been supported by Grant-in-Aid for Scientific
+Research on Priority Area "Genome Information Science" (MEXT, Japan),
+Grant-in-Aid for Scientific Research on Priority Area "Systems
+Genomics" (MEXT, Japan), Core Research for Evolutional Science &
+Technology (CREST) "Information Mobility Project" (JST, Japan),
+Solution Oriented Research for Science and Technology (SORST) (JST,
+Japan), Genome Network Project (MEXT, Japan) and Grant-in-Aid for
+Specially Promoted Research (MEXT, Japan).
+
+Annotations covered by this license must be attributed as follows:
+
+ Corpus annotations (c) GENIA Project
+
+Distributions including annotations covered by this licence must
+include this license text and Attribution section.
+
+6. References
+
+- GENIA Project : http://www.geniaproject.org
+- PubMed : http://www.pubmed.gov/
+- NLM (United States National Library of Medicine) : http://www.nlm.nih.gov/
+- MEXT (Ministry of Education, Culture, Sports, Science and Technology) :
http://www.mext.go.jp/
+- JST (Japan Science and Technology Agency) : http://www.jst.go.jp
Added:
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/README.md
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/README.md?rev=1405306&view=auto
==============================================================================
---
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/README.md
(added)
+++
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/README.md
Sat Nov 3 08:17:44 2012
@@ -0,0 +1,37 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+The OpenNLP model is trained based on the data provided by the International
Joint
+Workshop on Natural Language Processing in Biomedicine and its Applications.
+
+License
+-------
+
+The Annotations used to train the model are under the
+Creative Commons Attribution 3.0 Unported License.
+
+Corpus was build from abstracts of 2,000 abstracts selected from a full text
+search. Based on [1] resources published by PubMed (this includes abstracts)
+are under public domain.
+
+The original LICENSE file of the corpus is also included.
+
+
+More information about the corpus can be found at
+http://www.nactem.ac.uk/tsujii/GENIA/ERtask/report.html
+
+[1] http://www.ncbi.nlm.nih.gov/About/disclaimer.html
\ No newline at end of file
Added:
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/bionlp2004-DNA-en.bin
URL:
http://svn.apache.org/viewvc/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/bionlp2004-DNA-en.bin?rev=1405306&view=auto
==============================================================================
Binary file - no diff available.
Propchange:
stanbol/trunk/enhancer/engines/opennlp-ner/src/test/resources/org/apache/stanbol/data/opennlp/bionlp2004-DNA-en.bin
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
