Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java?rev=1403238&r1=1403237&r2=1403238&view=diff ============================================================================== --- stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java (original) +++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java Mon Oct 29 10:24:38 2012 @@ -16,6 +16,8 @@ */ package org.apache.stanbol.enhancer.engines.keywordextraction.engine; +import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.PHRASE_ANNOTATION; +import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.POS_ANNOTATION; import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_CREATOR; import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_LANGUAGE; import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_CONFIDENCE; @@ -32,11 +34,14 @@ import static org.junit.Assert.assertTru import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import junit.framework.Assert; + import opennlp.tools.tokenize.SimpleTokenizer; import org.apache.clerezza.rdf.core.Literal; @@ -48,17 +53,24 @@ import org.apache.clerezza.rdf.core.UriR import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl; import org.apache.clerezza.rdf.core.impl.TripleImpl; import org.apache.stanbol.commons.opennlp.OpenNLP; -import org.apache.stanbol.commons.opennlp.TextAnalyzer.TextAnalyzerConfig; import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory; import org.apache.stanbol.enhancer.engines.keywordextraction.engine.KeywordLinkingEngine; -import org.apache.stanbol.enhancer.engines.keywordextraction.impl.ClasspathDataFileProvider; +import org.apache.stanbol.enhancer.engines.keywordextraction.impl.EntityLinker; +import org.apache.stanbol.enhancer.engines.keywordextraction.impl.LinkedEntity; +import org.apache.stanbol.enhancer.engines.keywordextraction.impl.Suggestion; import org.apache.stanbol.enhancer.engines.keywordextraction.impl.TestSearcherImpl; -import org.apache.stanbol.enhancer.engines.keywordextraction.linking.EntityLinker; import org.apache.stanbol.enhancer.engines.keywordextraction.linking.EntityLinkerConfig; import org.apache.stanbol.enhancer.engines.keywordextraction.linking.EntityLinkerConfig.RedirectProcessingMode; -import org.apache.stanbol.enhancer.engines.keywordextraction.linking.LinkedEntity; -import org.apache.stanbol.enhancer.engines.keywordextraction.linking.Suggestion; -import org.apache.stanbol.enhancer.engines.keywordextraction.linking.impl.OpenNlpAnalysedContentFactory; +import org.apache.stanbol.enhancer.engines.keywordextraction.linking.TextProcessingConfig; +import org.apache.stanbol.enhancer.engines.keywordextraction.linking.impl.OpenNlpLabelTokenizer; +import org.apache.stanbol.enhancer.nlp.model.AnalysedText; +import org.apache.stanbol.enhancer.nlp.model.AnalysedTextFactory; +import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils; +import org.apache.stanbol.enhancer.nlp.model.annotation.Value; +import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag; +import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory; +import org.apache.stanbol.enhancer.nlp.pos.Pos; +import org.apache.stanbol.enhancer.nlp.pos.PosTag; import org.apache.stanbol.enhancer.servicesapi.ContentItem; import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory; import org.apache.stanbol.enhancer.servicesapi.EngineException; @@ -92,9 +104,12 @@ public class KeywordLinkingEngineTest { */ public static final String TEST_TEXT = "Dr. Patrick Marshall (1869 - November 1950) was a" + " geologist who lived in New Zealand and worked at the University of Otago."; - public static final String TEST_TEXT2 = "A CBS televised debate between Australia's " + - "candidates for Prime Minister in the upcoming US election has been rescheduled " + - "and shortend, to avoid a clash with popular cookery sow MasterChef."; + + private static AnalysedText TEST_ANALYSED_TEXT; + +// public static final String TEST_TEXT2 = "A CBS televised debate between Australia's " + +// "candidates for Prime Minister in the upcoming US election has been rescheduled " + +// "and shortend, to avoid a clash with popular cookery sow MasterChef."; private static final ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance(); @@ -102,7 +117,6 @@ public class KeywordLinkingEngineTest { static TestSearcherImpl searcher; static ValueFactory factory = InMemoryValueFactory.getInstance(); - private static OpenNLP openNLP; public static final String NAME = NamespaceEnum.rdfs+"label"; public static final String TYPE = NamespaceEnum.rdf+"type"; @@ -110,7 +124,6 @@ public class KeywordLinkingEngineTest { @BeforeClass public static void setUpServices() throws IOException { - openNLP = new OpenNLP(new ClasspathDataFileProvider("DUMMY_SYMBOLIC_NAME")); searcher = new TestSearcherImpl(NAME,SimpleTokenizer.INSTANCE); //add some terms to the searcher Representation rep = factory.createRepresentation("urn:test:PatrickMarshall"); @@ -153,6 +166,46 @@ public class KeywordLinkingEngineTest { rep.addNaturalText(NAME, "University of Otago (Texas)"); rep.addReference(TYPE, OntologicalClasses.DBPEDIA_ORGANISATION.getUnicodeString()); searcher.addEntity(rep); + + Value<PhraseTag> nounPhrase = Value.value(new PhraseTag("NP",LexicalCategory.Noun),1d); + TEST_ANALYSED_TEXT = AnalysedTextFactory.getDefaultInstance().createAnalysedText( + ciFactory.createBlob(new StringSource(TEST_TEXT))); + TEST_ANALYSED_TEXT.addSentence(0, TEST_ANALYSED_TEXT.getEnd()); + //add some noun phrases + TEST_ANALYSED_TEXT.addChunk(0, "Dr. Patrick Marshall".length()).addAnnotation(PHRASE_ANNOTATION, nounPhrase); + TEST_ANALYSED_TEXT.addChunk(TEST_TEXT.indexOf("New Zealand"), TEST_TEXT.indexOf("New Zealand")+"New Zealand".length()) + .addAnnotation(PHRASE_ANNOTATION, nounPhrase); + TEST_ANALYSED_TEXT.addChunk(TEST_TEXT.indexOf("geologist"), TEST_TEXT.indexOf("geologist")+"geologist".length()) + .addAnnotation(PHRASE_ANNOTATION, nounPhrase); + TEST_ANALYSED_TEXT.addChunk(TEST_TEXT.indexOf("the University of Otago"), + TEST_TEXT.length()-1).addAnnotation(PHRASE_ANNOTATION, nounPhrase); + //add some tokens + TEST_ANALYSED_TEXT.addToken(0, 2).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.Abbreviation),1d)); + TEST_ANALYSED_TEXT.addToken(2, 3).addAnnotation(POS_ANNOTATION, Value.value(new PosTag(".",Pos.Point),1d)); + TEST_ANALYSED_TEXT.addToken(4, 11).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d)); + TEST_ANALYSED_TEXT.addToken(12, 20).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d)); + int start = TEST_TEXT.indexOf("(1869 - November 1950)"); + TEST_ANALYSED_TEXT.addToken(start,start+1).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("(",Pos.OpenBracket),1d)); + TEST_ANALYSED_TEXT.addToken(start+1,start+5).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NUM",Pos.Numeral),1d)); + TEST_ANALYSED_TEXT.addToken(start+6,start+7).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("-",Pos.Hyphen),1d)); + TEST_ANALYSED_TEXT.addToken(start+8,start+16).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d)); + TEST_ANALYSED_TEXT.addToken(start+17,start+21).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NUM",Pos.Numeral),1d)); + TEST_ANALYSED_TEXT.addToken(start+21,start+22).addAnnotation(POS_ANNOTATION, Value.value(new PosTag(")",Pos.CloseBracket),1d)); + + start = TEST_TEXT.indexOf("geologist"); + TEST_ANALYSED_TEXT.addToken(start,start+9).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d)); + + start = TEST_TEXT.indexOf("New Zealand"); + TEST_ANALYSED_TEXT.addToken(start,start+3).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d)); + TEST_ANALYSED_TEXT.addToken(start+4,start+11).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d)); + + start = TEST_TEXT.indexOf("the University of Otago"); + TEST_ANALYSED_TEXT.addToken(start,start+3).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("ART",Pos.Article),1d)); + TEST_ANALYSED_TEXT.addToken(start+4,start+14).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d)); + TEST_ANALYSED_TEXT.addToken(start+15,start+17).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("OF",LexicalCategory.PronounOrDeterminer),1d)); + TEST_ANALYSED_TEXT.addToken(start+18,start+23).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d)); + TEST_ANALYSED_TEXT.addToken(start+23,start+24).addAnnotation(POS_ANNOTATION, Value.value(new PosTag(".",Pos.Point),1d)); + } @Before @@ -172,17 +225,19 @@ public class KeywordLinkingEngineTest { } /** * This tests the EntityLinker functionality (if the expected Entities - * are linked) + * are linked). In this case with the default configurations for + * {@link LexicalCategory#Noun}. * @throws Exception */ @Test - public void testTaxonomyLinker() throws Exception { - OpenNlpAnalysedContentFactory acf = OpenNlpAnalysedContentFactory.getInstance(openNLP, - new TextAnalyzerConfig()); + public void testEntityLinkerWithNouns() throws Exception { + TextProcessingConfig tpc = new TextProcessingConfig(); + tpc.setProcessedLexicalCategories(KeywordLinkingEngine.DEFAULT_PROCESSED_LEXICAL_CATEGORIES); + tpc.setProcessedPos(Collections.EMPTY_SET); EntityLinkerConfig config = new EntityLinkerConfig(); config.setRedirectProcessingMode(RedirectProcessingMode.FOLLOW); - EntityLinker linker = new EntityLinker( - acf.create(TEST_TEXT,"en"), searcher, config); + EntityLinker linker = new EntityLinker(TEST_ANALYSED_TEXT,"en", + tpc, searcher, config, new OpenNlpLabelTokenizer(null)); linker.process(); Map<String,List<String>> expectedResults = new HashMap<String,List<String>>(); expectedResults.put("Patrick Marshall", new ArrayList<String>( @@ -193,10 +248,45 @@ public class KeywordLinkingEngineTest { Arrays.asList("urn:test:NewZealand"))); expectedResults.put("University of Otago", new ArrayList<String>( Arrays.asList("urn:test:UniversityOfOtago","urn:test:UniversityOfOtago_Texas"))); + validateEntityLinkerResults(linker, expectedResults); + } + /** + * This tests the EntityLinker functionality (if the expected Entities + * are linked). In this case with the default configurations for + * {@link Pos#ProperNoun}. + * @throws Exception + */ + @Test + public void testEntityLinkerWithProperNouns() throws Exception { + TextProcessingConfig tpc = new TextProcessingConfig(); + tpc.setProcessedLexicalCategories(Collections.EMPTY_SET); + tpc.setProcessedPos(KeywordLinkingEngine.DEFAULT_PROCESSED_POS_TYPES); + EntityLinkerConfig config = new EntityLinkerConfig(); + config.setRedirectProcessingMode(RedirectProcessingMode.FOLLOW); + EntityLinker linker = new EntityLinker(TEST_ANALYSED_TEXT,"en", + tpc, searcher, config, new OpenNlpLabelTokenizer(null)); + linker.process(); + Map<String,List<String>> expectedResults = new HashMap<String,List<String>>(); + expectedResults.put("Patrick Marshall", new ArrayList<String>( + Arrays.asList("urn:test:PatrickMarshall"))); + //Geologist is a common noun and MUST NOT be found + //expectedResults.put("geologist", new ArrayList<String>( + // Arrays.asList("urn:test:redirect:Geologist"))); //the redirected entity + expectedResults.put("New Zealand", new ArrayList<String>( + Arrays.asList("urn:test:NewZealand"))); + expectedResults.put("University of Otago", new ArrayList<String>( + Arrays.asList("urn:test:UniversityOfOtago","urn:test:UniversityOfOtago_Texas"))); + validateEntityLinkerResults(linker, expectedResults); + } + private void validateEntityLinkerResults(EntityLinker linker, Map<String,List<String>> expectedResults) { + log.info("---------------------"); + log.info("- Validating Results-"); + log.info("---------------------"); for(LinkedEntity linkedEntity : linker.getLinkedEntities().values()){ + log.info("> LinkedEntity {}",linkedEntity); List<String> expectedSuggestions = expectedResults.remove(linkedEntity.getSelectedText()); - assertNotNull("LinkedEntity "+linkedEntity.getSelectedText()+ - "is not an expected Result (or was found twice)", expectedSuggestions); + assertNotNull("LinkedEntity '"+linkedEntity.getSelectedText()+ + "' is not an expected Result (or was found twice)", expectedSuggestions); linkedEntity.getSuggestions().iterator(); assertEquals("Number of suggestions "+linkedEntity.getSuggestions().size()+ " != number of expected suggestions "+expectedSuggestions.size()+ @@ -217,6 +307,8 @@ public class KeywordLinkingEngineTest { score = suggestion.getScore(); } } + Assert.assertTrue("The expected Result(s) "+expectedResults+" wehre not found", + expectedResults.isEmpty()); } /** * This tests if the Enhancements created by the Engine confirm to the @@ -228,12 +320,14 @@ public class KeywordLinkingEngineTest { public void testEngine() throws IOException, EngineException { EntityLinkerConfig linkerConfig = new EntityLinkerConfig(); linkerConfig.setRedirectProcessingMode(RedirectProcessingMode.FOLLOW); - KeywordLinkingEngine engine = KeywordLinkingEngine.createInstance(openNLP, searcher, new TextAnalyzerConfig(), - linkerConfig); + KeywordLinkingEngine engine = KeywordLinkingEngine.createInstance(searcher, new TextProcessingConfig(), + linkerConfig, new OpenNlpLabelTokenizer()); engine.referencedSiteName = TEST_REFERENCED_SITE_NAME; ContentItem ci = ciFactory.createContentItem(new StringSource(TEST_TEXT)); //tells the engine that this is an English text ci.getMetadata().add(new TripleImpl(ci.getUri(), DC_LANGUAGE, new PlainLiteralImpl("en"))); + //and add the AnalysedText instance used for this test + ci.addPart(AnalysedText.ANALYSED_TEXT_URI, TEST_ANALYSED_TEXT); //compute the enhancements engine.computeEnhancements(ci); //validate the enhancement results
Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java?rev=1403238&r1=1403237&r2=1403238&view=diff ============================================================================== --- stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java (original) +++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java Mon Oct 29 10:24:38 2012 @@ -74,7 +74,7 @@ public class TestSearcherImpl implements public Collection<? extends Representation> lookup(String field, Set<String> includeFields, List<String> search, - String... languages) throws IllegalStateException { + String[] languages,Integer numResults) throws IllegalStateException { if(field.equals(nameField)){ //we do not need sorting //Representation needs to implement equals, therefore results filters multiple matches Added: stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties?rev=1403238&view=auto ============================================================================== --- stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties (added) +++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties Mon Oct 29 10:24:38 2012 @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n +log4j.logger.org.apache.stanbol.enhancer.engines.keywordextraction=DEBUG \ No newline at end of file
