On Mar 17, 2010, at 11:34 AM, Paul Taylor wrote:
> Grant Ingersoll wrote:
>> What's your current chain of TokenFilters? How many exceptions do you
>> expect? That is, could you enumerate them?
>>
> Very few, yes I could enumerate them, but not sure what exactly you are
> suggesting, what I was going to do would be add to the charConvertMap (when I
> posted I thought this was only for individual chars not strings)
You could have modify whichever filter is removing them to take in a protected
words list and then short circuit to not remove that token. This would be a
hash map lookup, which should be faster than the char replacement you are
considering. Many of the stemmers do this.
>
>
> This is my analyzer:
>
> package org.musicbrainz.search.analysis;
>
> import org.apache.lucene.analysis.Analyzer;
> import org.apache.lucene.analysis.TokenStream;
> import org.apache.lucene.analysis.CharFilter;
> import org.apache.lucene.analysis.MappingCharFilter;
> import org.apache.lucene.analysis.NormalizeCharMap;
>
>
> import java.io.IOException;
> import java.io.Reader;
>
> import com.ibm.icu.text.Transliterator;
> import org.apache.lucene.util.Version;
> import org.musicbrainz.search.LuceneVersion;
>
> /**
> * Filters StandardTokenizer with StandardFilter, ICUTransformFilter,
> AccentFilter, LowerCaseFilter
> * and no stop words.
> */
> public class StandardUnaccentAnalyzer extends Analyzer {
>
> private NormalizeCharMap charConvertMap;
>
> private void setCharConvertMap() {
> charConvertMap = new NormalizeCharMap();
> charConvertMap.add("&","and");
>
> //Hebrew chars converted to western cases so matches both
> charConvertMap.add("\u05f3","'");
> charConvertMap.add("\u05be","-");
> charConvertMap.add("\u05f4","\"");
>
>
> }
>
> public StandardUnaccentAnalyzer() {
> setCharConvertMap();
> }
>
> public TokenStream tokenStream(String fieldName, Reader reader) {
> CharFilter mappingCharFilter = new MappingCharFilter(charConvertMap,reader);
> StandardTokenizer tokenStream = new
> StandardTokenizer(LuceneVersion.LUCENE_VERSION, mappingCharFilter);
> TokenStream result = new ICUTransformFilter(tokenStream,
> Transliterator.getInstance("[ー[:Script=Katakana:]]Katakana-Hiragana"));
> result = new ICUTransformFilter(result,
> Transliterator.getInstance("Traditional-Simplified"));
> result = new StandardFilter(result);
> result = new AccentFilter(result);
> result = new LowercaseFilter(result);
> return result;
> }
>
> private static final class SavedStreams {
> StandardTokenizer tokenStream;
> TokenStream filteredTokenStream;
> }
>
> public TokenStream reusableTokenStream(String fieldName, Reader reader)
> throws IOException {
> SavedStreams streams = (SavedStreams)getPreviousTokenStream();
> if (streams == null) {
> streams = new SavedStreams();
> setPreviousTokenStream(streams);
> streams.tokenStream = new StandardTokenizer(LuceneVersion.LUCENE_VERSION, new
> MappingCharFilter(charConvertMap, reader));
> streams.filteredTokenStream = new ICUTransformFilter(streams.tokenStream,
> Transliterator.getInstance("[ー [:Script=Katakana:]]Katakana-Hiragana"));
> streams.filteredTokenStream = new
> ICUTransformFilter(streams.filteredTokenStream,
> Transliterator.getInstance("Traditional-Simplified"));
> streams.filteredTokenStream = new StandardFilter(streams.filteredTokenStream);
> streams.filteredTokenStream = new AccentFilter(streams.filteredTokenStream);
> streams.filteredTokenStream = new
> LowercaseFilter(streams.filteredTokenStream);
> }
> else {
> streams.tokenStream.reset(new MappingCharFilter(charConvertMap,reader));
> }
> return streams.filteredTokenStream;
> }
> }
>
--------------------------
Grant Ingersoll
http://www.lucidimagination.com/
Search the Lucene ecosystem using Solr/Lucene:
http://www.lucidimagination.com/search
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
