[
https://issues.apache.org/jira/browse/LUCENE-5230?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Littlestar updated LUCENE-5230:
-------------------------------
Description:
@Test
public void test_AlphaNumAnalyzer() throws IOException {
Analyzer analyzer = new CJKAnalyzer(Version.LUCENE_44);
TokenStream token = analyzer.tokenStream("test", new
StringReader("0009bf2d97e9f86a7188002a64a84b351379323870284;0009bf2e97e9f8707188002a64a84b351379323870273;000ae1f0b4390779eed1002a64a8a7950;0001e87997e9f0017188000a64a84b351378869697875;fff205ce319b68ff1a3c002964a820841377769850018;000ae1f0b439077beed1002a64a8a7950;000ae1f1b439077deed1002a64a8a7950;0009bf2d97e9f86c7188002a64a84b351379323870281;0015adfd0c69d870debb000a64a8477c1378809423441"));
while (token.incrementToken()) {
final CharTermAttribute termAtt =
token.addAttribute(CharTermAttribute.class);
System.out.println(termAtt.toString());
}
analyzer.close();
}
was:
@Test
public void test_AlphaNumAnalyzer() throws IOException {
Analyzer analyzer = new CJKAnalyzer(Version.LUCENE_44);
TokenStream token = analyzer.tokenStream("test", new
StringReader("中国"));
//TokenStream token = analyzer.tokenStream("test", new
StringReader("0009bf2d97e9f86a7188002a64a84b351379323870284;0009bf2e97e9f8707188002a64a84b351379323870273;000ae1f0b4390779eed1002a64a8a7950;0001e87997e9f0017188000a64a84b351378869697875;fff205ce319b68ff1a3c002964a820841377769850018;000ae1f0b439077beed1002a64a8a7950;000ae1f1b439077deed1002a64a8a7950;0009bf2d97e9f86c7188002a64a84b351379323870281;0015adfd0c69d870debb000a64a8477c1378809423441"));
while (token.incrementToken()) {
final CharTermAttribute termAtt =
token.addAttribute(CharTermAttribute.class);
System.out.println(termAtt.toString());
}
analyzer.close();
}
============================
java.lang.NullPointerException
at
org.apache.lucene.analysis.standard.StandardTokenizerImpl.zzRefill(StandardTokenizerImpl.java:923)
at
org.apache.lucene.analysis.standard.StandardTokenizerImpl.getNextToken(StandardTokenizerImpl.java:1133)
at
org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken(StandardTokenizer.java:171)
at
org.apache.lucene.analysis.cjk.CJKWidthFilter.incrementToken(CJKWidthFilter.java:63)
at
org.apache.lucene.analysis.core.LowerCaseFilter.incrementToken(LowerCaseFilter.java:54)
at
org.apache.lucene.analysis.cjk.CJKBigramFilter.doNext(CJKBigramFilter.java:240)
at
org.apache.lucene.analysis.cjk.CJKBigramFilter.incrementToken(CJKBigramFilter.java:169)
at
org.apache.lucene.analysis.util.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:81)
Summary: CJKAnalyzer can't split ";" (was: CJKAnalyzer
java.lang.NullPointerException)
> CJKAnalyzer can't split ";"
> ---------------------------
>
> Key: LUCENE-5230
> URL: https://issues.apache.org/jira/browse/LUCENE-5230
> Project: Lucene - Core
> Issue Type: Bug
> Components: modules/analysis
> Affects Versions: 4.4
> Reporter: Littlestar
> Priority: Minor
>
> @Test
> public void test_AlphaNumAnalyzer() throws IOException {
> Analyzer analyzer = new CJKAnalyzer(Version.LUCENE_44);
> TokenStream token = analyzer.tokenStream("test", new
> StringReader("0009bf2d97e9f86a7188002a64a84b351379323870284;0009bf2e97e9f8707188002a64a84b351379323870273;000ae1f0b4390779eed1002a64a8a7950;0001e87997e9f0017188000a64a84b351378869697875;fff205ce319b68ff1a3c002964a820841377769850018;000ae1f0b439077beed1002a64a8a7950;000ae1f1b439077deed1002a64a8a7950;0009bf2d97e9f86c7188002a64a84b351379323870281;0015adfd0c69d870debb000a64a8477c1378809423441"));
> while (token.incrementToken()) {
> final CharTermAttribute termAtt =
> token.addAttribute(CharTermAttribute.class);
> System.out.println(termAtt.toString());
> }
> analyzer.close();
> }
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
