Hi Paul,
You need to call tokenizer.reset() before you call incrementToken()
Alan Woodward
www.flax.co.uk
On 26 Feb 2013, at 12:26, Paul Taylor wrote:
> This works in 3.6, but in 4.1 fails whats wrong with the code
>
> public void testTokenization() throws IOException
> {
> StringBuffer sb = new StringBuffer();
>
>
> for(char i=0;i<100;i++)
> {
> Character c = new Character(i);
> if(!Character.isWhitespace(c)) {
> sb.append(new Character(i).toString() + ' ');
> }
> }
> System.out.println(sb.toString());
> Tokenizer tokenizer = new
> WhitespaceTokenizer(LuceneVersion.LUCENE_VERSION,new
> StringReader(sb.toString()));
> while(tokenizer.incrementToken())
> {
> System.out.println("inctoken");
>
> }
> }
>
> throwing exception
>
> java.lang.ArrayIndexOutOfBoundsException: -1
> at java.lang.Character.codePointAtImpl(Character.java:2405)
> at java.lang.Character.codePointAt(Character.java:2369)
> at
> org.apache.lucene.analysis.util.CharacterUtils$Java5CharacterUtils.codePointAt(CharacterUtils.java:164)
> at
> org.apache.lucene.analysis.util.CharTokenizer.incrementToken(CharTokenizer.java:166)
> at
> org.musicbrainz.search.analysis.CompareNormalizationFiltersTest.testTokenization(CompareNormalizationFiltersTest.java:36)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:157)
> at
> com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:76)
> at
> com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:195)
> at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:63)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120)
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: [email protected]
> For additional commands, e-mail: [email protected]
>
