/*
 * CJKBigramFilterTest.java
 *
 */

package net.nutch.analysis;

import junit.framework.*;

import java.io.*;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Token;
import net.nutch.analysis.*; // in case when this test is moved to another pack
/**
 *
 * @author  juncai
 */
public class CJKBigramFilterTest extends TestCase{  

  /** Creates a new instance of DefaultCategorizerTest */
  public CJKBigramFilterTest(String name) {
    super(name);
  }

  public static void main(String args[]) {
    junit.textui.TestRunner.run(suite());
  }  

  public static Test suite() {
    return new TestSuite(CJKBigramFilterTest.class);
  }  
  
  public void testNext()throws IOException{
    TokenStream ts = new NutchDocumentTokenizer(new StringReader("\u8521cai jun"));
    ts = new CJKBigramFilter(ts);
    assertTrue(flat(ts).equals("\u8521 cai jun "));
    
    ts = new NutchDocumentTokenizer(new StringReader("cai\u8521\u9a8f"));
    ts = new CJKBigramFilter(ts);
    assertTrue(flat(ts).equals("cai \u8521 \u8521\u9a8f \u9a8f "));
    
    ts = new NutchDocumentTokenizer(new StringReader("cai\u8521"));
    ts = new CJKBigramFilter(ts);
    assertTrue(flat(ts).equals("cai \u8521 "));
    
    ts = new NutchDocumentTokenizer(new StringReader("\u8521 cai"));
    ts = new CJKBigramFilter(ts);
    assertTrue(flat(ts).equals("\u8521 cai "));
    
    ts = new NutchDocumentTokenizer(new StringReader("blah\u8521\u9a8f\u5b66cai\u4e60jun"));
    ts = new CJKBigramFilter(ts);
    assertTrue(flat(ts).equals("blah \u8521 \u8521\u9a8f \u9a8f \u9a8f\u5b66 \u5b66 cai \u4e60 jun "));
  }
  
  protected String flat(TokenStream ts) throws IOException{
    Token token;
    StringBuffer sb = new StringBuffer();
    while ((token = ts.next()) != null) {
      //System.out.println("Token: " + token.termText() + " type: " + token.type());
      sb.append(token.termText() + " ");
    }    
    return sb.toString();
  }
}