/*
 * CJKBigramFilter.java
 *
 */

package net.nutch.analysis;

import java.io.*;
import java.util.*;
import java.util.logging.Logger;

import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Token;

import net.nutch.util.*;
/**
 * A filter to token CJK into bi-gram. For example: <BR/>
 * "blah C1C2C3 blah C4 blah" -> <BR/>
 * "blah", "C1", "C1C2"+0, "C2", "C2C3"+0, "C3", "blah", "C4", "blah" <BR/>
 * (+0 to indicate a positionIncrement of zero, rather than 1.)
 *
 * (Implementation according to Doug's reply.)
 *
 * @author <a href="mailto:juncaitech@yahoo.com">Jun Cai</a>
 */
public class CJKBigramFilter extends TokenFilter{
  private static final Logger LOG =
    LogFormatter.getLogger("net.nutch.analysis.CJKBigramFilter");  
  
  private Token preCJK;  
  private LinkedList nextQueue = new LinkedList();
  
  /** Creates a new instance of CJKBigramFilter */
  public CJKBigramFilter(TokenStream input) {
    super(input);
  }
  
  /** Inserts bi-grams for CJK into a token stream. */
  public Token next() throws IOException {
    Token current = input.next();
    Token temp = null;
    
    if (null != current){
      if (current.type().equals("<SIGRAM>")){
        if (null != preCJK){
          nextQueue.add(gramToken(preCJK, current));        
        }
        preCJK = current;      
      } else {
        preCJK = null;
      }
      nextQueue.add(current);
    }
    
    return popNext(); 
  }
  
  /** Pops nextQueue or, if empty, reads a new token. */
  private Token popNext() throws IOException {
    if (nextQueue.size() > 0){
      return (Token)nextQueue.removeFirst();
    } else {
      return null;
    }
  }

  /** Construct a compound token. */
  private Token gramToken(Token first, Token second) {    
    Token result = new Token(first.termText() + second.termText(),
                             first.startOffset(), second.endOffset(),
                             "gram");
    result.setPositionIncrement(0);
    return result;
  }
  
  /** For debugging. */
  public static void main(String[] args) throws Exception {
    StringBuffer text = new StringBuffer();
    for (int i = 0; i < args.length; i++) {
      text.append(args[i]);
      text.append(' ');
    }
    //TokenStream ts = new NutchDocumentTokenizer(new StringReader(text.toString()));    
    TokenStream ts = new NutchDocumentTokenizer(new StringReader("blah\u8521\u9a8f\u5b66cai\u4e60jun"));
    //TokenStream ts = new NutchDocumentTokenizer(new StringReader("\u8521cai jun"));
    
    ts = new CJKBigramFilter(ts);
    Token token;
    while ((token = ts.next()) != null) {
      System.out.println("Token: " + token.termText() + " type: " + token.type());
    }

  }  
}