Author: natalia
Date: Sat Oct 13 19:41:50 2007
New Revision: 584476

URL: http://svn.apache.org/viewvc?rev=584476&view=rev
Log:
Lucene-based full text search in the content of DOM nodes

Added:
    xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/
    
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
   (with props)
    
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
   (with props)
    
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
   (with props)
    
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
   (with props)
    
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java 
  (with props)

Added: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
URL: 
http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java?rev=584476&view=auto
==============================================================================
--- 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
 (added)
+++ 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
 Sat Oct 13 19:41:50 2007
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.xindice.xml.dom.NodeImpl;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Node;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Collection;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Implementation of IndexReader that works with set of DOM nodes in memory.
+ * Set of nodes is constant - no modifications of it are allowed.
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeReader extends IndexReader {
+    // list of DOM nodes
+    private final ArrayList[] nodes;
+
+    private byte[] norms;
+
+    // maps term to a list of documents where it appears
+    private final SortedMap termMap;
+
+    protected NodeReader(NodeList list, Analyzer analyzer) {
+        super(null);
+
+        nodes = new ArrayList[list.getLength()];
+
+        for (int i = 0; i < nodes.length; i++) {
+            nodes[i] = new ArrayList();
+            Node node = list.item(i);
+
+            String text;
+            if (node instanceof NodeImpl) {
+                // DOM Level 3 method
+                text = ((NodeImpl) node).getTextContent();
+            } else {
+                text = getTextContent(node);
+            }
+
+            TokenStream stream = analyzer.tokenStream("", new 
StringReader(text));
+            try {
+                Token token;
+                while ((token = stream.next()) != null) {
+                    nodes[i].add(token.termText());
+                }
+            } catch (IOException e) {
+                // won't happen
+            }
+        }
+
+        // init norms
+        norms = new byte[nodes.length];
+        Arrays.fill(norms, DefaultSimilarity.encodeNorm(1.0f));
+
+        // build term enumeration
+        termMap = buildTermMap();
+    }
+
+    public TermFreqVector[] getTermFreqVectors(int docNumber) {
+        throw new UnsupportedOperationException();
+    }
+
+    public TermFreqVector getTermFreqVector(int docNumber, String field) {
+        throw new UnsupportedOperationException();
+    }
+
+    public int numDocs() {
+        return nodes.length;
+    }
+
+    public int maxDoc() {
+        return nodes.length;
+    }
+
+    /**
+     * Method is not supported.
+     */
+    public Document document(int n, FieldSelector fieldSelector) {
+        return null;
+    }
+
+    /**
+     * Deletion is not supported.
+     */
+    public boolean isDeleted(int n) {
+        return false;
+    }
+
+    /**
+     * Deletion is not supported.
+     */
+    public boolean hasDeletions() {
+        return false;
+    }
+
+    public byte[] norms(String field) throws IOException {
+        return field.length() == 0 ? norms : null;
+    }
+
+    public void norms(String field, byte[] bytes, int offset) {
+        System.arraycopy(norms, 0, bytes, offset, maxDoc());
+    }
+
+    protected void doSetNorm(int doc, String field, byte value) {
+        if (field.length() > 0) {
+            return;
+        }
+
+        norms[doc] = value;
+    }
+
+    public TermEnum terms() {
+        return new NodeTermEnum(termMap);
+    }
+
+    public TermEnum terms(Term t) {
+        return new NodeTermEnum(termMap, t);
+    }
+
+    /**
+     * Builds the map of all the terms in all the nodes to the list of
+     * node numbers where those terms appear.
+     * @return Map with keys sorted in ascending order
+     */
+    private SortedMap buildTermMap() {
+        SortedMap map = new TreeMap();
+
+        for (int i = 0; i < nodes.length; i++) {
+            for (int j = 0; j < nodes[i].size(); j++) {
+                String term = (String) nodes[i].get(j);
+
+                List docs;
+                if (map.containsKey(term)) {
+                    docs = (List) map.get(term);
+                } else {
+                    docs = new ArrayList();
+                }
+
+                docs.add(new Integer(i));
+                map.put(term, docs);
+            }
+        }
+
+        return map;
+    }
+
+    public int docFreq(Term t) {
+        List docs = (List) termMap.get(t.text());
+
+        // no such term
+        if (docs == null) {
+            return 0;
+        }
+
+        HashSet set = new HashSet();
+        set.addAll(docs);
+        return set.size();
+    }
+
+    public TermDocs termDocs() {
+        return new NodeTermDocs(this);
+    }
+
+    public TermPositions termPositions() {
+        return new NodeTermPositions(this);
+    }
+
+    /**
+     * Deletion is not supported.
+     */
+    protected void doDelete(int docNum) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Deletion is not supported.
+     */
+    protected void doUndeleteAll() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Not applicable
+     */
+    protected void doCommit() {
+    }
+
+    /**
+     * Not applicable
+     */
+    protected void doClose() {
+    }
+
+    /**
+     * Field names are not supported.
+     */
+    public Collection getFieldNames(FieldOption fldOption) {
+        throw new UnsupportedOperationException();
+    }
+
+    ArrayList[] getNodes() {
+        return nodes;
+    }
+
+    SortedMap getTermMap() {
+        return termMap;
+    }
+
+    /**
+     * Get text content of a DOM node. This is the same as DOM Level 3 method
+     * getTextContent().
+     * @param node
+     * @return The text content of this node and its descendants.
+     */
+    private String getTextContent(Node node) {
+        String text = null;
+        switch (node.getNodeType()) {
+            case Node.ATTRIBUTE_NODE:
+            case Node.CDATA_SECTION_NODE:
+            case Node.COMMENT_NODE:
+            case Node.PROCESSING_INSTRUCTION_NODE:
+            case Node.TEXT_NODE:
+                text = node.getNodeValue();
+                break;
+            case Node.ELEMENT_NODE:
+            case Node.DOCUMENT_FRAGMENT_NODE:
+            case Node.ENTITY_NODE:
+            case Node.ENTITY_REFERENCE_NODE:
+                StringBuffer val = new StringBuffer();
+
+                NodeList children = node.getChildNodes();
+                if (children == null || children.getLength() == 0) {
+                    text = "";
+                    break;
+                }
+
+                for (int i = 0; i < children.getLength(); i++) {
+                    val.append(getTextContent(children.item(i)));
+                }
+                text = val.toString();
+                break;
+        }
+
+        return text;
+    }
+}

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
------------------------------------------------------------------------------
    svn:keywords = Id Revision Author Date

Added: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
URL: 
http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java?rev=584476&view=auto
==============================================================================
--- 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
 (added)
+++ 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
 Sat Oct 13 19:41:50 2007
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.List;
+
+/**
+ * TermDocs for searching over DOM nodes in memory
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeTermDocs implements TermDocs {
+    protected int num;
+    protected ArrayList[] nodes;
+    protected Map termMap;
+    protected int[] freqs;
+    private boolean init;
+
+    public NodeTermDocs(NodeReader reader) {
+        nodes = reader.getNodes();
+        termMap = reader.getTermMap();
+
+        freqs = new int[nodes.length];
+    }
+
+    public void seek(Term term) {
+        List docs = (List) termMap.get(term.text());
+        if (docs != null) {
+            for (Iterator i = docs.iterator(); i.hasNext(); ) {
+                int idx = ((Integer) i.next()).intValue();
+                freqs[idx]++;
+            }
+        }
+    }
+
+    public void seek(TermEnum termEnum)  {
+        seek(termEnum.term());
+    }
+
+    public int doc() {
+        return num;
+    }
+
+    public int freq() {
+        return freqs[num];
+    }
+
+    public boolean next() {
+        if (!init) {
+            init = true;
+            num = 0;
+        } else {
+            num++;
+        }
+
+        while(num < nodes.length && freqs[num] == 0) {
+            num++;
+        }
+
+        return num < nodes.length;
+    }
+
+    public int read(int[] docs, int[] freqs) {
+        int count = 0;
+        for (int i = 0; i < docs.length && num < nodes.length; i++, num++) {
+            if (this.freqs[num] > 0) {
+                freqs[count] = this.freqs[num];
+                docs[count] = num;
+                count++;
+            }
+        }
+
+        return count;
+    }
+
+    public boolean skipTo(int target) {
+        num = target;
+        return num < nodes.length;
+    }
+
+    /**
+     * Not applicable
+     */
+    public void close() {
+    }
+}

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
------------------------------------------------------------------------------
    svn:keywords = Id Revision Author Date

Added: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
URL: 
http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java?rev=584476&view=auto
==============================================================================
--- 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
 (added)
+++ 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
 Sat Oct 13 19:41:50 2007
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.Term;
+
+import java.util.Iterator;
+import java.util.SortedMap;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Ordered set of terms used by NodeReader.
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeTermEnum extends TermEnum {
+    // map that backs this enumeration
+    private SortedMap termMap;
+
+    private Iterator iterator;
+    
+    // value of the current term
+    private String curTerm;
+
+    public NodeTermEnum(SortedMap termMap) {
+        this.termMap = termMap;
+        iterator = this.termMap.keySet().iterator();
+        if (iterator.hasNext()) {
+            curTerm = (String) iterator.next();
+        }
+    }
+
+    public NodeTermEnum(SortedMap termMap, Term t) {
+        this(termMap.tailMap(t.text()));
+    }
+
+    public boolean next() {
+        if (iterator.hasNext()) {
+            curTerm = (String) iterator.next();
+            return true;
+        }
+
+        return false;
+    }
+
+    public Term term() {
+        if (curTerm != null) {
+            return new Term("", curTerm);
+        }
+
+        return null;
+    }
+
+    public int docFreq() {
+        List docs = (List) termMap.get(curTerm);
+
+        HashSet set = new HashSet();
+        set.addAll(docs);
+        return set.size();
+    }
+
+    /**
+     * Not applicable
+     */
+    public void close() {
+    }
+}

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
------------------------------------------------------------------------------
    svn:keywords = Id Revision Author Date

Added: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
URL: 
http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java?rev=584476&view=auto
==============================================================================
--- 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
 (added)
+++ 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
 Sat Oct 13 19:41:50 2007
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.Term;
+
+import java.io.IOException;
+
+/**
+ * TermPositions for searching over DOM nodes in memory
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeTermPositions extends NodeTermDocs implements TermPositions {
+    private int pos;
+    private String term;
+
+    public NodeTermPositions(NodeReader reader) {
+        super(reader);
+    }
+
+    public void seek(Term term) {
+        this.term = term.text();
+        super.seek(term);
+    }
+
+    public int nextPosition() throws IOException {
+        while (pos < nodes[num].size()) {
+            if ((nodes[num].get(pos)).equals(term)) {
+                return pos;
+            }
+            pos++;
+        }
+
+        return 0;
+    }
+
+    public boolean next() {
+        if (super.next()) {
+            pos = 0;
+            return true;
+        }
+
+        return false;
+    }
+
+    public int getPayloadLength() {
+        throw new UnsupportedOperationException();
+    }
+
+    public byte[] getPayload(byte[] data, int offset) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    public boolean isPayloadAvailable() {
+        return false;
+    }
+}

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
------------------------------------------------------------------------------
    svn:keywords = Id Revision Author Date

Added: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java
URL: 
http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java?rev=584476&view=auto
==============================================================================
--- 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java 
(added)
+++ 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java 
Sat Oct 13 19:41:50 2007
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.w3c.dom.NodeList;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.Query;
+import org.apache.xindice.core.data.NodeSet;
+
+import java.io.IOException;
+
+/**
+ * Searcher executes full text queries against list of nodes and returns
+ * nodes that match the query in the order of relevance (score) - most relevant
+ * results will be return first.<br>
+ * <br>
+ * Query syntax is the same as syntax of Lucene query, except it does not use
+ * field names.
+ *
+ * @version $Revision$, $Date$
+ */
+public class Searcher {
+    private NodeList nodes;
+    private Analyzer analyzer;
+
+    /**
+     * Builds new Searcher based on list of nodes and analyzer.
+     *
+     * @param nodes List of nodes to search
+     * @param analyzer Analyzer that will be used to tokenize text of the 
nodes.
+     * Choice of analyzer affects query results.
+     * @see org.apache.lucene.analysis.Analyzer
+     */
+    public Searcher(NodeList nodes, Analyzer analyzer) {
+        this.nodes = nodes;
+        this.analyzer = analyzer;
+    }
+
+    /**
+     * Executes query against list of nodes and returns matches in the order
+     * of relevance (score).
+     *
+     * @param query Full text query
+     * @return NodeSet that contains matching nodes
+     * @throws ParseException Query failed to be parsed
+     */
+    public NodeSet search(String query) throws ParseException {
+        Query compQuery = new QueryParser("", analyzer).parse(query);
+        NodeReader reader = new NodeReader(nodes, analyzer);
+        IndexSearcher searcher = new IndexSearcher(reader);
+        Hits hits = null;
+        try {
+            hits = searcher.search(compQuery);
+        } catch (IOException e) {
+            // this searcher does not use file IO, exception won't happen
+        }
+
+        return new ResultSet(hits);
+    }
+
+    private class ResultSet implements NodeSet {
+        private Hits hits;
+        private int count;
+
+        private ResultSet(Hits hits) {
+            this.hits = hits;
+        }
+
+        public boolean hasMoreNodes() {
+            return count < hits.length();
+        }
+
+        public Object getNextNode() {
+            try {
+                return nodes.item(hits.id(count++));
+            } catch (IOException e) {
+                // ignore, does not use IO
+            }
+
+            return null;
+        }
+    }
+}

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: 
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java
------------------------------------------------------------------------------
    svn:keywords = Id Revision Author Date


Reply via email to