Thanks Ian,
the point is that I keep the readers open to share them across search. Is
this wrong?
On Fri, May 18, 2012 at 9:58 AM, Ian Lea <ian....@gmail.com> wrote:
> You may need to cut it down to something simpler, but I can't see any
> reader.close() calls.
>
>
> --
> Ian.
>
>
> On Fri, May 18, 2012 at 5:47 PM, Michel Blase <mblas...@gmail.com> wrote:
> > This is the code in charge of managing the Lucene index. Thanks for your
> > help!
> >
> >
> >
> > package luz.aurora.lucene;
> >
> > import java.io.File;
> > import java.io.IOException;
> > import java.util.*;
> > import luz.aurora.search.ExtendedQueryParser;
> > import org.apache.lucene.analysis.Analyzer;
> > import org.apache.lucene.analysis.standard.StandardAnalyzer;
> > import org.apache.lucene.document.Document;
> > import org.apache.lucene.index.*;
> > import org.apache.lucene.queryParser.ParseException;
> > import org.apache.lucene.search.IndexSearcher;
> > import org.apache.lucene.search.Query;
> > import org.apache.lucene.search.TopDocs;
> > import org.apache.lucene.search.highlight.Highlighter;
> > import org.apache.lucene.search.highlight.QueryScorer;
> > import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
> > import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
> > import org.apache.lucene.store.Directory;
> > import org.apache.lucene.store.FSDirectory;
> >
> >
> > public class LuceneManager {
> >
> > private HashMap<Integer,String> IndexesPaths;
> > private HashMap<Integer,IndexWriter> Writers;
> >
> > private int CurrentOpenIndex_ID;
> > private String CurrentOpenIndex_TablePrefix;
> >
> > public LuceneManager(int CurrentOpenIndex_ID,String
> > CurrentOpenIndex_TablePrefix, HashMap<Integer,String> IndexesPaths)
> throws
> > Exception {
> > this.CurrentOpenIndex_ID = CurrentOpenIndex_ID;
> > this.IndexesPaths = IndexesPaths;
> > this.Writers = new HashMap<Integer,IndexWriter>();
> > this.CurrentOpenIndex_TablePrefix = CurrentOpenIndex_TablePrefix;
> >
> > SetUpWriters();
> > }
> >
> > private void SetUpWriters() throws Exception {
> > Set set = IndexesPaths.entrySet();
> > Iterator i = set.iterator();
> >
> > while(i.hasNext()){
> > Map.Entry index = (Map.Entry)i.next();
> > int id = (Integer)index.getKey();
> > String path = (String)index.getValue();
> >
> > File app = new File(path);
> > Directory dir = FSDirectory.open(app);
> > IndexWriterConfig config = new
> > IndexWriterConfig(LuceneVersion.CurrentVersion,new
> > StandardAnalyzer(LuceneVersion.CurrentVersion));
> >
> > //config.setMaxBufferedDocs(50);
> > config.setRAMBufferSizeMB(400);
> > TieredMergePolicy mp =
> > (TieredMergePolicy)config.getMergePolicy();
> > mp.setUseCompoundFile(true);
> > config.setMergePolicy(mp);
> >
> > /*
> > LogMergePolicy lmp = (LogMergePolicy)config.getMergePolicy();
> > lmp.setUseCompoundFile(true);
> > lmp.setMaxMergeDocs(10000);
> > config.setMergePolicy(lmp);
> > */
> >
> > Writers.put(id, new IndexWriter(dir,config));
> > }
> > }
> >
> > public void AddDocument(int IndexId,Document doc,Analyzer analyzer)
> > throws CorruptIndexException, IOException {
> > IndexWriter im = Writers.get(IndexId);
> > im.addDocument(doc, analyzer);
> > }
> >
> > public void AddDocument(Document doc,Analyzer analyzer) throws
> > CorruptIndexException, IOException {
> > IndexWriter im = Writers.get(this.CurrentOpenIndex_ID);
> > im.addDocument(doc, analyzer);
> > }
> >
> > public void DeleteDoc(int IndexId,int SegmentIdFromDb) throws
> > CorruptIndexException, IOException {
> > IndexWriter im = Writers.get(IndexId);
> > Term term = new
> Term("SegmentID",Integer.toString(SegmentIdFromDb));
> > im.deleteDocuments(term);
> > }
> >
> > public void DeleteDocuments(String query) throws ParseException,
> > CorruptIndexException, IOException {
> >
> > ExtendedQueryParser parser = new
> > ExtendedQueryParser(LuceneVersion.CurrentVersion,"ID",new
> > StandardAnalyzer(LuceneVersion.CurrentVersion));
> > Query q = parser.parse(query);
> >
> > Set set = Writers.entrySet();
> > Iterator i = set.iterator();
> >
> > while(i.hasNext()){
> > Map.Entry app = (Map.Entry)i.next();
> > IndexWriter im = (IndexWriter)app.getValue();
> > im.deleteDocuments(q);
> > }
> > }
> >
> > private IndexSearcher getSearcher() throws CorruptIndexException,
> > IOException {
> > int NumberOfIndexes = Writers.size();
> >
> > ArrayList<IndexReader> readers = new ArrayList<IndexReader>();
> > IndexReader[] readerList = new IndexReader[NumberOfIndexes];
> >
> > Set set = Writers.entrySet();
> > Iterator i = set.iterator();
> > while(i.hasNext()){
> > Map.Entry index = (Map.Entry)i.next();
> > IndexWriter iw = (IndexWriter)index.getValue();
> > readers.add(IndexReader.open(iw, true));
> > }
> >
> > MultiReader mr = new MultiReader(readers.toArray(readerList));
> > return new IndexSearcher(mr);
> > }
> >
> > public void close() throws CorruptIndexException, IOException {
> > Set set = Writers.entrySet();
> > Iterator i = set.iterator();
> > while(i.hasNext()){
> > Map.Entry index = (Map.Entry)i.next();
> > IndexWriter iw = (IndexWriter)index.getValue();
> > iw.close();
> > }
> > }
> >
> > public void commit() throws CorruptIndexException, IOException,
> > Exception {
> > Set set = Writers.entrySet();
> > Iterator i = set.iterator();
> > while(i.hasNext()){
> > Map.Entry index = (Map.Entry)i.next();
> > IndexWriter iw = (IndexWriter)index.getValue();
> > iw.commit();
> > }
> >
> > }
> >
> > public int getCurrentOpenIndexId() {
> > return this.CurrentOpenIndex_ID;
> > }
> >
> > public String getCurrentOpenIndexTablePrefix() {
> > return this.CurrentOpenIndex_TablePrefix;
> > }
> >
> >
> > //SEARCH START
> >
> > public TopDocs Search(String q,Analyzer analyzer,int NumberOfResults)
> > throws Exception {
> > ExtendedQueryParser parser = new
> > ExtendedQueryParser(LuceneVersion.CurrentVersion,"ID",analyzer);
> > Query query = parser.parse(q);
> > //Filter f = NumericRangeFilter.newIntRange("SegmentID", 393, 393,
> > true, true);
> > //FilteredQuery fq = new FilteredQuery(query,f);
> > //TopDocs docs = searcher.search(fq, NumberOfResults);
> > //System.out.println(searcher.getIndexReader().)
> > return getSearcher().search(query, NumberOfResults);
> > }
> >
> > public TopDocs SearchAndHighlight(String q,Analyzer analyzer,int
> > NumberOfResults) throws Exception {
> > ExtendedQueryParser parser = new
> > ExtendedQueryParser(LuceneVersion.CurrentVersion,"ID",analyzer);
> > Query query = parser.parse(q);
> > return getSearcher().search(query, NumberOfResults);
> > }
> >
> > public Highlighter getHighlighter(String query,Analyzer
> analyzer,String
> > OpeningTag,String ClosingTag) throws ParseException {
> > ExtendedQueryParser qp = new
> > ExtendedQueryParser(LuceneVersion.CurrentVersion,"ID",analyzer);
> > Query q = qp.parse(query);
> > QueryScorer qs = new QueryScorer(q);
> > SimpleHTMLFormatter formatter = new
> > SimpleHTMLFormatter(OpeningTag,ClosingTag);
> > Highlighter hl = new Highlighter(formatter,qs);
> > hl.setTextFragmenter(new SimpleSpanFragmenter(qs));
> > return hl;
> > }
> >
> > public Document getDoc(int i) throws CorruptIndexException,
> IOException
> > {
> > return getSearcher().doc(i);
> > }
> >
> >
> >
> > }
> >
> >
> >
> >
> >
> > On Thu, May 17, 2012 at 10:37 PM, findbestopensource <
> > findbestopensou...@gmail.com> wrote:
> >
> >> Post complete code. You are not closing the objects (IndexWriter / Index
> >> Searcher) properly.
> >>
> >> Regards
> >> Aditya
> >> www.findbestopensource.com
> >>
> >>
> >> On Fri, May 18, 2012 at 6:51 AM, Michel Blase <mblas...@gmail.com>
> wrote:
> >>
> >> > Hi all,
> >> >
> >> > I have few problems Indexing. I keep hitting "Too many open files". It
> >> > seems like Lucene is not releasing files handlers after deleting
> >> segments.
> >> >
> >> > This is a piece from the lsof output showing the problem:
> >> >
> >> >
> >> > java 23024 root *347r REG 251,0 2660 149376
> >> > /home/INDEXES_ROOT/SMPL_1/_bvq.cfs (deleted)
> >> > java 23024 root *348r REG 251,0 2477 149382
> >> > /home/INDEXES_ROOT/SMPL_1/_bvr.cfs (deleted)
> >> > java 23024 root *349r REG 251,0 2747 149392
> >> > /home/INDEXES_ROOT/SMPL_1/_bvu.cfs (deleted)
> >> > java 23024 root *350r REG 251,0 2339 149384
> >> > /home/INDEXES_ROOT/SMPL_1/_bvs.cfs (deleted)
> >> >
> >> > and this is the code I'm using (I'm using Lucene3.6)
> >> >
> >> > -- IndexWriter creation:
> >> >
> >> > File app = new File(path);
> >> > Directory dir = FSDirectory.open(app);
> >> > IndexWriterConfig config = new
> >> > IndexWriterConfig(LuceneVersion.CurrentVersion,new
> >> > StandardAnalyzer(LuceneVersion.CurrentVersion));
> >> >
> >> > //these are random tries attempting to solve the problem:
> >> > config.setRAMBufferSizeMB(400);
> >> > TieredMergePolicy mp = (TieredMergePolicy)config.getMergePolicy();
> >> > mp.setUseCompoundFile(true);
> >> > config.setMergePolicy(mp);
> >> > IndexWriter im = new IndexWriter(dir,config);
> >> >
> >> > -- Then just a loop over my doc list calling for indexing:
> >> > im.addDocument(doc, analyzer);
> >> >
> >> >
> >> > Any idea?
> >> > Thanks,
> >> > Luca
> >> >
> >>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscr...@lucene.apache.org
> For additional commands, e-mail: java-user-h...@lucene.apache.org
>
>
