Thanks for the patch. It helped in increasing the
search speed to a good extent. But when we tried to
give about 100 queries in 10 seconds, then again we
found that after about 15 seconds, the response time
per query increased. Enclosed is the dump which we
took after about 30 seconds of starting the search.
The maximum query time has reduced from 200-300
seconds to about 50 seconds.
We were able to simplify the searches further by
consolidating the fields in the index but that
resulted in increasing the index size to 2.5 GB as we
required fields 2-5 and fields 1-7 in different
searches. Our indexes are on the local disk therefor
there is no network i/o involved.
Thanks
Jayant
--- Doug Cutting <[EMAIL PROTECTED]> wrote: > Doug
Cutting wrote:
> > Please tell me if you are able to simplify your
> queries and if that
> > speeds things. I'll look into a ThreadLocal-based
> solution too.
>
> I've attached a patch that should help with the
> thread contention,
> although I've not tested it extensively.
>
> I still don't fully understand why your searches are
> so slow, though.
> Are the indexes stored on the local disk of the
> machine? Indexes
> accessed over the network can be very slow.
>
> Anyway, give this patch a try. Also, if anyone else
> can try this and
> report back whether it makes multi-threaded
> searching faster, or
> anything else slower, or is buggy, that would be
> great.
>
> Thanks,
>
> Doug
> > Index:
>
src/java/org/apache/lucene/index/TermInfosReader.java
>
===================================================================
> RCS file:
>
/home/cvs/jakarta-lucene/src/java/org/apache/lucene/index/TermInfosReader.java,v
> retrieving revision 1.6
> diff -u -u -r1.6 TermInfosReader.java
> ---
>
src/java/org/apache/lucene/index/TermInfosReader.java
> 20 May 2004 11:23:53 -0000 1.6
> +++
>
src/java/org/apache/lucene/index/TermInfosReader.java
> 4 Jun 2004 21:45:15 -0000
> @@ -29,7 +29,8 @@
> private String segment;
> private FieldInfos fieldInfos;
>
> - private SegmentTermEnum enumerator;
> + private ThreadLocal enumerators = new
> ThreadLocal();
> + private SegmentTermEnum origEnum;
> private long size;
>
> TermInfosReader(Directory dir, String seg,
> FieldInfos fis)
> @@ -38,19 +39,19 @@
> segment = seg;
> fieldInfos = fis;
>
> - enumerator = new
> SegmentTermEnum(directory.openFile(segment +
> ".tis"),
> - fieldInfos, false);
> - size = enumerator.size;
> + origEnum = new
> SegmentTermEnum(directory.openFile(segment +
> ".tis"),
> + fieldInfos,
> false);
> + size = origEnum.size;
> readIndex();
> }
>
> public int getSkipInterval() {
> - return enumerator.skipInterval;
> + return origEnum.skipInterval;
> }
>
> final void close() throws IOException {
> - if (enumerator != null)
> - enumerator.close();
> + if (origEnum != null)
> + origEnum.close();
> }
>
> /** Returns the number of term/value pairs in the
> set. */
> @@ -58,6 +59,15 @@
> return size;
> }
>
> + private SegmentTermEnum getEnum() {
> + SegmentTermEnum enum =
> (SegmentTermEnum)enumerators.get();
> + if (enum == null) {
> + enum = terms();
> + enumerators.set(enum);
> + }
> + return enum;
> + }
> +
> Term[] indexTerms = null;
> TermInfo[] indexInfos;
> long[] indexPointers;
> @@ -102,16 +112,17 @@
> }
>
> private final void seekEnum(int indexOffset)
> throws IOException {
> - enumerator.seek(indexPointers[indexOffset],
> - (indexOffset * enumerator.indexInterval) -
> 1,
> + getEnum().seek(indexPointers[indexOffset],
> + (indexOffset * getEnum().indexInterval) - 1,
> indexTerms[indexOffset],
> indexInfos[indexOffset]);
> }
>
> /** Returns the TermInfo for a Term in the set,
> or null. */
> - final synchronized TermInfo get(Term term) throws
> IOException {
> + TermInfo get(Term term) throws IOException {
> if (size == 0) return null;
>
> - // optimize sequential access: first try
> scanning cached enumerator w/o seeking
> + // optimize sequential access: first try
> scanning cached enum w/o seeking
> + SegmentTermEnum enumerator = getEnum();
> if (enumerator.term() != null
> // term is at or past current
> && ((enumerator.prev != null &&
> term.compareTo(enumerator.prev) > 0)
> || term.compareTo(enumerator.term()) >= 0)) {
> @@ -128,6 +139,7 @@
>
> /** Scans within block for matching term. */
> private final TermInfo scanEnum(Term term) throws
> IOException {
> + SegmentTermEnum enumerator = getEnum();
> while (term.compareTo(enumerator.term()) > 0 &&
> enumerator.next()) {}
> if (enumerator.term() != null &&
> term.compareTo(enumerator.term()) == 0)
> return enumerator.termInfo();
> @@ -136,10 +148,12 @@
> }
>
> /** Returns the nth term in the set. */
> - final synchronized Term get(int position) throws
> IOException {
> + final Term get(int position) throws IOException {
> if (size == 0) return null;
>
> - if (enumerator != null && enumerator.term() !=
> null && position >= enumerator.position &&
> + SegmentTermEnum enumerator = getEnum();
> + if (enumerator != null && enumerator.term() !=
> null &&
> + position >= enumerator.position &&
> position < (enumerator.position +
> enumerator.indexInterval))
> return scanEnum(position); // can avoid
> seek
>
> @@ -148,6 +162,7 @@
> }
>
> private final Term scanEnum(int position) throws
> IOException {
> + SegmentTermEnum enumerator = getEnum();
> while(enumerator.position < position)
> if (!enumerator.next())
> return null;
> @@ -156,12 +171,13 @@
> }
>
> /** Returns the position of a Term in the set or
> -1. */
> - final synchronized long getPosition(Term term)
> throws IOException {
> + final long getPosition(Term term) throws
> IOException {
> if (size == 0) return -1;
>
> int indexOffset = getIndexOffset(term);
> seekEnum(indexOffset);
>
> + SegmentTermEnum enumerator = getEnum();
> while(term.compareTo(enumerator.term()) > 0 &&
> enumerator.next()) {}
>
> if (term.compareTo(enumerator.term()) == 0)
> @@ -171,17 +187,13 @@
> }
>
> /** Returns an enumeration of all the Terms and
> TermInfos in the set. */
> - final synchronized SegmentTermEnum terms() throws
> IOException {
> - if (enumerator.position != -1) // if not at
> start
> - seekEnum(0); // reset to start
> - return (SegmentTermEnum)enumerator.clone();
> + public SegmentTermEnum terms() {
> + return (SegmentTermEnum)origEnum.clone();
> }
>
> /** Returns an enumeration of terms starting at
> or after the named term. */
> - final synchronized SegmentTermEnum terms(Term
> term) throws IOException {
> - get(term); // seek enumerator to term
> - return (SegmentTermEnum)enumerator.clone();
> + public SegmentTermEnum terms(Term term) throws
> IOException {
> + get(term);
> + return (SegmentTermEnum)getEnum().clone();
> }
> -
> -
> }
>
> >
---------------------------------------------------------------------
> To unsubscribe, e-mail:
> [EMAIL PROTECTED]
> For additional commands, e-mail:
[EMAIL PROTECTED]
________________________________________________________________________
Yahoo! India Matrimony: Find your partner online.
http://yahoo.shaadi.com/india-matrimony/Thread-24" prio=1 tid=0x0816a390 nid=0x2868 waiting for monitor entry
[69c55000..69c558c8]
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:217)
- waiting to lock <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:83)
at org.apache.lucene.index.SegmentTermDocs.next(SegmentTermDocs.java:102)
at
org.apache.lucene.index.SegmentTermPositions.next(SegmentTermPositions.java:60)
at org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:195)
at org.apache.lucene.search.PhrasePositions.skipTo(PhrasePositions.java:47)
at org.apache.lucene.search.PhraseScorer.doNext(PhraseScorer.java:72)
at org.apache.lucene.search.PhraseScorer.next(PhraseScorer.java:65)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.Scorer.score(Scorer.java:37)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:121)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"Thread-23" prio=1 tid=0x08169f38 nid=0x2867 waiting for monitor entry
[69bd4000..69bd48c8]
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:217)
- waiting to lock <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:83)
at
org.apache.lucene.index.SegmentTermPositions.next(SegmentTermPositions.java:58)
at org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:195)
at org.apache.lucene.search.PhrasePositions.skipTo(PhrasePositions.java:47)
at org.apache.lucene.search.PhraseScorer.doNext(PhraseScorer.java:72)
at org.apache.lucene.search.PhraseScorer.next(PhraseScorer.java:65)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.Scorer.score(Scorer.java:37)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:121)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"Thread-22" prio=1 tid=0x08159f78 nid=0x2866 waiting for monitor entry
[69b53000..69b538c8]
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:217)
- waiting to lock <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:86)
at org.apache.lucene.index.SegmentTermDocs.read(SegmentTermDocs.java:126)
at org.apache.lucene.search.TermScorer.next(TermScorer.java:55)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.Scorer.score(Scorer.java:37)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:121)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"Thread-21" prio=1 tid=0x08169bd8 nid=0x2865 waiting for monitor entry
[69ad2000..69ad28c8]
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:217)
- waiting to lock <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:86)
at
org.apache.lucene.index.SegmentTermPositions.next(SegmentTermPositions.java:58)
at org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:195)
at org.apache.lucene.search.PhrasePositions.skipTo(PhrasePositions.java:47)
at org.apache.lucene.search.PhraseScorer.doNext(PhraseScorer.java:72)
at org.apache.lucene.search.PhraseScorer.next(PhraseScorer.java:65)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.Scorer.score(Scorer.java:37)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:121)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"Thread-19" prio=1 tid=0x081695f0 nid=0x2863 waiting for monitor entry
[698f8000..698f88c8]
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:217)
- waiting to lock <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:83)
at
org.apache.lucene.index.SegmentTermPositions.nextPosition(SegmentTermPositions.java:48)
at
org.apache.lucene.search.PhrasePositions.nextPosition(PhrasePositions.java:65)
at
org.apache.lucene.search.PhrasePositions.firstPosition(PhrasePositions.java:60)
at
org.apache.lucene.search.ExactPhraseScorer.phraseFreq(ExactPhraseScorer.java:32)
at org.apache.lucene.search.PhraseScorer.doNext(PhraseScorer.java:78)
at org.apache.lucene.search.PhraseScorer.skipTo(PhraseScorer.java:100)
at org.apache.lucene.search.ConjunctionScorer.doNext(ConjunctionScorer.java:53)
at org.apache.lucene.search.ConjunctionScorer.next(ConjunctionScorer.java:48)
at org.apache.lucene.search.Scorer.score(Scorer.java:37)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:121)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"Thread-18" prio=1 tid=0x0815ae40 nid=0x2862 waiting for monitor entry
[69877000..698778c8]
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:217)
- waiting to lock <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:83)
at org.apache.lucene.index.SegmentTermDocs.read(SegmentTermDocs.java:126)
at org.apache.lucene.search.TermScorer.next(TermScorer.java:55)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.Scorer.score(Scorer.java:37)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:121)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"Thread-8" prio=1 tid=0x080c13c8 nid=0x2858 waiting for monitor entry
[694f0000..694f08c8]
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:217)
- waiting to lock <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:83)
at org.apache.lucene.index.SegmentTermDocs.read(SegmentTermDocs.java:131)
at org.apache.lucene.search.TermScorer.next(TermScorer.java:55)
at org.apache.lucene.search.ConjunctionScorer.next(ConjunctionScorer.java:46)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at
org.apache.lucene.search.BooleanScorer$SubScorer.<init>(BooleanScorer.java:48)
at org.apache.lucene.search.BooleanScorer.add(BooleanScorer.java:76)
at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:166)
at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:164)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:113)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"Thread-2" prio=1 tid=0x080f93e8 nid=0x2852 runnable [69112000..691128c8]
at java.io.RandomAccessFile.readBytes(Native Method)
at java.io.RandomAccessFile.read(RandomAccessFile.java:307)
at org.apache.lucene.store.FSInputStream.readInternal(FSDirectory.java:410)
- locked <0x46f1be28> (a org.apache.lucene.store.FSInputStream$Descriptor)
at org.apache.lucene.store.InputStream.readBytes(InputStream.java:61)
at
org.apache.lucene.index.CompoundFileReader$CSInputStream.readInternal(CompoundFileReader.java:222)
- locked <0x46f1b828> (a org.apache.lucene.store.FSInputStream)
at org.apache.lucene.store.InputStream.refill(InputStream.java:158)
at org.apache.lucene.store.InputStream.readByte(InputStream.java:43)
at org.apache.lucene.store.InputStream.readVInt(InputStream.java:83)
at org.apache.lucene.index.SegmentTermDocs.read(SegmentTermDocs.java:126)
at org.apache.lucene.search.TermScorer.next(TermScorer.java:55)
at org.apache.lucene.search.BooleanScorer.next(BooleanScorer.java:112)
at org.apache.lucene.search.Scorer.score(Scorer.java:37)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:121)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:64)
at org.apache.lucene.search.Hits.<init>(Hits.java:51)
at org.apache.lucene.search.Searcher.search(Searcher.java:41)
at resdex.searchinc.getHits(searchinc.java:744)
at resdex.searchinc.Search(searchinc.java:937)
at resdex.searchinctest.conductTestSearch(searchinctest.java:99)
at resdex.Server$Handler.run(Server.java:62)
at java.lang.Thread.run(Thread.java:534)
"DestroyJavaVM" prio=1 tid=0x68f02ae0 nid=0x2842 waiting on condition [0..bfffd5c4]
"Thread-0" prio=1 tid=0x68f00658 nid=0x284b runnable [68ef2000..68ef28c8]
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:353)
- locked <0x47663a28> (a java.net.PlainSocketImpl)
at java.net.ServerSocket.implAccept(ServerSocket.java:448)
at java.net.ServerSocket.accept(ServerSocket.java:419)
at resdex.Server.run(Server.java:93)
at java.lang.Thread.run(Thread.java:534)
"Signal Dispatcher" daemon prio=1 tid=0x0808faa0 nid=0x2848 waiting on condition [0..0]
"Finalizer" daemon prio=1 tid=0x0808a5b8 nid=0x2846 in Object.wait()
[68ad1000..68ad18c8]
at java.lang.Object.wait(Native Method)
- waiting on <0x46e89e40> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:111)
- locked <0x46e89e40> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:127)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
"Reference Handler" daemon prio=1 tid=0x08089940 nid=0x2845 in Object.wait()
[68a50000..68a508c8]
at java.lang.Object.wait(Native Method)
- waiting on <0x46e89ea8> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:429)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:115)
- locked <0x46e89ea8> (a java.lang.ref.Reference$Lock)
"VM Thread" prio=1 tid=0x08086730 nid=0x2844 runnable
"VM Periodic Task Thread" prio=1 tid=0x080ab790 nid=0x284a waiting on condition
"Suspend Checker Thread" prio=1 tid=0x0808f068 nid=0x2847 runnable
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
