perSegFilter
class:org.apache.solr.search.LRUCache
description:LRU Cache(maxSize=10, initialSize=0, autowarmCount=10,
regenerator=org.apache.solr.search.NoOpRegenerator@642f416a)
stats:
CACHE.searcher.perSegFilter.cumulative_evictions:0
CACHE.searcher.perSegFilter.cumulative_evictionsIdleTime:0
CACHE.searcher.perSegFilter.cumulative_evictionsRamUsage:0
CACHE.searcher.perSegFilter.cumulative_hitratio:0
CACHE.searcher.perSegFilter.cumulative_hits:0
CACHE.searcher.perSegFilter.cumulative_inserts:0
CACHE.searcher.perSegFilter.cumulative_lookups:0
CACHE.searcher.perSegFilter.evictions:0
CACHE.searcher.perSegFilter.evictionsIdleTime:0
CACHE.searcher.perSegFilter.evictionsRamUsage:0
CACHE.searcher.perSegFilter.hitratio:0
CACHE.searcher.perSegFilter.hits:0
CACHE.searcher.perSegFilter.inserts:0
CACHE.searcher.perSegFilter.lookups:0
CACHE.searcher.perSegFilter.maxIdleTime:-1
CACHE.searcher.perSegFilter.maxRamMB:-1
CACHE.searcher.perSegFilter.maxSize:10
CACHE.searcher.perSegFilter.ramBytesUsed:160
CACHE.searcher.perSegFilter.size:0
CACHE.searcher.perSegFilter.warmupTime:0
queryResultCache
class:org.apache.solr.search.LRUCache
description:LRU Cache(maxSize=512, initialSize=512)
stats:
CACHE.searcher.queryResultCache.cumulative_evictions:18328
CACHE.searcher.queryResultCache.cumulative_evictionsIdleTime:0
CACHE.searcher.queryResultCache.cumulative_evictionsRamUsage:0
CACHE.searcher.queryResultCache.cumulative_hitratio:0.52
CACHE.searcher.queryResultCache.cumulative_hits:41849
CACHE.searcher.queryResultCache.cumulative_inserts:38338
CACHE.searcher.queryResultCache.cumulative_lookups:80187
CACHE.searcher.queryResultCache.evictions:10172
CACHE.searcher.queryResultCache.evictionsIdleTime:0
CACHE.searcher.queryResultCache.evictionsRamUsage:0
CACHE.searcher.queryResultCache.hitratio:0.44
CACHE.searcher.queryResultCache.hits:8385
CACHE.searcher.queryResultCache.inserts:10684
CACHE.searcher.queryResultCache.lookups:19069
CACHE.searcher.queryResultCache.maxIdleTime:-1
CACHE.searcher.queryResultCache.maxRamMB:-1
CACHE.searcher.queryResultCache.maxSize:512
CACHE.searcher.queryResultCache.ramBytesUsed:2582208
CACHE.searcher.queryResultCache.size:512
CACHE.searcher.queryResultCache.warmupTime:0
fieldValueCache
class:org.apache.solr.search.FastLRUCache
description:Concurrent LRU Cache(maxSize=10000, initialSize=10,
minSize=9000, acceptableSize=9500, cleanupThread=false)
stats:
CACHE.searcher.fieldValueCache.cleanupThread:false
CACHE.searcher.fieldValueCache.cumulative_evictions:0
CACHE.searcher.fieldValueCache.cumulative_hitratio:0
CACHE.searcher.fieldValueCache.cumulative_hits:0
CACHE.searcher.fieldValueCache.cumulative_idleEvictions:0
CACHE.searcher.fieldValueCache.cumulative_inserts:0
CACHE.searcher.fieldValueCache.cumulative_lookups:0
CACHE.searcher.fieldValueCache.evictions:0
CACHE.searcher.fieldValueCache.hitratio:0
CACHE.searcher.fieldValueCache.hits:0
CACHE.searcher.fieldValueCache.idleEvictions:0
CACHE.searcher.fieldValueCache.inserts:0
CACHE.searcher.fieldValueCache.lookups:0
CACHE.searcher.fieldValueCache.maxRamMB:-1
CACHE.searcher.fieldValueCache.ramBytesUsed:1328
CACHE.searcher.fieldValueCache.size:0
CACHE.searcher.fieldValueCache.warmupTime:0
fieldCache
class:org.apache.solr.search.SolrFieldCacheBean
description:Provides introspection of the Solr FieldCache
stats:
CACHE.core.fieldCache.entries_count:13
CACHE.core.fieldCache.entry#0:segment='org.apache.lucene.index.IndexReader$CacheKey@​17c9d12b',
field='vmEnabled', size =~ 19 KB
CACHE.core.fieldCache.entry#1:segment='org.apache.lucene.index.IndexReader$CacheKey@​8c7641c',
field='vmEnabled', size =~ 27.9 KB
CACHE.core.fieldCache.entry#10:segment='org.apache.lucene.index.IndexReader$CacheKey@​4124d95b',
field='vmEnabled', size =~ 4 MB
CACHE.core.fieldCache.entry#11:segment='org.apache.lucene.index.IndexReader$CacheKey@​2b6b13f0',
field='vmEnabled', size =~ 2.3 MB
CACHE.core.fieldCache.entry#12:segment='org.apache.lucene.index.IndexReader$CacheKey@​68b638d3',
field='vmEnabled', size =~ 333.7 KB
CACHE.core.fieldCache.entry#2:segment='org.apache.lucene.index.IndexReader$CacheKey@​4ead0cea',
field='vmEnabled', size =~ 4.1 MB
CACHE.core.fieldCache.entry#3:segment='org.apache.lucene.index.IndexReader$CacheKey@​5a181271',
field='vmEnabled', size =~ 197.1 KB
CACHE.core.fieldCache.entry#4:segment='org.apache.lucene.index.IndexReader$CacheKey@​f0fad35',
field='vmEnabled', size =~ 31.4 KB
CACHE.core.fieldCache.entry#5:segment='org.apache.lucene.index.IndexReader$CacheKey@​237effff',
field='vmEnabled', size =~ 207 KB
CACHE.core.fieldCache.entry#6:segment='org.apache.lucene.index.IndexReader$CacheKey@​4a5ef757',
field='vmEnabled', size =~ 4 MB
CACHE.core.fieldCache.entry#7:segment='org.apache.lucene.index.IndexReader$CacheKey@​ed3d5a8',
field='vmEnabled', size =~ 3.9 MB
CACHE.core.fieldCache.entry#8:segment='org.apache.lucene.index.IndexReader$CacheKey@​42602d42',
field='vmEnabled', size =~ 241.7 KB
CACHE.core.fieldCache.entry#9:segment='org.apache.lucene.index.IndexReader$CacheKey@​19f94cee',
field='vmEnabled', size =~ 17.6 KB
CACHE.core.fieldCache.total_size:19.3 MB
filterCache
class:org.apache.solr.search.FastLRUCache
description:Concurrent LRU Cache(maxSize=512, initialSize=512, minSize=460,
acceptableSize=486, cleanupThread=false)
stats:
CACHE.searcher.filterCache.cleanupThread:false
CACHE.searcher.filterCache.cumulative_evictions:2217
CACHE.searcher.filterCache.cumulative_hitratio:0.95
CACHE.searcher.filterCache.cumulative_hits:1213572
CACHE.searcher.filterCache.cumulative_idleEvictions:0
CACHE.searcher.filterCache.cumulative_inserts:67924
CACHE.searcher.filterCache.cumulative_lookups:1279606
CACHE.searcher.filterCache.evictions:530
CACHE.searcher.filterCache.hitratio:1
CACHE.searcher.filterCache.hits:304411
CACHE.searcher.filterCache.idleEvictions:0
CACHE.searcher.filterCache.inserts:1409
CACHE.searcher.filterCache.lookups:305327
CACHE.searcher.filterCache.maxRamMB:-1
CACHE.searcher.filterCache.ramBytesUsed:1716745269
CACHE.searcher.filterCache.size:471
CACHE.searcher.filterCache.warmupTime:0
documentCache
class:org.apache.solr.search.LRUCache
description:LRU Cache(maxSize=512, initialSize=512)
stats:
CACHE.searcher.documentCache.cumulative_evictions:15097
CACHE.searcher.documentCache.cumulative_evictionsIdleTime:0
CACHE.searcher.documentCache.cumulative_evictionsRamUsage:0
CACHE.searcher.documentCache.cumulative_hitratio:0.45
CACHE.searcher.documentCache.cumulative_hits:20482
CACHE.searcher.documentCache.cumulative_inserts:25025
CACHE.searcher.documentCache.cumulative_lookups:45507
CACHE.searcher.documentCache.evictions:7651
CACHE.searcher.documentCache.evictionsIdleTime:0
CACHE.searcher.documentCache.evictionsRamUsage:0
CACHE.searcher.documentCache.hitratio:0.51
CACHE.searcher.documentCache.hits:8566
CACHE.searcher.documentCache.inserts:8164
CACHE.searcher.documentCache.lookups:16730
CACHE.searcher.documentCache.maxIdleTime:-1
CACHE.searcher.documentCache.maxRamMB:-1
CACHE.searcher.documentCache.maxSize:512
CACHE.searcher.documentCache.ramBytesUsed:1077408
CACHE.searcher.documentCache.size:512
CACHE.searcher.documentCache.warmupTime:0
On Fri, Mar 18, 2022 at 9:22 PM Vincenzo D'Amore <v.dam...@gmail.com> wrote:
> You were right, *:* first query rows=1 QTime=1947, second query rows=2
> QTime=0
>
> This is the CACHE, not sure how read this:
>
>
> - perSegFilter
> - class:org.apache.solr.search.LRUCache
> - description:LRU Cache(maxSize=10, initialSize=0,
> autowarmCount=10,
> regenerator=org.apache.solr.search.NoOpRegenerator@642f416a)
> - stats:
> - CACHE.searcher.perSegFilter.cumulative_evictions:0
> - CACHE.searcher.perSegFilter.cumulative_evictionsIdleTime:0
> - CACHE.searcher.perSegFilter.cumulative_evictionsRamUsage:0
> - CACHE.searcher.perSegFilter.cumulative_hitratio:0
> - CACHE.searcher.perSegFilter.cumulative_hits:0
> - CACHE.searcher.perSegFilter.cumulative_inserts:0
> - CACHE.searcher.perSegFilter.cumulative_lookups:0
> - CACHE.searcher.perSegFilter.evictions:0
> - CACHE.searcher.perSegFilter.evictionsIdleTime:0
> - CACHE.searcher.perSegFilter.evictionsRamUsage:0
> - CACHE.searcher.perSegFilter.hitratio:0
> - CACHE.searcher.perSegFilter.hits:0
> - CACHE.searcher.perSegFilter.inserts:0
> - CACHE.searcher.perSegFilter.lookups:0
> - CACHE.searcher.perSegFilter.maxIdleTime:-1
> - CACHE.searcher.perSegFilter.maxRamMB:-1
> - CACHE.searcher.perSegFilter.maxSize:10
> - CACHE.searcher.perSegFilter.ramBytesUsed:160
> - CACHE.searcher.perSegFilter.size:0
> - CACHE.searcher.perSegFilter.warmupTime:0
> - queryResultCache
> - class:org.apache.solr.search.LRUCache
> - description:LRU Cache(maxSize=512, initialSize=512)
> - stats:
> - CACHE.searcher.queryResultCache.cumulative_evictions:18328
> - CACHE.searcher.queryResultCache.cumulative_evictionsIdleTime:0
> - CACHE.searcher.queryResultCache.cumulative_evictionsRamUsage:0
> - CACHE.searcher.queryResultCache.cumulative_hitratio:0.52
> - CACHE.searcher.queryResultCache.cumulative_hits:41849
> - CACHE.searcher.queryResultCache.cumulative_inserts:38338
> - CACHE.searcher.queryResultCache.cumulative_lookups:80187
> - CACHE.searcher.queryResultCache.evictions:10172
> - CACHE.searcher.queryResultCache.evictionsIdleTime:0
> - CACHE.searcher.queryResultCache.evictionsRamUsage:0
> - CACHE.searcher.queryResultCache.hitratio:0.44
> - CACHE.searcher.queryResultCache.hits:8385
> - CACHE.searcher.queryResultCache.inserts:10684
> - CACHE.searcher.queryResultCache.lookups:19069
> - CACHE.searcher.queryResultCache.maxIdleTime:-1
> - CACHE.searcher.queryResultCache.maxRamMB:-1
> - CACHE.searcher.queryResultCache.maxSize:512
> - CACHE.searcher.queryResultCache.ramBytesUsed:2582208
> - CACHE.searcher.queryResultCache.size:512
> - CACHE.searcher.queryResultCache.warmupTime:0
> - fieldValueCache
> - class:org.apache.solr.search.FastLRUCache
> - description:Concurrent LRU Cache(maxSize=10000, initialSize=10,
> minSize=9000, acceptableSize=9500, cleanupThread=false)
> - stats:
> - CACHE.searcher.fieldValueCache.cleanupThread:false
> - CACHE.searcher.fieldValueCache.cumulative_evictions:0
> - CACHE.searcher.fieldValueCache.cumulative_hitratio:0
> - CACHE.searcher.fieldValueCache.cumulative_hits:0
> - CACHE.searcher.fieldValueCache.cumulative_idleEvictions:0
> - CACHE.searcher.fieldValueCache.cumulative_inserts:0
> - CACHE.searcher.fieldValueCache.cumulative_lookups:0
> - CACHE.searcher.fieldValueCache.evictions:0
> - CACHE.searcher.fieldValueCache.hitratio:0
> - CACHE.searcher.fieldValueCache.hits:0
> - CACHE.searcher.fieldValueCache.idleEvictions:0
> - CACHE.searcher.fieldValueCache.inserts:0
> - CACHE.searcher.fieldValueCache.lookups:0
> - CACHE.searcher.fieldValueCache.maxRamMB:-1
> - CACHE.searcher.fieldValueCache.ramBytesUsed:1328
> - CACHE.searcher.fieldValueCache.size:0
> - CACHE.searcher.fieldValueCache.warmupTime:0
> - fieldCache
> - class:org.apache.solr.search.SolrFieldCacheBean
> - description:Provides introspection of the Solr FieldCache
> - stats:
> - CACHE.core.fieldCache.entries_count:13
> - CACHE.core.fieldCache.entry#0:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​17c9d12b',
> field='vmEnabled', size =~ 19 KB
> - CACHE.core.fieldCache.entry#1:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​8c7641c',
> field='vmEnabled', size =~ 27.9 KB
> - CACHE.core.fieldCache.entry#10:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​4124d95b',
> field='vmEnabled', size =~ 4 MB
> - CACHE.core.fieldCache.entry#11:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​2b6b13f0',
> field='vmEnabled', size =~ 2.3 MB
> - CACHE.core.fieldCache.entry#12:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​68b638d3',
> field='vmEnabled', size =~ 333.7 KB
> - CACHE.core.fieldCache.entry#2:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​4ead0cea',
> field='vmEnabled', size =~ 4.1 MB
> - CACHE.core.fieldCache.entry#3:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​5a181271',
> field='vmEnabled', size =~ 197.1 KB
> - CACHE.core.fieldCache.entry#4:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​f0fad35',
> field='vmEnabled', size =~ 31.4 KB
> - CACHE.core.fieldCache.entry#5:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​237effff',
> field='vmEnabled', size =~ 207 KB
> - CACHE.core.fieldCache.entry#6:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​4a5ef757',
> field='vmEnabled', size =~ 4 MB
> - CACHE.core.fieldCache.entry#7:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​ed3d5a8',
> field='vmEnabled', size =~ 3.9 MB
> - CACHE.core.fieldCache.entry#8:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​42602d42',
> field='vmEnabled', size =~ 241.7 KB
> - CACHE.core.fieldCache.entry#9:
>
> segment='org.apache.lucene.index.IndexReader$CacheKey@​19f94cee',
> field='vmEnabled', size =~ 17.6 KB
> - CACHE.core.fieldCache.total_size:19.3 MB
> - filterCache
> - class:org.apache.solr.search.FastLRUCache
> - description:Concurrent LRU Cache(maxSize=512, initialSize=512,
> minSize=460, acceptableSize=486, cleanupThread=false)
> - stats:
> - CACHE.searcher.filterCache.cleanupThread:false
> - CACHE.searcher.filterCache.cumulative_evictions:2217
> - CACHE.searcher.filterCache.cumulative_hitratio:0.95
> - CACHE.searcher.filterCache.cumulative_hits:1213572
> - CACHE.searcher.filterCache.cumulative_idleEvictions:0
> - CACHE.searcher.filterCache.cumulative_inserts:67924
> - CACHE.searcher.filterCache.cumulative_lookups:1279606
> - CACHE.searcher.filterCache.evictions:530
> - CACHE.searcher.filterCache.hitratio:1
> - CACHE.searcher.filterCache.hits:304411
> - CACHE.searcher.filterCache.idleEvictions:0
> - CACHE.searcher.filterCache.inserts:1409
> - CACHE.searcher.filterCache.lookups:305327
> - CACHE.searcher.filterCache.maxRamMB:-1
> - CACHE.searcher.filterCache.ramBytesUsed:1716745269
> - CACHE.searcher.filterCache.size:471
> - CACHE.searcher.filterCache.warmupTime:0
> - documentCache
> - class:org.apache.solr.search.LRUCache
> - description:LRU Cache(maxSize=512, initialSize=512)
> - stats:
> - CACHE.searcher.documentCache.cumulative_evictions:15097
> - CACHE.searcher.documentCache.cumulative_evictionsIdleTime:0
> - CACHE.searcher.documentCache.cumulative_evictionsRamUsage:0
> - CACHE.searcher.documentCache.cumulative_hitratio:0.45
> - CACHE.searcher.documentCache.cumulative_hits:20482
> - CACHE.searcher.documentCache.cumulative_inserts:25025
> - CACHE.searcher.documentCache.cumulative_lookups:45507
> - CACHE.searcher.documentCache.evictions:7651
> - CACHE.searcher.documentCache.evictionsIdleTime:0
> - CACHE.searcher.documentCache.evictionsRamUsage:0
> - CACHE.searcher.documentCache.hitratio:0.51
> - CACHE.searcher.documentCache.hits:8566
> - CACHE.searcher.documentCache.inserts:8164
> - CACHE.searcher.documentCache.lookups:16730
> - CACHE.searcher.documentCache.maxIdleTime:-1
> - CACHE.searcher.documentCache.maxRamMB:-1
> - CACHE.searcher.documentCache.maxSize:512
> - CACHE.searcher.documentCache.ramBytesUsed:1077408
> - CACHE.searcher.documentCache.size:512
> - CACHE.searcher.documentCache.warmupTime:0
>
>
> On Fri, Mar 18, 2022 at 8:52 PM matthew sporleder <msporle...@gmail.com>
> wrote:
>
>> My guess is that it's trashing on a "cold" open of the index file. I'm
>> sure the next query of *:*&rows=2 is pretty fast since caches get
>> populated.
>>
>> I don't know what to say for next steps - lower the jvm memory and/or
>> check
>> the stats in the admin console -> core selct -> Plugins/Stats -> CACHE.
>>
>> What are the storage speeds? IMHO you are disk bound.
>>
>>
>> On Fri, Mar 18, 2022 at 3:42 PM Vincenzo D'Amore <v.dam...@gmail.com>
>> wrote:
>>
>> > Is it possible that there are too frequent commits? I mean if each
>> commit
>> > usually invalidates the cache, even the a stupid *:* rows=1 can be
>> > affected.
>> > How can I see how frequent commits are? Or when the latest commit has
>> been
>> > done?
>> >
>> > On Fri, Mar 18, 2022 at 8:36 PM Vincenzo D'Amore <v.dam...@gmail.com>
>> > wrote:
>> >
>> > > Ok, everything you said is right, but nevertheless even right now a
>> > stupid
>> > > *:* rows=1 runs in almost 2 seconds.
>> > > The average document size is pretty small, less than roughly 100/200
>> > > bytes.
>> > > Does someone know if the average doc size is available in the metrics?
>> > >
>> > > {
>> > > "responseHeader":{
>> > > "zkConnected":true,
>> > > "status":0,
>> > > "QTime":2033,
>> > > "params":{
>> > > "q":"*:*",
>> > > "rows":"1"}},
>> > >
>> > > On Fri, Mar 18, 2022 at 7:50 PM matthew sporleder <
>> msporle...@gmail.com>
>> > > wrote:
>> > >
>> > >> You are getting this general advice but, sadly, it depends on your
>> doc
>> > >> sizes, query complexity, write frequency, and a bunch of other stuff
>> I
>> > >> don't know about.
>> > >>
>> > >> I prefer to run with the *minimum* JVM memory to handle throughput
>> > >> (without
>> > >> OOM) and let the OS do caching because I update/write to the index
>> every
>> > >> few minutes making my *solr* caching pretty worthless.
>> > >>
>> > >> tuning solr also includes tuning queries. Start with timing id:123
>> type
>> > >> K:V lookups and work your complexity up from there. Use debug=true
>> and
>> > >> attempt to read it.
>> > >>
>> > >> There are many many knobs. You need to set a baseline, then a
>> target,
>> > >> then
>> > >> a thesis on how to get there.
>> > >>
>> > >>
>> > >>
>> > >> On Fri, Mar 18, 2022 at 2:36 PM Vincenzo D'Amore <v.dam...@gmail.com
>> >
>> > >> wrote:
>> > >>
>> > >> > We have modified the kubernetes configuration and restarted
>> SolrCloud
>> > >> > cluster, now we have 16 cores per Solr instance.
>> > >> > The performance does not seem to be improved though.
>> > >> > The load average is 0.43 0.83 1.00, to me it seems an IO bound
>> > problem.
>> > >> > Looking at the index I see 162M documents, 234M maxDocs, 71M
>> > deleted...
>> > >> > maybe this core needs to be optimized.
>> > >> > The INDEX.size is 70GB, what do you think if I raise the size
>> > allocated
>> > >> > from the JVM to 64GB in order to have the index in memory?
>> > >> > At last, I'm looking at Solr metric but really not sure how to
>> > >> understand
>> > >> > if it is CPU bound or IO bound.
>> > >> >
>> > >> > On Fri, Mar 18, 2022 at 6:34 PM Walter Underwood <
>> > wun...@wunderwood.org
>> > >> >
>> > >> > wrote:
>> > >> >
>> > >> > > First look at the system metrics. Is it CPU bound or IO bound?
>> Each
>> > >> > > request is single threaded, so a CPU bound system will have one
>> core
>> > >> used
>> > >> > > at roughly 100% for that time. An IO bound system will not be
>> using
>> > >> much
>> > >> > > CPU but will have threads in iowait and lots of disk reads.
>> > >> > >
>> > >> > > After you know that, then you know what to work on. If it is IO
>> > bound,
>> > >> > get
>> > >> > > enough RAM for the OS, JVM, and index files to all be in memory.
>> If
>> > >> it is
>> > >> > > CPU bound, get a faster processor and work on the config to have
>> the
>> > >> > > request do less work. Sharding can also help.
>> > >> > >
>> > >> > > I’m not a fan of always choosing 31 GB for the JVM. Allocate
>> only as
>> > >> much
>> > >> > > as is needed. Java will use the whole heap whether it is needed
>> or
>> > >> not.
>> > >> > You
>> > >> > > might only need 8 GB. All of our clusters run with 16 GB. That
>> > >> includes
>> > >> > > some machines with 36 cores.
>> > >> > >
>> > >> > >
>> > >> > --
>> > >> > Vincenzo D'Amore
>> > >> >
>> > >>
>> > >
>> > >
>> > > --
>> > > Vincenzo D'Amore
>> > >
>> > >
>> >
>> > --
>> > Vincenzo D'Amore
>> >
>>
>
>
> --
> Vincenzo D'Amore
>
>
--
Vincenzo D'Amore
