[JENKINS] Lucene-Solr-7.x-Windows (64bit/jdk-9.0.4) - Build # 697 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Windows/697/
Java: 64bit/jdk-9.0.4 -XX:-UseCompressedOops -XX:+UseParallelGC

65 tests failed.
FAILED:  org.apache.lucene.replicator.nrt.TestNRTReplication.testCrashReplica

Error Message:
Address already in use: connect

Stack Trace:
java.net.BindException: Address already in use: connect
at 
__randomizedtesting.SeedInfo.seed([6922E3245CB14148:2DCDD060D30C8EDB]:0)
at java.base/java.net.DualStackPlainSocketImpl.connect0(Native Method)
at 
java.base/java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79)
at 
java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:400)
at 
java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:243)
at 
java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:225)
at java.base/java.net.PlainSocketImpl.connect(PlainSocketImpl.java:148)
at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:402)
at java.base/java.net.Socket.connect(Socket.java:591)
at java.base/java.net.Socket.connect(Socket.java:540)
at java.base/java.net.Socket.(Socket.java:436)
at java.base/java.net.Socket.(Socket.java:246)
at 
org.apache.lucene.replicator.nrt.Connection.(Connection.java:43)
at 
org.apache.lucene.replicator.nrt.TestNRTReplication.testCrashReplica(TestNRTReplication.java:745)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 103 - Still Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/103/

5 tests failed.
FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
Error from server at http://127.0.0.1:38736/g_pvi/y: At least one of the 
node(s) specified [127.0.0.1:38736_g_pvi%2Fy] are not currently active in [], 
no action taken.

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at http://127.0.0.1:38736/g_pvi/y: At least one of the node(s) 
specified [127.0.0.1:38736_g_pvi%2Fy] are not currently active in [], no action 
taken.
at 
__randomizedtesting.SeedInfo.seed([4112D9BCB6B83DAC:C946E66618445054]:0)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:425)
at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:341)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1006)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 

[JENKINS] Lucene-Solr-BadApples-master-Linux (64bit/jdk-10) - Build # 66 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-BadApples-master-Linux/66/
Java: 64bit/jdk-10 -XX:+UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
We think that split was successful but sub-shard states were not updated even 
after 2 minutes.

Stack Trace:
java.lang.AssertionError: We think that split was successful but sub-shard 
states were not updated even after 2 minutes.
at 
__randomizedtesting.SeedInfo.seed([712A285BA6AE8802:FA0DFB8AE7A82386]:0)
at org.junit.Assert.fail(Assert.java:93)
at 
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:555)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 

[JENKINS] Lucene-Solr-master-Windows (64bit/jdk1.8.0_172) - Build # 7424 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Windows/7424/
Java: 64bit/jdk1.8.0_172 -XX:-UseCompressedOops -XX:+UseSerialGC

55 tests failed.
FAILED:  
org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest.missingDefaultTest

Error Message:
{responseHeader={zkConnected=true,status=0,QTime=25,params={o.misr.s.long_ld=missing(long_ld),_stateVer_=collection1:4,indent=true,o.misr.s.date_dtd=missing(date_dtd),rows=0,o.misr.s.int_id=missing(int_id),version=2,q=*:*,o.misr.s.double_dd=missing(double_dd),olap=true,o.misr.s.float_fd=missing(float_fd),o.misr.s.string_sd=missing(string_sd),wt=javabin}},response={numFound=100,start=0,maxScore=1.0,docs=[]},stats={misr={long_ld=3,date_dtd=9,string_sd=4,int_id=2,double_dd=3,float_fd=2}}}
 expected:<4> but was:<2>

Stack Trace:
java.lang.AssertionError: 
{responseHeader={zkConnected=true,status=0,QTime=25,params={o.misr.s.long_ld=missing(long_ld),_stateVer_=collection1:4,indent=true,o.misr.s.date_dtd=missing(date_dtd),rows=0,o.misr.s.int_id=missing(int_id),version=2,q=*:*,o.misr.s.double_dd=missing(double_dd),olap=true,o.misr.s.float_fd=missing(float_fd),o.misr.s.string_sd=missing(string_sd),wt=javabin}},response={numFound=100,start=0,maxScore=1.0,docs=[]},stats={misr={long_ld=3,date_dtd=9,string_sd=4,int_id=2,double_dd=3,float_fd=2}}}
 expected:<4> but was:<2>
at 
__randomizedtesting.SeedInfo.seed([F84F6F13419CCB51:FE2030FC575CDF5C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at 
org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest.missingDefaultTest(LegacyNoFacetCloudTest.java:528)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 

[jira] [Commented] (SOLR-12297) Add Http2SolrClient, capable of HTTP/1.1, HTTP/2, and asynchronous requests.

[Cao Manh Dat (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12297?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547351#comment-16547351
 ] 

Cao Manh Dat commented on SOLR-12297:
-

My idea for this ticket :
* Add Http2SolrClient.java
* A minimal test for Http2SolrClient based on BasicHttpSolrClient (this ensure 
the consistency in behaviors between Http2SolrClient and HttpSolrClient)
* Enable Http2 in JettySolrRunner for *test environment* only.

By doing that we can slowly introduce more supports on Http2 
* All the changes will be tested for a very long time before we switch from 
Http to Http2
* Won't impact the current final build/release. 

I attached a *nearly finished* patch for this ticket :
* Http2SolrClient's behaviors consistent with HttpSolrClient (include error 
handling, V2Request, multipart request, queryParams)
* Add Http2SolrClientTest which mimic BasicHttpSolrClientTest

I wonder that should I keep continue working on this issue, or create another 
one. Because my vision for this issue seems not match with others.

> Add Http2SolrClient, capable of HTTP/1.1, HTTP/2, and asynchronous requests.
> 
>
> Key: SOLR-12297
> URL: https://issues.apache.org/jira/browse/SOLR-12297
> Project: Solr
>  Issue Type: New Feature
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Mark Miller
>Assignee: Mark Miller
>Priority: Major
> Attachments: SOLR-12297.patch, SOLR-12297.patch, 
> starburst-ivy-fixes.patch
>
>
> Blocking or async support as well as HTTP2 compatible with multiplexing.
> Once it supports enough and is stable, replace internal usage, allowing 
> async, and eventually move to HTTP2 connector and allow multiplexing. Could 
> support HTTP1.1 and HTTP2 on different ports depending on state of the world 
> then.
> The goal of the client itself is to work against HTTP1.1 or HTTP2 with 
> minimal or no code path differences and the same for async requests (should 
> initially work for both 1.1 and 2 and share majority of code).
> The client should also be able to replace HttpSolrClient and plug into the 
> other clients the same way.
> I doubt it would make sense to keep ConcurrentUpdateSolrClient eventually 
> though.
> I evaluated some clients and while there are a few options, I went with 
> Jetty's HttpClient. It's more mature than Apache HttpClient's support (in 5 
> beta) and we would have to update to a new API for Apache HttpClient anyway.
> Meanwhile, the Jetty guys have been very supportive of helping Solr with any 
> issues and I like having the client and server from the same project.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-12297) Add Http2SolrClient, capable of HTTP/1.1, HTTP/2, and asynchronous requests.

[Cao Manh Dat (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12297?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Cao Manh Dat updated SOLR-12297:

Attachment: SOLR-12297.patch

> Add Http2SolrClient, capable of HTTP/1.1, HTTP/2, and asynchronous requests.
> 
>
> Key: SOLR-12297
> URL: https://issues.apache.org/jira/browse/SOLR-12297
> Project: Solr
>  Issue Type: New Feature
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Mark Miller
>Assignee: Mark Miller
>Priority: Major
> Attachments: SOLR-12297.patch, SOLR-12297.patch, 
> starburst-ivy-fixes.patch
>
>
> Blocking or async support as well as HTTP2 compatible with multiplexing.
> Once it supports enough and is stable, replace internal usage, allowing 
> async, and eventually move to HTTP2 connector and allow multiplexing. Could 
> support HTTP1.1 and HTTP2 on different ports depending on state of the world 
> then.
> The goal of the client itself is to work against HTTP1.1 or HTTP2 with 
> minimal or no code path differences and the same for async requests (should 
> initially work for both 1.1 and 2 and share majority of code).
> The client should also be able to replace HttpSolrClient and plug into the 
> other clients the same way.
> I doubt it would make sense to keep ConcurrentUpdateSolrClient eventually 
> though.
> I evaluated some clients and while there are a few options, I went with 
> Jetty's HttpClient. It's more mature than Apache HttpClient's support (in 5 
> beta) and we would have to update to a new API for Apache HttpClient anyway.
> Meanwhile, the Jetty guys have been very supportive of helping Solr with any 
> issues and I like having the client and server from the same project.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Varun Thacker (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547338#comment-16547338
 ] 

Varun Thacker commented on SOLR-11598:
--

Took another stab adding some more randomness to tests. I think I'll spend 
maybe another few hours seeing if 

testMultipleSorts can test more edge cases.

 

One change made to StringValue:

In setCurrentValue replaced this section from
{code:java}
int ord = currentVals.ordValue();
if(globalOrds != null) {
currentOrd = (int)globalOrds.get(ord);
} else {
currentOrd = ord;
}{code}
to 
{code:java}
protected LongValues toGlobal = LongValues.IDENTITY; // this segment to global 
ordinal. NN;
^^ ensuring that for non MultiDocValues.MultiSortedDocValues instances it 
simmply returns the ord itself

//and then we can just do

if (docId == docValues.docID()) {
currentOrd = (int) toGlobal.get(docValues.ordValue());
}{code}
Copied this technique from FacetFieldProcessorByHashDV. ExpandComponent also 
does the same thing roughly

> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
>   at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
>   at 
> 

[jira] [Updated] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Varun Thacker (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Varun Thacker updated SOLR-11598:
-
Attachment: SOLR-11598.patch

> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
>   at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>   at 
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
>   at 
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
>   at 
> 

[JENKINS] Lucene-Solr-master-Linux (32bit/jdk1.8.0_172) - Build # 22473 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/22473/
Java: 32bit/jdk1.8.0_172 -client -XX:+UseSerialGC

1 tests failed.
FAILED:  
org.apache.solr.cloud.LeaderElectionIntegrationTest.testSimpleSliceLeaderElection

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
at 
__randomizedtesting.SeedInfo.seed([4F0ECE95004820B:5AC421A7137B1EEC]:0)
at sun.nio.ch.Net.bind0(Native Method)
at sun.nio.ch.Net.bind(Net.java:433)
at sun.nio.ch.Net.bind(Net.java:425)
at 
sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
at 
org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:331)
at 
org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:299)
at 
org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
at 
org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:235)
at 
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.server.Server.doStart(Server.java:398)
at 
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)
at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)
at 
org.apache.solr.cloud.LeaderElectionIntegrationTest.testSimpleSliceLeaderElection(LeaderElectionIntegrationTest.java:107)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 

[jira] [Commented] (LUCENE-8409) Cannot delete individual fields from index

[Alexandre Rafalovitch (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8409?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547319#comment-16547319
 ] 

Alexandre Rafalovitch commented on LUCENE-8409:
---

I seem to remember the discussion about doing segment-by-segment rewrite with 
some sort of filtered reader that would clear the remove fields in process. But 
I cannot find the discussion of whether it was fully relevant.

But yes I agree that it is quite problematic in terms of index evolution.

> Cannot delete individual fields from index
> --
>
> Key: LUCENE-8409
> URL: https://issues.apache.org/jira/browse/LUCENE-8409
> Project: Lucene - Core
>  Issue Type: Bug
>Reporter: Cetra Free
>Priority: Major
>
> This is based upon the following tickets:
> https://issues.apache.org/jira/browse/SOLR-12185
> https://issues.apache.org/jira/browse/LUCENE-8235
> I'd like a way to be able to clear and recreate a specific field so I don't 
> have to completely reindex if I change a field type.
> It's a real pain if you change a specific field from single valued to 
> multivalued, you have to delete the entire index from disk and start from 
> scratch.
> As being able to modify a field is not an [intended 
> feature|https://issues.apache.org/jira/browse/LUCENE-8235?focusedCommentId=16530918=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-16530918],
>  It'd be preferable if a field could be at least deleted and recreated to 
> deal with this scenario.
>  
>  
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8409) Cannot delete individual fields from index

[Erick Erickson (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8409?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547308#comment-16547308
 ] 

Erick Erickson commented on LUCENE-8409:


Hmm, we already have UnInvertingblahblah, I wonder if it could be extended?

> Cannot delete individual fields from index
> --
>
> Key: LUCENE-8409
> URL: https://issues.apache.org/jira/browse/LUCENE-8409
> Project: Lucene - Core
>  Issue Type: Bug
>Reporter: Cetra Free
>Priority: Major
>
> This is based upon the following tickets:
> https://issues.apache.org/jira/browse/SOLR-12185
> https://issues.apache.org/jira/browse/LUCENE-8235
> I'd like a way to be able to clear and recreate a specific field so I don't 
> have to completely reindex if I change a field type.
> It's a real pain if you change a specific field from single valued to 
> multivalued, you have to delete the entire index from disk and start from 
> scratch.
> As being able to modify a field is not an [intended 
> feature|https://issues.apache.org/jira/browse/LUCENE-8235?focusedCommentId=16530918=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-16530918],
>  It'd be preferable if a field could be at least deleted and recreated to 
> deal with this scenario.
>  
>  
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8408) Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8408?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547303#comment-16547303
 ] 

David Smiley commented on LUCENE-8408:
--

Glad you noticed Uwe :)  It was written before LUCENE-5640 and made obsolete by 
that issue.

> Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY
> 
>
> Key: LUCENE-8408
> URL: https://issues.apache.org/jira/browse/LUCENE-8408
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Michael Braun
>Priority: Trivial
> Attachments: LUCENE-8408.patch
>
>
> At the top of TokenStreamFromTermVector:
> {code}
>  //This attribute factory uses less memory when captureState() is called.
>   public static final AttributeFactory ATTRIBUTE_FACTORY =
>   AttributeFactory.getStaticImplementation(
>   AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, 
> PackedTokenAttributeImpl.class);
> {code}
> This is the default if super() was called with no-args from the constructor, 
> so I believe this can go away.  CC [~dsmiley]



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8403) Support 'filtered' term vectors - don't require all terms to be present

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8403?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547298#comment-16547298
 ] 

David Smiley commented on LUCENE-8403:
--

{quote}Doesn't the matches API remove the need for that feature? It should 
allow to retrieve offsets from the inverted index directly?
{quote}
I assume you refer to 
{{org.apache.lucene.search.uhighlight.UnifiedHighlighter.OffsetSource#POSTINGS_WITH_TERM_VECTORS}}
   No, it's definitely not made obsolete; I don't think the new Weight.Matches 
is related at all.  But good question though!  Consider the case of an index 
that only has offsets in postings (in the inverted index), there are no term 
vectors, and you're asked to highlight a MultiTermQuery.  Without the benefit 
of term vectors for the document you want to highlight, you'd have to set an 
MTQ loose on the entire index to find potentially thousands of terms (maybe 
only one is in this document?) which could be very high overhead.  At least the 
worse case would be terrible.  The UH (and PostingsHighlighter before it) sees 
an MTQ and switches over to Analysis.  Perhaps it isn't necessarily always more 
costly than term vectors or analysis – like if the MTQ matches a small number 
of terms.  I suppose a smarter highlighter could check first.  CC [~romseygeek]

> Support 'filtered' term vectors - don't require all terms to be present
> ---
>
> Key: LUCENE-8403
> URL: https://issues.apache.org/jira/browse/LUCENE-8403
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Michael Braun
>Priority: Minor
>
> The genesis of this was a conversation and idea from [~dsmiley] several years 
> ago.
> In order to optimize term vector storage, we may not actually need all tokens 
> to be present in the term vectors - and if so, ideally our codec could just 
> opt not to store them.
> I attempted to fork the standard codec and override the TermVectorsFormat and 
> TermVectorsWriter to ignore storing certain Terms within a field. This 
> worked, however, CheckIndex checks that the terms present in the standard 
> postings are also present in the TVs, if TVs enabled. So this then doesn't 
> work as 'valid' according to CheckIndex.
> Can the TermVectorsFormat be made in such a way to support configuration of 
> tokens that should not be stored (benefits: less storage, more optimal 
> retrieval per doc)? Is this valuable to the wider community? Is there a way 
> we can design this to not break CheckIndex's contract while at the same time 
> lessening storage for unneeded tokens?



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8408) Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8408?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547293#comment-16547293
 ] 

Uwe Schindler commented on LUCENE-8408:
---

+1

Why was this added at all?

> Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY
> 
>
> Key: LUCENE-8408
> URL: https://issues.apache.org/jira/browse/LUCENE-8408
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Michael Braun
>Priority: Trivial
> Attachments: LUCENE-8408.patch
>
>
> At the top of TokenStreamFromTermVector:
> {code}
>  //This attribute factory uses less memory when captureState() is called.
>   public static final AttributeFactory ATTRIBUTE_FACTORY =
>   AttributeFactory.getStaticImplementation(
>   AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, 
> PackedTokenAttributeImpl.class);
> {code}
> This is the default if super() was called with no-args from the constructor, 
> so I believe this can go away.  CC [~dsmiley]



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (LUCENE-8408) Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY

[David Smiley (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8408?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

David Smiley updated LUCENE-8408:
-
Attachment: LUCENE-8408.patch

> Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY
> 
>
> Key: LUCENE-8408
> URL: https://issues.apache.org/jira/browse/LUCENE-8408
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Michael Braun
>Priority: Trivial
> Attachments: LUCENE-8408.patch
>
>
> At the top of TokenStreamFromTermVector:
> {code}
>  //This attribute factory uses less memory when captureState() is called.
>   public static final AttributeFactory ATTRIBUTE_FACTORY =
>   AttributeFactory.getStaticImplementation(
>   AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, 
> PackedTokenAttributeImpl.class);
> {code}
> This is the default if super() was called with no-args from the constructor, 
> so I believe this can go away.  CC [~dsmiley]



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8408) Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8408?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547288#comment-16547288
 ] 

David Smiley commented on LUCENE-8408:
--

Correct; I wrote that.  While it used to be of use back in Solr 4, I think it's 
now obsolete.

> Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY
> 
>
> Key: LUCENE-8408
> URL: https://issues.apache.org/jira/browse/LUCENE-8408
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Michael Braun
>Priority: Trivial
>
> At the top of TokenStreamFromTermVector:
> {code}
>  //This attribute factory uses less memory when captureState() is called.
>   public static final AttributeFactory ATTRIBUTE_FACTORY =
>   AttributeFactory.getStaticImplementation(
>   AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, 
> PackedTokenAttributeImpl.class);
> {code}
> This is the default if super() was called with no-args from the constructor, 
> so I believe this can go away.  CC [~dsmiley]



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8405) Remove TopHits.maxScore

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8405?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547286#comment-16547286
 ] 

David Smiley commented on LUCENE-8405:
--

A comment is perfect; thanks.

> Remove TopHits.maxScore
> ---
>
> Key: LUCENE-8405
> URL: https://issues.apache.org/jira/browse/LUCENE-8405
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Adrien Grand
>Priority: Major
> Fix For: master (8.0)
>
> Attachments: LUCENE-8405.patch, LUCENE-8405.patch
>
>
> I would like to propose removing TopDocs.maxScore. The reasoning is that 
> either you are sorting by score and then its value is easy to access via the 
> score of the best hit. Or you sort by one or more fields and computing it is 
> wasteful:
>  - term frequencies and norms need to be read and decoded for every match
>  - scores need to be computed on every match
>  - early-termination optimizations are disabled
> It would be more efficient to collect hits twice: once with scores disabled 
> to get the top hits, and once to get the best score which would run 
> efficiently thanks to impacts and MAXSCORE, especially with a size of 1:
> {code:java}
> TopDocs topHits = searcher.search(query, 1);
> float maxScore = topHits.scoreDocs.length == 0 ? Float.NaN : 
> topHits.scoreDocs[0].score;
> {code}
> The {{doDocScores}} option of TopFieldCollector has drawbacks as well but at 
> least doesn't disable early-termination optimizations and doesn't require 
> scores to be computed on every hit.
> As this would be a significant breaking change, I'm targeting 8.0.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12412) Leader should give up leadership when IndexWriter.tragedy occur

[Varun Thacker (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12412?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547281#comment-16547281
 ] 

Varun Thacker commented on SOLR-12412:
--

Got it!
{code:java}
//TODO better way to test this
Thread.sleep(5000);
Replica leader = getCollectionState(collection).getSlice("shard1").getLeader();
assertEquals(leader.getName(), oldLeader.getName());

if (otherReplicaJetty != null) {
  // won't be able to do anything here, since this replica can't recovery from 
the leader
  otherReplicaJetty.start();
}{code}
Should we start the otherReplicaJetty and then check if the leader doesn't 
change in the test i.e reverse the order here ? 

Also maybe we can add the explanation as comments to the test code ? To someone 
new it would make it a lot easier to understand what this test is trying to do.

> Leader should give up leadership when IndexWriter.tragedy occur
> ---
>
> Key: SOLR-12412
> URL: https://issues.apache.org/jira/browse/SOLR-12412
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Cao Manh Dat
>Assignee: Cao Manh Dat
>Priority: Major
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12412.patch, SOLR-12412.patch, 
> jenkins-failure-2325.log
>
>
> When a leader meets some kind of unrecoverable exception (ie: 
> CorruptedIndexException). The shard will go into the readable state and human 
> has to intervene. 
> In that case, if there are another active replica in the same shard, the 
> leader should give up its leadership.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-12412) Leader should give up leadership when IndexWriter.tragedy occur

[Cao Manh Dat (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12412?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Cao Manh Dat updated SOLR-12412:

Description: 
When a leader meets some kind of unrecoverable exception (ie: 
CorruptedIndexException). The shard will go into the readable state and human 
has to intervene. 

In that case, if there are another active replica in the same shard, the leader 
should give up its leadership.

  was:When a leader meets some kind of unrecoverable exception (ie: 
CorruptedIndexException). The shard will go into the readable state and human 
has to intervene. In that case, it will be the best if the leader gives up its 
leadership and let other replicas become the leader. 


> Leader should give up leadership when IndexWriter.tragedy occur
> ---
>
> Key: SOLR-12412
> URL: https://issues.apache.org/jira/browse/SOLR-12412
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Cao Manh Dat
>Assignee: Cao Manh Dat
>Priority: Major
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12412.patch, SOLR-12412.patch, 
> jenkins-failure-2325.log
>
>
> When a leader meets some kind of unrecoverable exception (ie: 
> CorruptedIndexException). The shard will go into the readable state and human 
> has to intervene. 
> In that case, if there are another active replica in the same shard, the 
> leader should give up its leadership.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12412) Leader should give up leadership when IndexWriter.tragedy occur

[Cao Manh Dat (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12412?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547278#comment-16547278
 ] 

Cao Manh Dat commented on SOLR-12412:
-

[~varunthacker] The leader will only give up its leadership only in the case 
there are another active replica in the same shard. In the 
{{testOtherReplicasAreNotActive}}, we randomly 2 cases:
# A shard with only 1 replica
# A shard with 2 replica but the *non leader replica* is DOWN

In both case, the leader should not give up its leadership.

> Leader should give up leadership when IndexWriter.tragedy occur
> ---
>
> Key: SOLR-12412
> URL: https://issues.apache.org/jira/browse/SOLR-12412
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Cao Manh Dat
>Assignee: Cao Manh Dat
>Priority: Major
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12412.patch, SOLR-12412.patch, 
> jenkins-failure-2325.log
>
>
> When a leader meets some kind of unrecoverable exception (ie: 
> CorruptedIndexException). The shard will go into the readable state and human 
> has to intervene. In that case, it will be the best if the leader gives up 
> its leadership and let other replicas become the leader. 



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Status of solr tests

[Steve Rowe]

> On Jun 15, 2018, at 8:29 AM, David Smiley  wrote:
> 
> I'm +1 to modify the Lucene-side JIRA QA bot (Yetus) to not execute Solr 
> tests.

Right now, Yetus only executes Solr tests when there is a Solr change in the 
patch; otherwise only Lucene tests are executed.

I just committed a modification to the Lucene/Solr Yetus personality that adds 
"-Dtests.badapples=false" to the per-modified-module “ant test” cmdline.  This 
should reduce the noise appreciably.

--
Steve
www.lucidworks.com


-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk1.8.0_172) - Build # 22472 - Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/22472/
Java: 64bit/jdk1.8.0_172 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest

Error Message:


Stack Trace:
java.lang.AssertionError
at 
__randomizedtesting.SeedInfo.seed([97E4A053826B684D:1492FFA1541266EC]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at 
org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest(TestCloudRecovery.java:200)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.handler.component.InfixSuggestersTest.testShutdownDuringBuild

Error Message:
junit.framework.AssertionFailedError: Unexpected wrapped exception type, 
expected CoreIsClosedException

Stack Trace:
java.util.concurrent.ExecutionException: junit.framework.AssertionFailedError: 

[jira] [Commented] (LUCENE-2562) Make Luke a Lucene/Solr Module

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-2562?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547232#comment-16547232
 ] 

Uwe Schindler commented on LUCENE-2562:
---

Thanks, looks good in your commit.

I am still unhappy that Luke will stop working with Java 11 (e.g., which will 
be the default after September's Update of Ubuntu 18.04).

> Make Luke a Lucene/Solr Module
> --
>
> Key: LUCENE-2562
> URL: https://issues.apache.org/jira/browse/LUCENE-2562
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Mark Miller
>Priority: Major
>  Labels: gsoc2014
> Attachments: LUCENE-2562-Ivy.patch, LUCENE-2562-Ivy.patch, 
> LUCENE-2562-Ivy.patch, LUCENE-2562-ivy.patch, LUCENE-2562.patch, 
> LUCENE-2562.patch, Luke-ALE-1.png, Luke-ALE-2.png, Luke-ALE-3.png, 
> Luke-ALE-4.png, Luke-ALE-5.png, luke-javafx1.png, luke-javafx2.png, 
> luke-javafx3.png, luke1.jpg, luke2.jpg, luke3.jpg, lukeALE-documents.png
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> see
> "RE: Luke - in need of maintainer": 
> http://markmail.org/message/m4gsto7giltvrpuf
> "Web-based Luke": http://markmail.org/message/4xwps7p7ifltme5q
> I think it would be great if there was a version of Luke that always worked 
> with trunk - and it would also be great if it was easier to match Luke jars 
> with Lucene versions.
> While I'd like to get GWT Luke into the mix as well, I think the easiest 
> starting point is to straight port Luke to another UI toolkit before 
> abstracting out DTO objects that both GWT Luke and Pivot Luke could share.
> I've started slowly converting Luke's use of thinlet to Apache Pivot. I 
> haven't/don't have a lot of time for this at the moment, but I've plugged 
> away here and there over the past work or two. There is still a *lot* to do.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-2562) Make Luke a Lucene/Solr Module

[Steve Rowe (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-2562?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547219#comment-16547219
 ] 

Steve Rowe commented on LUCENE-2562:


Thanks [~thetaphi], I put in an Ant property to control it.

That reveals a lot forbidden-api violations left to tackle now:

{noformat}
-check-forbidden-all:
[forbidden-apis] Reading bundled API signatures: jdk-unsafe-1.8
[forbidden-apis] Reading bundled API signatures: jdk-deprecated-1.8
[forbidden-apis] Reading bundled API signatures: jdk-internal-1.8
[forbidden-apis] Reading bundled API signatures: jdk-reflection
[forbidden-apis] Reading API signatures: 
/Users/sarowe/git/lucene-solr/lucene/tools/forbiddenApis/base.txt
[forbidden-apis] Reading API signatures: 
/Users/sarowe/git/lucene-solr/lucene/tools/forbiddenApis/lucene.txt
[forbidden-apis] Loading classes to check...
[forbidden-apis] Scanning classes for violations...
[forbidden-apis] Forbidden method invocation: java.util.Locale#toString() [use 
Locale#toLanguageTag() for a standardized BCP47 locale name]
[forbidden-apis]   in 
org.apache.lucene.luke.app.controllers.fragments.search.QueryParserController 
(QueryParserController.java:159)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.overview.OverviewImpl 
(OverviewImpl.java:159)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.overview.OverviewImpl 
(OverviewImpl.java:164)
[forbidden-apis] Forbidden method invocation: java.lang.String#getBytes() [Uses 
default charset]
[forbidden-apis]   in org.apache.lucene.luke.models.analysis.TestAnalysisImpl 
(TestAnalysisImpl.java:93)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in 
org.apache.lucene.luke.app.controllers.dto.overview.TermCount 
(TermCount.java:29)
[forbidden-apis] Forbidden method invocation: 
java.io.PrintStream#(java.io.OutputStream,boolean) [Uses default charset]
[forbidden-apis]   in org.apache.lucene.luke.app.util.TextAreaPrintStream 
(TextAreaPrintStream.java:38)
[forbidden-apis] Forbidden method invocation: java.lang.String#getBytes() [Uses 
default charset]
[forbidden-apis]   in org.apache.lucene.luke.app.util.TextAreaPrintStream 
(TextAreaPrintStream.java:49)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:235)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:236)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:237)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:238)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:239)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:240)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:241)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:242)
[forbidden-apis] Forbidden method invocation: java.util.Locale#toString() [use 
Locale#toLanguageTag() for a standardized BCP47 locale name]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:243)
[forbidden-apis] Forbidden method invocation: 
java.lang.String#format(java.lang.String,java.lang.Object[]) [Uses default 
locale]
[forbidden-apis]   in org.apache.lucene.luke.models.search.QueryParserConfig 
(QueryParserConfig.java:243)
[forbidden-apis] Forbidden 

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 264 - Still unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/264/

1 tests failed.
FAILED:  org.apache.solr.cloud.ShardRoutingTest.test

Error Message:
expected:<3> but was:<8>

Stack Trace:
java.lang.AssertionError: expected:<3> but was:<8>
at 
__randomizedtesting.SeedInfo.seed([26AD81B84C13231B:AEF9BE62E2EF4EE3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at 
org.apache.solr.cloud.ShardRoutingTest.doTestNumRequests(ShardRoutingTest.java:256)
at 
org.apache.solr.cloud.ShardRoutingTest.test(ShardRoutingTest.java:110)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  

[jira] [Comment Edited] (LUCENE-2562) Make Luke a Lucene/Solr Module

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-2562?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547192#comment-16547192
 ] 

Uwe Schindler edited comment on LUCENE-2562 at 7/17/18 11:02 PM:
-

On top of that: Starting with Java 11, JavaFX is no longer shipped with the 
JDK, so forbiddenapis is correct: it's not portable.

See 
https://www.infoworld.com/article/3261066/java/javafx-will-be-removed-from-the-java-jdk.html


was (Author: thetaphi):
On top of that: Starting with Java 11, JavaFX is no longer shipped with the 
JDK, so forbiddenapis is correct: it's not portable.

> Make Luke a Lucene/Solr Module
> --
>
> Key: LUCENE-2562
> URL: https://issues.apache.org/jira/browse/LUCENE-2562
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Mark Miller
>Priority: Major
>  Labels: gsoc2014
> Attachments: LUCENE-2562-Ivy.patch, LUCENE-2562-Ivy.patch, 
> LUCENE-2562-Ivy.patch, LUCENE-2562-ivy.patch, LUCENE-2562.patch, 
> LUCENE-2562.patch, Luke-ALE-1.png, Luke-ALE-2.png, Luke-ALE-3.png, 
> Luke-ALE-4.png, Luke-ALE-5.png, luke-javafx1.png, luke-javafx2.png, 
> luke-javafx3.png, luke1.jpg, luke2.jpg, luke3.jpg, lukeALE-documents.png
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> see
> "RE: Luke - in need of maintainer": 
> http://markmail.org/message/m4gsto7giltvrpuf
> "Web-based Luke": http://markmail.org/message/4xwps7p7ifltme5q
> I think it would be great if there was a version of Luke that always worked 
> with trunk - and it would also be great if it was easier to match Luke jars 
> with Lucene versions.
> While I'd like to get GWT Luke into the mix as well, I think the easiest 
> starting point is to straight port Luke to another UI toolkit before 
> abstracting out DTO objects that both GWT Luke and Pivot Luke could share.
> I've started slowly converting Luke's use of thinlet to Apache Pivot. I 
> haven't/don't have a lot of time for this at the moment, but I've plugged 
> away here and there over the past work or two. There is still a *lot* to do.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-2562) Make Luke a Lucene/Solr Module

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-2562?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547192#comment-16547192
 ] 

Uwe Schindler commented on LUCENE-2562:
---

On top of that: Starting with Java 11, JavaFX is no longer shipped with the 
JDK, so forbiddenapis is correct: it's not portable.

> Make Luke a Lucene/Solr Module
> --
>
> Key: LUCENE-2562
> URL: https://issues.apache.org/jira/browse/LUCENE-2562
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Mark Miller
>Priority: Major
>  Labels: gsoc2014
> Attachments: LUCENE-2562-Ivy.patch, LUCENE-2562-Ivy.patch, 
> LUCENE-2562-Ivy.patch, LUCENE-2562-ivy.patch, LUCENE-2562.patch, 
> LUCENE-2562.patch, Luke-ALE-1.png, Luke-ALE-2.png, Luke-ALE-3.png, 
> Luke-ALE-4.png, Luke-ALE-5.png, luke-javafx1.png, luke-javafx2.png, 
> luke-javafx3.png, luke1.jpg, luke2.jpg, luke3.jpg, lukeALE-documents.png
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> see
> "RE: Luke - in need of maintainer": 
> http://markmail.org/message/m4gsto7giltvrpuf
> "Web-based Luke": http://markmail.org/message/4xwps7p7ifltme5q
> I think it would be great if there was a version of Luke that always worked 
> with trunk - and it would also be great if it was easier to match Luke jars 
> with Lucene versions.
> While I'd like to get GWT Luke into the mix as well, I think the easiest 
> starting point is to straight port Luke to another UI toolkit before 
> abstracting out DTO objects that both GWT Luke and Pivot Luke could share.
> I've started slowly converting Luke's use of thinlet to Apache Pivot. I 
> haven't/don't have a lot of time for this at the moment, but I've plugged 
> away here and there over the past work or two. There is still a *lot* to do.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-2562) Make Luke a Lucene/Solr Module

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-2562?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547191#comment-16547191
 ] 

Uwe Schindler commented on LUCENE-2562:
---

Hi,
the forbiddenapis problems are caused by the way how javafx was exposed in 
previous versions of the JDK. With JDK8 it was shipped inside rt.jar, but on 
other versions it is not included at all and in Java 10 it's now part of module 
system.
The portable runtime stuff only allows classes from the well known java 
packages, which does not yet include {{javafx.**}}: 
https://github.com/policeman-tools/forbidden-apis/blob/master/src/main/java/de/thetaphi/forbiddenapis/AsmUtils.java#L39-L40

The problem JavaFX is still not an official part of the Java SE platform, it's 
still optional! In addition this needs to be some special exceptional check, as 
javafx is only valid as runtime class in Java 8, in Java 7 it's not indluded.

If You run forbiddenapis with Java 10, it should pass. The reason is: Although 
it's shipped with the JDK and the class name is not in the above pattern, but 
because it's an external module - it works as expected (I have not verified 
this, maybe you can do it). The reason is the module name, it's external. With 
Java 8 we have no module names, so the check can only use the above whitelist 
of package names.

In short: That's a limitation of forbiddenapis and my ignorance for JavaFX. I'd 
suggest to redefine the forbiddenapis target in the luke module to not use the 
"jdk-non-portable" signature and instead the more static "jdk-internal" 
signature (maybe thorugh an ANT property). Nevertheless, as described above, 
JavaFX is a "non-portable" part of the JDK as it's not part of the spec. Not 
sure what the best way to solve this. I'd go for the workaround now and use 
"jdk-internal" instead of "jdk-non-portable" for the luke module (not for 
others, please).

> Make Luke a Lucene/Solr Module
> --
>
> Key: LUCENE-2562
> URL: https://issues.apache.org/jira/browse/LUCENE-2562
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Mark Miller
>Priority: Major
>  Labels: gsoc2014
> Attachments: LUCENE-2562-Ivy.patch, LUCENE-2562-Ivy.patch, 
> LUCENE-2562-Ivy.patch, LUCENE-2562-ivy.patch, LUCENE-2562.patch, 
> LUCENE-2562.patch, Luke-ALE-1.png, Luke-ALE-2.png, Luke-ALE-3.png, 
> Luke-ALE-4.png, Luke-ALE-5.png, luke-javafx1.png, luke-javafx2.png, 
> luke-javafx3.png, luke1.jpg, luke2.jpg, luke3.jpg, lukeALE-documents.png
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> see
> "RE: Luke - in need of maintainer": 
> http://markmail.org/message/m4gsto7giltvrpuf
> "Web-based Luke": http://markmail.org/message/4xwps7p7ifltme5q
> I think it would be great if there was a version of Luke that always worked 
> with trunk - and it would also be great if it was easier to match Luke jars 
> with Lucene versions.
> While I'd like to get GWT Luke into the mix as well, I think the easiest 
> starting point is to straight port Luke to another UI toolkit before 
> abstracting out DTO objects that both GWT Luke and Pivot Luke could share.
> I've started slowly converting Luke's use of thinlet to Apache Pivot. I 
> haven't/don't have a lot of time for this at the moment, but I've plugged 
> away here and there over the past work or two. There is still a *lot* to do.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Resolved] (SOLR-12538) log4j exceptions during startup on Windows

[Erick Erickson (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12538?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Erick Erickson resolved SOLR-12538.
---
   Resolution: Fixed
Fix Version/s: 7.5
   master (8.0)

Closing as fixed since we tested it on windows and nobody has reported back. We 
can re-open if it recurs.

Meanwhile, a temporary workaround should be to changes instances of:
{code}
file:%EXAMPLE_DIR%\resources\log4j2.xml
to
file:///%EXAMPLE_DIR%\resources\log4j2.xml
{code}

> log4j exceptions during startup on Windows
> --
>
> Key: SOLR-12538
> URL: https://issues.apache.org/jira/browse/SOLR-12538
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: scripts and tools
>Affects Versions: 7.4
>Reporter: Jakob Furrer
>Assignee: Erick Erickson
>Priority: Minor
> Fix For: master (8.0), 7.5
>
>
> Note that there has been some input regarding this issue on the Solr 
> mailinglist:
>  
> [http://lucene.472066.n3.nabble.com/Logging-fails-when-starting-Solr-in-Windows-using-solr-cmd-td4396671.html]
> Problem description 
>  ==
> System: Microsoft Windows 10 Enterprise Version 10.0.16299 Build 16299
> Steps to reproduce the problem: 
>  1) Download solr-7.4.0.zip
> 2) Unzip to C:\solr-7.4.0
> 3) No changes (configuration or otherwise) whatsoever
> 4) Open cmd.exe
> 5) Execute the following command: *cd c:\solr-7.4.0\bin*
> 6) Execute the following command: *solr.cmd start -p 8983*
> 7) The following console output appears:
> {code:java}
> c:\solr-7.4.0\bin>solr.cmd start -p 8983 
> ERROR StatusLogger Unable to access 
> file:/c:/solr-7.4.0/server/file:c:/solr-7.4.0/server/scripts/cloud-scripts/log4j2.xml
>  
>   java.io.FileNotFoundException: 
> c:\solr-7.4.0\server\file:c:\solr-7.4.0\server\scripts\cloud-scripts\log4j2.xml
>  
> (Die Syntax für den Dateinamen, Verzeichnisnamen oder die 
> Datenträgerbezeichnung ist falsch) 
>          at java.io.FileInputStream.open0(Native Method) 
>          at java.io.FileInputStream.open(FileInputStream.java:195) 
>          at java.io.FileInputStream.(FileInputStream.java:138) 
>          at java.io.FileInputStream.(FileInputStream.java:93) 
>          at 
> sun.net.www.protocol.file.FileURLConnection.connect(FileURLConnection.java:90)
>  
>          at 
> sun.net.www.protocol.file.FileURLConnection.getInputStream(FileURLConnection.java:188)
>  
>          at java.net.URL.openStream(URL.java:1045) 
>          at 
> org.apache.logging.log4j.core.config.ConfigurationSource.fromUri(ConfigurationSource.java:247)
>  
>          at 
> org.apache.logging.log4j.core.config.ConfigurationFactory$Factory.getConfiguration(ConfigurationFactory.java:404)
>  
>          at 
> org.apache.logging.log4j.core.config.ConfigurationFactory$Factory.getConfiguration(ConfigurationFactory.java:346)
>  
>          at 
> org.apache.logging.log4j.core.config.ConfigurationFactory.getConfiguration(ConfigurationFactory.java:260)
>  
>          at 
> org.apache.logging.log4j.core.LoggerContext.reconfigure(LoggerContext.java:615)
>  
>          at 
> org.apache.logging.log4j.core.LoggerContext.reconfigure(LoggerContext.java:636)
>  
>          at 
> org.apache.logging.log4j.core.LoggerContext.start(LoggerContext.java:231) 
>          at 
> org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:153)
>  
>          at 
> org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:45)
>  
>          at 
> org.apache.logging.log4j.LogManager.getContext(LogManager.java:194) 
>          at 
> org.apache.logging.log4j.spi.AbstractLoggerAdapter.getContext(AbstractLoggerAdapter.java:121)
>  
>          at 
> org.apache.logging.slf4j.Log4jLoggerFactory.getContext(Log4jLoggerFactory.java:43)
>  
>          at 
> org.apache.logging.log4j.spi.AbstractLoggerAdapter.getLogger(AbstractLoggerAdapter.java:46)
>  
>          at 
> org.apache.logging.slf4j.Log4jLoggerFactory.getLogger(Log4jLoggerFactory.java:29)
>  
>          at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:358) 
>          at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:383) 
>          at org.apache.solr.util.SolrCLI.(SolrCLI.java:228) 
> ERROR StatusLogger Unable to access 
> file:/c:/solr-7.4.0/server/file:c:/solr-7.4.0/server/resources/log4j2.xml 
>   java.io.FileNotFoundException: 
> c:\solr-7.4.0\server\file:c:\solr-7.4.0\server\resources\log4j2.xml (Die 
> Syntax für den Dateinamen, Verzeichnisnamen oder die 
> Datenträgerbezeichnung ist falsch) 
>          at java.io.FileInputStream.open0(Native Method) 
>          at java.io.FileInputStream.open(FileInputStream.java:195) 
>          at java.io.FileInputStream.(FileInputStream.java:138) 
>          at 

[jira] [Assigned] (SOLR-12538) log4j exceptions during startup on Windows

[Erick Erickson (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12538?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Erick Erickson reassigned SOLR-12538:
-

Assignee: Erick Erickson

> log4j exceptions during startup on Windows
> --
>
> Key: SOLR-12538
> URL: https://issues.apache.org/jira/browse/SOLR-12538
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: scripts and tools
>Affects Versions: 7.4
>Reporter: Jakob Furrer
>Assignee: Erick Erickson
>Priority: Minor
>
> Note that there has been some input regarding this issue on the Solr 
> mailinglist:
>  
> [http://lucene.472066.n3.nabble.com/Logging-fails-when-starting-Solr-in-Windows-using-solr-cmd-td4396671.html]
> Problem description 
>  ==
> System: Microsoft Windows 10 Enterprise Version 10.0.16299 Build 16299
> Steps to reproduce the problem: 
>  1) Download solr-7.4.0.zip
> 2) Unzip to C:\solr-7.4.0
> 3) No changes (configuration or otherwise) whatsoever
> 4) Open cmd.exe
> 5) Execute the following command: *cd c:\solr-7.4.0\bin*
> 6) Execute the following command: *solr.cmd start -p 8983*
> 7) The following console output appears:
> {code:java}
> c:\solr-7.4.0\bin>solr.cmd start -p 8983 
> ERROR StatusLogger Unable to access 
> file:/c:/solr-7.4.0/server/file:c:/solr-7.4.0/server/scripts/cloud-scripts/log4j2.xml
>  
>   java.io.FileNotFoundException: 
> c:\solr-7.4.0\server\file:c:\solr-7.4.0\server\scripts\cloud-scripts\log4j2.xml
>  
> (Die Syntax für den Dateinamen, Verzeichnisnamen oder die 
> Datenträgerbezeichnung ist falsch) 
>          at java.io.FileInputStream.open0(Native Method) 
>          at java.io.FileInputStream.open(FileInputStream.java:195) 
>          at java.io.FileInputStream.(FileInputStream.java:138) 
>          at java.io.FileInputStream.(FileInputStream.java:93) 
>          at 
> sun.net.www.protocol.file.FileURLConnection.connect(FileURLConnection.java:90)
>  
>          at 
> sun.net.www.protocol.file.FileURLConnection.getInputStream(FileURLConnection.java:188)
>  
>          at java.net.URL.openStream(URL.java:1045) 
>          at 
> org.apache.logging.log4j.core.config.ConfigurationSource.fromUri(ConfigurationSource.java:247)
>  
>          at 
> org.apache.logging.log4j.core.config.ConfigurationFactory$Factory.getConfiguration(ConfigurationFactory.java:404)
>  
>          at 
> org.apache.logging.log4j.core.config.ConfigurationFactory$Factory.getConfiguration(ConfigurationFactory.java:346)
>  
>          at 
> org.apache.logging.log4j.core.config.ConfigurationFactory.getConfiguration(ConfigurationFactory.java:260)
>  
>          at 
> org.apache.logging.log4j.core.LoggerContext.reconfigure(LoggerContext.java:615)
>  
>          at 
> org.apache.logging.log4j.core.LoggerContext.reconfigure(LoggerContext.java:636)
>  
>          at 
> org.apache.logging.log4j.core.LoggerContext.start(LoggerContext.java:231) 
>          at 
> org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:153)
>  
>          at 
> org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:45)
>  
>          at 
> org.apache.logging.log4j.LogManager.getContext(LogManager.java:194) 
>          at 
> org.apache.logging.log4j.spi.AbstractLoggerAdapter.getContext(AbstractLoggerAdapter.java:121)
>  
>          at 
> org.apache.logging.slf4j.Log4jLoggerFactory.getContext(Log4jLoggerFactory.java:43)
>  
>          at 
> org.apache.logging.log4j.spi.AbstractLoggerAdapter.getLogger(AbstractLoggerAdapter.java:46)
>  
>          at 
> org.apache.logging.slf4j.Log4jLoggerFactory.getLogger(Log4jLoggerFactory.java:29)
>  
>          at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:358) 
>          at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:383) 
>          at org.apache.solr.util.SolrCLI.(SolrCLI.java:228) 
> ERROR StatusLogger Unable to access 
> file:/c:/solr-7.4.0/server/file:c:/solr-7.4.0/server/resources/log4j2.xml 
>   java.io.FileNotFoundException: 
> c:\solr-7.4.0\server\file:c:\solr-7.4.0\server\resources\log4j2.xml (Die 
> Syntax für den Dateinamen, Verzeichnisnamen oder die 
> Datenträgerbezeichnung ist falsch) 
>          at java.io.FileInputStream.open0(Native Method) 
>          at java.io.FileInputStream.open(FileInputStream.java:195) 
>          at java.io.FileInputStream.(FileInputStream.java:138) 
>          at java.io.FileInputStream.(FileInputStream.java:93) 
>          at 
> sun.net.www.protocol.file.FileURLConnection.connect(FileURLConnection.java:90)
>  
>          at 
> sun.net.www.protocol.file.FileURLConnection.getInputStream(FileURLConnection.java:188)
>  
>          at java.net.URL.openStream(URL.java:1045) 
>          at 
> 

[jira] [Commented] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Varun Thacker (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547162#comment-16547162
 ] 

Varun Thacker commented on SOLR-11598:
--

{quote}The tests passed with this change. So was that no needed or do our tests 
not catch this? I want to fix this as part of this patch
{quote}
I was chatting to Hoss offline and he pointed out that to sort we obviously 
need to global ords. I'm going to leave this for now, but the usage in 
ExportWriter's StringValue#setCurrentValue vs ExpandComponent#collect differs 
slightly.

FYI no failing test is still a concern. I manually added 4 docs 
{code:java}
[
{"id":"1", "cat_s" : "a"},
{"id":"2", "cat_s" : "ABC"}
]

commit 

[
{"id":"3", "cat_s" : "xyz"}, 
{"id":"4", "cat_s" : "a"} 
]

commit{code}
 

The sort order is wrong . The way I cross checked was against the select 
handler. Maybe we should do some validation tests against the select handler as 
well?

> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
>   at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>   at 
> 

[jira] [Commented] (LUCENE-2562) Make Luke a Lucene/Solr Module

[Steve Rowe (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-2562?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547130#comment-16547130
 ] 

Steve Rowe commented on LUCENE-2562:


[~Tomoko Uchida]: thanks for the pull request.  I pulled it into a branch named 
{{jira/lucene-2562}}, along with a few fixes and additions, mostly to get {{ant 
precommit}} to succeed, which it does not yet do.  I haven't tried to run the 
code yet :).

A few things I noticed:

# {{validate-source-patterns}} complains about non-static-final loggers:
{noformat}
validate-source-patterns:
[source-patterns] invalid logging pattern [not private static final, uses 
static class name]: 
lucene/luke/src/java/org/apache/lucene/luke/models/commits/CommitsImpl.java
[source-patterns] invalid logging pattern [not private static final, uses 
static class name]: 
lucene/luke/src/java/org/apache/lucene/luke/models/documents/DocumentsImpl.java
[source-patterns] invalid logging pattern [not private static final, uses 
static class name]: 
lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java
{noformat}
I fixed a few others like these, but ^^ have ctors that take a logger, 
apparently just to be able to turn logging off under testing.  Is this really 
necessary?  Right now I have a nocommit that allows {{ant precommit}} to ignore 
these, but it needs to be resolved.
# It should be possible to run Luke from a source checkout, but it's not now.
# I think Luke should be packaged with all other artifacts in Lucene's binary 
packages, but currently {{ant package-tgz}} and {{-zip}} don't include 
everything (e.g. scripts under {{bin/}}); instead, the build creates a {{.zip}} 
file that contains a copy of all the dependencies and the {{bin/}} scripts.  I 
think we can get rid of this Luke-only distribution, in favor of the Lucene 
distribution?  I don't like the idea of including the 
Luke-with-all-dependencies {{.zip}} in the Lucene distribution, since it will 
include two copies of many modules' jars.
# forbidden-apis is very unhappy about classes in {{javafx.\*}} packages, e.g.:
{noformat}
[forbidden-apis] Scanning classes for violations...
[forbidden-apis] Forbidden class/interface use: 
javafx.beans.property.BooleanProperty [non-portable or internal runtime class]
[forbidden-apis]   in 
org.apache.lucene.luke.app.controllers.dto.search.SelectedField 
(SelectedField.java, field declaration of 'selected')
[forbidden-apis] Forbidden class/interface use: 
javafx.beans.property.SimpleBooleanProperty [non-portable or internal runtime 
class]
[forbidden-apis]   in 
org.apache.lucene.luke.app.controllers.dto.search.SelectedField 
(SelectedField.java:29)
[forbidden-apis] Forbidden class/interface use: 
javafx.beans.property.SimpleBooleanProperty [non-portable or internal runtime 
class]
[...]
{noformat}
[~thetaphi]: do you understand what ^^ is about?

> Make Luke a Lucene/Solr Module
> --
>
> Key: LUCENE-2562
> URL: https://issues.apache.org/jira/browse/LUCENE-2562
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Mark Miller
>Priority: Major
>  Labels: gsoc2014
> Attachments: LUCENE-2562-Ivy.patch, LUCENE-2562-Ivy.patch, 
> LUCENE-2562-Ivy.patch, LUCENE-2562-ivy.patch, LUCENE-2562.patch, 
> LUCENE-2562.patch, Luke-ALE-1.png, Luke-ALE-2.png, Luke-ALE-3.png, 
> Luke-ALE-4.png, Luke-ALE-5.png, luke-javafx1.png, luke-javafx2.png, 
> luke-javafx3.png, luke1.jpg, luke2.jpg, luke3.jpg, lukeALE-documents.png
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> see
> "RE: Luke - in need of maintainer": 
> http://markmail.org/message/m4gsto7giltvrpuf
> "Web-based Luke": http://markmail.org/message/4xwps7p7ifltme5q
> I think it would be great if there was a version of Luke that always worked 
> with trunk - and it would also be great if it was easier to match Luke jars 
> with Lucene versions.
> While I'd like to get GWT Luke into the mix as well, I think the easiest 
> starting point is to straight port Luke to another UI toolkit before 
> abstracting out DTO objects that both GWT Luke and Pivot Luke could share.
> I've started slowly converting Luke's use of thinlet to Apache Pivot. I 
> haven't/don't have a lot of time for this at the moment, but I've plugged 
> away here and there over the past work or two. There is still a *lot* to do.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8405) Remove TopHits.maxScore

[Adrien Grand (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8405?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547112#comment-16547112
 ] 

Adrien Grand commented on LUCENE-8405:
--

Thanks Mike and David for looking!

bq. Why does MaxScoreCollector.scoreMode() return COMPLETE instead of 
TOP_SCORES?

It saves wrapping of the scorer since Scorer.setMinCompetitiveScore can't be 
used with multi-collectors in general. I will add a comment. Or I can change it 
if you think it's clearer.

> Remove TopHits.maxScore
> ---
>
> Key: LUCENE-8405
> URL: https://issues.apache.org/jira/browse/LUCENE-8405
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Adrien Grand
>Priority: Major
> Fix For: master (8.0)
>
> Attachments: LUCENE-8405.patch, LUCENE-8405.patch
>
>
> I would like to propose removing TopDocs.maxScore. The reasoning is that 
> either you are sorting by score and then its value is easy to access via the 
> score of the best hit. Or you sort by one or more fields and computing it is 
> wasteful:
>  - term frequencies and norms need to be read and decoded for every match
>  - scores need to be computed on every match
>  - early-termination optimizations are disabled
> It would be more efficient to collect hits twice: once with scores disabled 
> to get the top hits, and once to get the best score which would run 
> efficiently thanks to impacts and MAXSCORE, especially with a size of 1:
> {code:java}
> TopDocs topHits = searcher.search(query, 1);
> float maxScore = topHits.scoreDocs.length == 0 ? Float.NaN : 
> topHits.scoreDocs[0].score;
> {code}
> The {{doDocScores}} option of TopFieldCollector has drawbacks as well but at 
> least doesn't disable early-termination optimizations and doesn't require 
> scores to be computed on every hit.
> As this would be a significant breaking change, I'm targeting 8.0.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8403) Support 'filtered' term vectors - don't require all terms to be present

[Adrien Grand (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8403?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547104#comment-16547104
 ] 

Adrien Grand commented on LUCENE-8403:
--

bq. I can't seem to find those call-sites right now though.

This happens via MockDirectoryWrapper#close. MockDirectoryWrapper is often 
created via LuceneTestCase#newDirectory.

Doesn't the matches API remove the need for that feature? It should allow to 
retrieve offsets from the inverted index directly?

> Support 'filtered' term vectors - don't require all terms to be present
> ---
>
> Key: LUCENE-8403
> URL: https://issues.apache.org/jira/browse/LUCENE-8403
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Michael Braun
>Priority: Minor
>
> The genesis of this was a conversation and idea from [~dsmiley] several years 
> ago.
> In order to optimize term vector storage, we may not actually need all tokens 
> to be present in the term vectors - and if so, ideally our codec could just 
> opt not to store them.
> I attempted to fork the standard codec and override the TermVectorsFormat and 
> TermVectorsWriter to ignore storing certain Terms within a field. This 
> worked, however, CheckIndex checks that the terms present in the standard 
> postings are also present in the TVs, if TVs enabled. So this then doesn't 
> work as 'valid' according to CheckIndex.
> Can the TermVectorsFormat be made in such a way to support configuration of 
> tokens that should not be stored (benefits: less storage, more optimal 
> retrieval per doc)? Is this valuable to the wider community? Is there a way 
> we can design this to not break CheckIndex's contract while at the same time 
> lessening storage for unneeded tokens?



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8407) Add SpanTermQuery.getTermStates()

[Adrien Grand (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8407?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547102#comment-16547102
 ] 

Adrien Grand commented on LUCENE-8407:
--

+1

> Add SpanTermQuery.getTermStates()
> -
>
> Key: LUCENE-8407
> URL: https://issues.apache.org/jira/browse/LUCENE-8407
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: David Smiley
>Assignee: David Smiley
>Priority: Minor
>
> Adding a getTermStates() to the TermStates in a SpanTermQuery would be 
> useful, just like we have similarly for a TermQuery – LUCENE-8379.  It would 
> be useful for LUCENE-6513 to avoid a needless inner ScoreTerm class when a 
> SpanTermQuery would suffice.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (LUCENE-8408) Code cleanup - TokenStreamFromTermVector - ATTRIBUTE_FACTORY

[Michael Braun (JIRA)]

Michael Braun created LUCENE-8408:
-

 Summary: Code cleanup - TokenStreamFromTermVector - 
ATTRIBUTE_FACTORY
 Key: LUCENE-8408
 URL: https://issues.apache.org/jira/browse/LUCENE-8408
 Project: Lucene - Core
  Issue Type: Improvement
Reporter: Michael Braun


At the top of TokenStreamFromTermVector:
{code}
 //This attribute factory uses less memory when captureState() is called.
  public static final AttributeFactory ATTRIBUTE_FACTORY =
  AttributeFactory.getStaticImplementation(
  AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, 
PackedTokenAttributeImpl.class);
{code}
This is the default if super() was called with no-args from the constructor, so 
I believe this can go away.  CC [~dsmiley]



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Segfault in MergedIterator.pullTop()

[David Smiley]

Thanks for spreading the word!

On Tue, Jul 17, 2018 at 4:35 PM Adrien Grand  wrote:

> In the past weeks we got more and more reports of this segfault. It
> manifests with JDK 10 and processors that support AVX-512 instructions. For
> instance this bug has been reproduced by several users on AWS C5 and M5
> instances.
>
> I have no reason to think that only Elasticsearch is affected. Solr and
> other Lucene-based applications are probably affected too. It can be worked
> around by passing -XX:UseAVX=2 to the JVM arguments.
>
> This bug is tracked at https://bugs.openjdk.java.net/browse/JDK-8207746.
> Kudos to Jason Tedor for digging and reporting this bug.
>
> Le mer. 20 juin 2018 à 11:05, Adrien Grand  a écrit :
>
>> FYI we have had 3 independent reports of segfaults in
>> org.apache.lucene.util.MergedIterator.pullTop() when running Elasticsearch
>> with Java 10. Here is one of them:
>> https://github.com/elastic/elasticsearch/issues/31425. It's still
>> unclear to me what conditions need to be met for this bug to reproduce. I'm
>> wondering whether anyone has seen this JVM bug before? For the record,
>> Elasticsearch doesn't use MergedIterator explicitly, it only gets used via
>> MultiFields when merging terms.
>>
> --
Lucene/Solr Search Committer, Consultant, Developer, Author, Speaker
LinkedIn: http://linkedin.com/in/davidwsmiley | Book:
http://www.solrenterprisesearchserver.com

[jira] [Commented] (SOLR-12394) Remove managmentPath

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12394?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547076#comment-16547076
 ] 

David Smiley commented on SOLR-12394:
-

Lets just remove it on master (8x) and mark deprecated for 7x.  Add a warning 
for 7x if found in the config seems like a good idea, but isn't required either.

> Remove managmentPath 
> -
>
> Key: SOLR-12394
> URL: https://issues.apache.org/jira/browse/SOLR-12394
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Gus Heck
>Priority: Minor
> Attachments: SOLR-12394.patch
>
>
> NodeConfig has a property called managmentPath config which doesn't appear to 
> serve any coherent function and is in fact documented in 
> [https://lucene.apache.org/solr/guide/7_3/format-of-solr-xml.html] as:
> {quote}{{managementPath}}
> Currently non-operational.
> {quote}
> The code appears to have been added initially in SOLR-695, and that ticket 
> appears to relate to an elimination of a special case for single core 
> configurations. It seems that this may have been an attempt to support single 
> cores that had no name (a legacy mode of operation I guess, but before my 
> time) and yet still allow such single core setups to later have additional 
> cores added?
> So this ticket is a suggestion that we remove this configuration that 
> allegedly isn't working anyway, OR we make it work and give it good clear 
> documentation in code and in the ref guide so that folks don't have to waste 
> a lot of time figuring out what it does(n't do) to understand the code.
> Attaching patch to remove it. 
>  
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Segfault in MergedIterator.pullTop()

[Adrien Grand]

In the past weeks we got more and more reports of this segfault. It
manifests with JDK 10 and processors that support AVX-512 instructions. For
instance this bug has been reproduced by several users on AWS C5 and M5
instances.

I have no reason to think that only Elasticsearch is affected. Solr and
other Lucene-based applications are probably affected too. It can be worked
around by passing -XX:UseAVX=2 to the JVM arguments.

This bug is tracked at https://bugs.openjdk.java.net/browse/JDK-8207746.
Kudos to Jason Tedor for digging and reporting this bug.

Le mer. 20 juin 2018 à 11:05, Adrien Grand  a écrit :

> FYI we have had 3 independent reports of segfaults in
> org.apache.lucene.util.MergedIterator.pullTop() when running Elasticsearch
> with Java 10. Here is one of them:
> https://github.com/elastic/elasticsearch/issues/31425. It's still unclear
> to me what conditions need to be met for this bug to reproduce. I'm
> wondering whether anyone has seen this JVM bug before? For the record,
> Elasticsearch doesn't use MergedIterator explicitly, it only gets used via
> MultiFields when merging terms.
>

[jira] [Commented] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Varun Thacker (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547073#comment-16547073
 ] 

Varun Thacker commented on SOLR-11598:
--

Some changes I made on top of the last patch:

DoubleValue#setCurrentValue , the following if statement now throws an 
asssertion error. LongValue, IntValue and FloatValue were doing the same thing
{code:java}
if (docId < lastDocID) {
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID 
+ " vs doc=" + docId);
}{code}
Changed all the compareTo methods in IntAsc/Desc , LongAsc/Desc , 
DoubleAsc/Desc , FloatAsc/Desc to use Float.compare/Int.compare etc. We should 
add tests where we have float values like -0.0 and 0.0

Removed the 25 sort limit and a test which was trying to assert an error if max 
sort fields limit has been reached.

Prototyped NewIntValue which could replace IntValue . If this looks cleaner 
I'll implement it for the others . Thoughts ?

Lastly, I was trying to understand why are we using getSlowAtomicReader to get 
string doc-values. So as an experiment I am passing null to StringValue so that 
it will get per segment doc-values and not lookup against the 
getSlowAtomicReader
The tests passed with this change. So was that no needed or do our tests not 
catch this? I want to fix this as part of this patch

> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   

[jira] [Updated] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Varun Thacker (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Varun Thacker updated SOLR-11598:
-
Attachment: SOLR-11598.patch

> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
>   at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>   at 
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
>   at 
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
>   at 
> 

[jira] [Commented] (LUCENE-8407) Add SpanTermQuery.getTermStates()

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8407?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547071#comment-16547071
 ] 

David Smiley commented on LUCENE-8407:
--

This would go right below getTerm.  I copy pasted this from TermQuery, changed 
to reference different field name.
{code:java}
  /** Returns the {@link TermStates} passed to the constructor, or null if it 
was not passed.
   *
   * @lucene.experimental */
  public TermStates getTermStates() {
return termStates;
  }
{code}

> Add SpanTermQuery.getTermStates()
> -
>
> Key: LUCENE-8407
> URL: https://issues.apache.org/jira/browse/LUCENE-8407
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: David Smiley
>Assignee: David Smiley
>Priority: Minor
>
> Adding a getTermStates() to the TermStates in a SpanTermQuery would be 
> useful, just like we have similarly for a TermQuery – LUCENE-8379.  It would 
> be useful for LUCENE-6513 to avoid a needless inner ScoreTerm class when a 
> SpanTermQuery would suffice.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (LUCENE-8407) Add SpanTermQuery.getTermStates()

[David Smiley (JIRA)]

David Smiley created LUCENE-8407:


 Summary: Add SpanTermQuery.getTermStates()
 Key: LUCENE-8407
 URL: https://issues.apache.org/jira/browse/LUCENE-8407
 Project: Lucene - Core
  Issue Type: Improvement
Reporter: David Smiley
Assignee: David Smiley


Adding a getTermStates() to the TermStates in a SpanTermQuery would be useful, 
just like we have similarly for a TermQuery – LUCENE-8379.  It would be useful 
for LUCENE-6513 to avoid a needless inner ScoreTerm class when a SpanTermQuery 
would suffice.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12394) Remove managmentPath

[Christine Poerschke (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12394?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547039#comment-16547039
 ] 

Christine Poerschke commented on SOLR-12394:


+1 to removing unused code paths.

Wondering about what back-compatibility considerations apply here e.g.
* need the accessors stay around (as deprecated) for one 7.x version or all 7.x 
versions?
* should we start warning in one version if someone still has the element in 
their {{solr.xml}} and only in the next version would the 
https://github.com/apache/lucene-solr/blob/releases/lucene-solr/7.4.0/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java#L278
 style exceptions be thrown?
* is the code concerned too unused-and-never-used to not worry about orderly 
removal so long as {{solr/CHANGES.txt}} clearly mentions that a {{solr.xml}} 
update is needed?

> Remove managmentPath 
> -
>
> Key: SOLR-12394
> URL: https://issues.apache.org/jira/browse/SOLR-12394
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Gus Heck
>Priority: Minor
> Attachments: SOLR-12394.patch
>
>
> NodeConfig has a property called managmentPath config which doesn't appear to 
> serve any coherent function and is in fact documented in 
> [https://lucene.apache.org/solr/guide/7_3/format-of-solr-xml.html] as:
> {quote}{{managementPath}}
> Currently non-operational.
> {quote}
> The code appears to have been added initially in SOLR-695, and that ticket 
> appears to relate to an elimination of a special case for single core 
> configurations. It seems that this may have been an attempt to support single 
> cores that had no name (a legacy mode of operation I guess, but before my 
> time) and yet still allow such single core setups to later have additional 
> cores added?
> So this ticket is a suggestion that we remove this configuration that 
> allegedly isn't working anyway, OR we make it work and give it good clear 
> documentation in code and in the ref guide so that folks don't have to waste 
> a lot of time figuring out what it does(n't do) to understand the code.
> Attaching patch to remove it. 
>  
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (SOLR-12560) remove deprecated NodeConfigBuilder.setTransientCacheSize

[Christine Poerschke (JIRA)]

Christine Poerschke created SOLR-12560:
--

 Summary: remove deprecated NodeConfigBuilder.setTransientCacheSize
 Key: SOLR-12560
 URL: https://issues.apache.org/jira/browse/SOLR-12560
 Project: Solr
  Issue Type: Task
Reporter: Christine Poerschke


https://github.com/apache/lucene-solr/blob/releases/lucene-solr/7.4.0/solr/core/src/java/org/apache/solr/core/NodeConfig.java#L343-L348



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Robert Muir (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547032#comment-16547032
 ] 

Robert Muir commented on LUCENE-8406:
-

Yes but windows has solutions for such things too. I'm just not sure we should 
jump to writing java code quite yet here.

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12454) tweak Overseer leadership transition related logging

[Christine Poerschke (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12454?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547029#comment-16547029
 ] 

Christine Poerschke commented on SOLR-12454:


Returned to this ticket after some time and as per above have gone ahead and 
committed the patch -- as always though, further comments, suggestions, 
revisions etc. welcome -- thank you.

> tweak Overseer leadership transition related logging
> 
>
> Key: SOLR-12454
> URL: https://issues.apache.org/jira/browse/SOLR-12454
> Project: Solr
>  Issue Type: Task
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12454-overseer-scenario-run.patch, 
> SOLR-12454.patch, SOLR-12454.patch, overseer-scenario-run.sh
>
>
> Proposed patch to follow.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-12464) reduce Overseer.close() logging (for non-Overseer leaders)

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-12464:
---
Fix Version/s: 7.5
   master (8.0)

> reduce Overseer.close() logging (for non-Overseer leaders)
> --
>
> Key: SOLR-12464
> URL: https://issues.apache.org/jira/browse/SOLR-12464
> Project: Solr
>  Issue Type: Task
>  Components: logging
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12464-ea.patch, SOLR-12464.patch
>
>
> ZkController.init creates an Overseer object:
> * If that object becomes overseer leader then its start method is called.
> * If the object does not become overseer leader then its close method is 
> called and that currently emits mildly confusing {{Overseer (id=null) 
> closing}} info logging -- confusing especially since this happens at node 
> startup.
> ZkController.init creates an Overseer object:
> * If that object becomes overseer leader then its start method is called and 
> (if assertions are enabled) ObjectReleaseTracker.track is called.
> * If the object does not become overseer leader then its close method is 
> called and (if assertions are enabled) ObjectReleaseTracker.release is called 
> despite ObjectReleaseTracker.track not having been called previously.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-12454) tweak Overseer leadership transition related logging

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12454?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-12454:
---
Fix Version/s: 7.5
   master (8.0)

> tweak Overseer leadership transition related logging
> 
>
> Key: SOLR-12454
> URL: https://issues.apache.org/jira/browse/SOLR-12454
> Project: Solr
>  Issue Type: Task
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12454-overseer-scenario-run.patch, 
> SOLR-12454.patch, SOLR-12454.patch, overseer-scenario-run.sh
>
>
> Proposed patch to follow.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12464) reduce Overseer.close() logging (for non-Overseer leaders)

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12464?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547025#comment-16547025
 ] 

ASF subversion and git services commented on SOLR-12464:


Commit bb57871b4681559bc736c93f6a172e7cf3eca81b in lucene-solr's branch 
refs/heads/branch_7x from [~cpoerschke]
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=bb57871 ]

SOLR-12464: Reduce Overseer.close() logging (for non-Overseer leaders)


> reduce Overseer.close() logging (for non-Overseer leaders)
> --
>
> Key: SOLR-12464
> URL: https://issues.apache.org/jira/browse/SOLR-12464
> Project: Solr
>  Issue Type: Task
>  Components: logging
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Attachments: SOLR-12464-ea.patch, SOLR-12464.patch
>
>
> ZkController.init creates an Overseer object:
> * If that object becomes overseer leader then its start method is called.
> * If the object does not become overseer leader then its close method is 
> called and that currently emits mildly confusing {{Overseer (id=null) 
> closing}} info logging -- confusing especially since this happens at node 
> startup.
> ZkController.init creates an Overseer object:
> * If that object becomes overseer leader then its start method is called and 
> (if assertions are enabled) ObjectReleaseTracker.track is called.
> * If the object does not become overseer leader then its close method is 
> called and (if assertions are enabled) ObjectReleaseTracker.release is called 
> despite ObjectReleaseTracker.track not having been called previously.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12454) tweak Overseer leadership transition related logging

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12454?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547026#comment-16547026
 ] 

ASF subversion and git services commented on SOLR-12454:


Commit df08f56420738aa38b8db011ef956b3b488b86c5 in lucene-solr's branch 
refs/heads/branch_7x from [~cpoerschke]
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=df08f56 ]

SOLR-12454: Tweak Overseer leadership transition related logging for easier 
troubleshooting.


> tweak Overseer leadership transition related logging
> 
>
> Key: SOLR-12454
> URL: https://issues.apache.org/jira/browse/SOLR-12454
> Project: Solr
>  Issue Type: Task
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Attachments: SOLR-12454-overseer-scenario-run.patch, 
> SOLR-12454.patch, SOLR-12454.patch, overseer-scenario-run.sh
>
>
> Proposed patch to follow.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-7.x-Windows (64bit/jdk-10.0.1) - Build # 696 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Windows/696/
Java: 64bit/jdk-10.0.1 -XX:+UseCompressedOops -XX:+UseSerialGC

54 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.solr.analytics.facet.PivotFacetTest

Error Message:
Solr servers failed to register with ZK. Current count: 1; Expected count: 4

Stack Trace:
java.lang.IllegalStateException: Solr servers failed to register with ZK. 
Current count: 1; Expected count: 4
at __randomizedtesting.SeedInfo.seed([ECF4A64484E5B9CD]:0)
at 
org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:283)
at 
org.apache.solr.cloud.MiniSolrCloudCluster.(MiniSolrCloudCluster.java:263)
at 
org.apache.solr.cloud.SolrCloudTestCase$Builder.build(SolrCloudTestCase.java:198)
at 
org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
at 
org.apache.solr.analytics.SolrAnalyticsTestCase.setupCollection(SolrAnalyticsTestCase.java:60)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  
junit.framework.TestSuite.org.apache.solr.analytics.facet.PivotFacetTest

Error Message:
ObjectTracker found 17 object(s) that were not released!!! [InternalHttpClient, 
ZkController, SolrZkClient, InternalHttpClient, InternalHttpClient, 
InternalHttpClient, Overseer, InternalHttpClient, InternalHttpClient, 
InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, 
SolrZkClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.http.impl.client.InternalHttpClient  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319)
  at 
org.apache.solr.update.UpdateShardHandler.(UpdateShardHandler.java:113)  
at org.apache.solr.core.CoreContainer.load(CoreContainer.java:543)  at 
org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252)
  at 
org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172)  
at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at 
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)  
at 
org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477)
  at 
org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542)
  at 
org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186)
  at 

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Dawid Weiss (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547019#comment-16547019
 ] 

Dawid Weiss commented on LUCENE-8406:
-

Unfortunately we have to deal with Windows systems, so it'd have the advantage 
of working there. We create lots of quick throw-away indexes and syncing to 
persistent storage for these scenarios hurts.

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8405) Remove TopHits.maxScore

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8405?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547016#comment-16547016
 ] 

David Smiley commented on LUCENE-8405:
--

+1 nice simplification.  Solr changes looks good to me, though I have one 
question.  Why does MaxScoreCollector.scoreMode() return COMPLETE instead of 
TOP_SCORES?
{quote}That said I think Solr should probably remove the ability to get the 
maximum score when sorting by a field for the reasons mentioned above.
{quote}
Yes, feel free to just do that if you find it convenient.  

 

> Remove TopHits.maxScore
> ---
>
> Key: LUCENE-8405
> URL: https://issues.apache.org/jira/browse/LUCENE-8405
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Adrien Grand
>Priority: Major
> Fix For: master (8.0)
>
> Attachments: LUCENE-8405.patch, LUCENE-8405.patch
>
>
> I would like to propose removing TopDocs.maxScore. The reasoning is that 
> either you are sorting by score and then its value is easy to access via the 
> score of the best hit. Or you sort by one or more fields and computing it is 
> wasteful:
>  - term frequencies and norms need to be read and decoded for every match
>  - scores need to be computed on every match
>  - early-termination optimizations are disabled
> It would be more efficient to collect hits twice: once with scores disabled 
> to get the top hits, and once to get the best score which would run 
> efficiently thanks to impacts and MAXSCORE, especially with a size of 1:
> {code:java}
> TopDocs topHits = searcher.search(query, 1);
> float maxScore = topHits.scoreDocs.length == 0 ? Float.NaN : 
> topHits.scoreDocs[0].score;
> {code}
> The {{doDocScores}} option of TopFieldCollector has drawbacks as well but at 
> least doesn't disable early-termination optimizations and doesn't require 
> scores to be computed on every hit.
> As this would be a significant breaking change, I'm targeting 8.0.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Robert Muir (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547014#comment-16547014
 ] 

Robert Muir commented on LUCENE-8406:
-

{quote}
It'd be very temping (for me, personally) to have a RAMDirectory that could 
allocate larger block chunks outside of the heap (in direct memory pools)
{quote}

How much better would it be than mmapdir over tmpfs?

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Explore alternative build systems - LUCENE-5755

[Dawid Weiss]

> [...] things easier, then I don't think we should change it.  Dependency
> management is only a small part of what the build system does.

Well said. While I love Gradle's flexibility and power, complex built
scripts (like Elastic's) are a
headache to debug. Ant is a rough hammer I can understand most of the
time, although it comes with its own quirks.

> It would indeed be a big job.  If you tackle it, I would suggest setting
> it up so it exists in parallel with the ant build system for at least
> all of 8.x.  In the interests of stability, I don't think the work
> should affect branch_7x.

There's also a working (?) Maven build that is parallel with the one
we have in Ant. So it's definitely possible.

D.

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Dawid Weiss (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547006#comment-16547006
 ] 

Dawid Weiss commented on LUCENE-8406:
-

Thanks for comments. 

My initial experiment with implementing an alternative store for RamDirectory 
was very local (out of Solr/Lucene codebase) -- I modified RAMOutputStream and 
the corresponding input (in various ways) so that it assumed write-once mode 
(much like the index is written). I essentially don't allow concurrent readers 
of a RamDirectory file -- once the file is written and flushed, it is available 
for readers, but not before then. 

I then looked at modifying this in the codebase and the current complexity of 
those classes (and the congestion on locking) is a result of how these classes 
are used in many other places (as temporary buffers, essentially). This would 
have to be cleaned up first, I think, and there are comments in the code (by 
Mike) about proliferation of "writeable byte pool" classes for which the common 
functionality should perhaps be extracted and then reused. Namely: PagedBytes, 
ByteBlockPool,BytesStore ByteArrayDataOutput,GrowableByteArrayDataOutput, 
RAMIndexInput/Output... Perhaps more. They're not always identical, but there's 
definitely a recurring pattern. 

I'll try to chip at all this slowly, time permitting.

The initial look at the code also brought the reflection that if we make 
BBGuard, buffer management and the cleaner interface public but *not* make the 
buffer's native cleaner available we will force anyone downstream to reinvent 
the wheel Uwe has been so patient to figure out (different ways to clean up 
buffers in different JVM versions). If we do make that MMap's cleaner 
accessible we open up a lot of internal workings of how Lucene handles this 
stuff, which isn't good either, so I'm at crossroads here.

Perhaps I can start small by trying to clean up those RAMDirectory streams. 
It'd be very temping (for me, personally) to have a RAMDirectory that could 
allocate larger block chunks outside of the heap (in direct memory pools) -- if 
I can think of making it cleanly within the class then perhaps package-private 
scope for everything else is fine and one still has the flexibility of working 
with native buffers without caring about the details. 

I'll try to take a look at all this, although the number of references to 
ramdirectory from pretty much everywhere is sort of scary.





> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct 

[jira] [Commented] (SOLR-12454) tweak Overseer leadership transition related logging

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12454?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547001#comment-16547001
 ] 

ASF subversion and git services commented on SOLR-12454:


Commit e9c6effd07f5bd14ce87c8cea6ced64a33d8d6b0 in lucene-solr's branch 
refs/heads/master from [~cpoerschke]
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=e9c6eff ]

SOLR-12454: Tweak Overseer leadership transition related logging for easier 
troubleshooting.


> tweak Overseer leadership transition related logging
> 
>
> Key: SOLR-12454
> URL: https://issues.apache.org/jira/browse/SOLR-12454
> Project: Solr
>  Issue Type: Task
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Attachments: SOLR-12454-overseer-scenario-run.patch, 
> SOLR-12454.patch, SOLR-12454.patch, overseer-scenario-run.sh
>
>
> Proposed patch to follow.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12464) reduce Overseer.close() logging (for non-Overseer leaders)

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12464?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16547000#comment-16547000
 ] 

ASF subversion and git services commented on SOLR-12464:


Commit f7aed2876076aa489bc227d6d8d2e8c6ac96d8a0 in lucene-solr's branch 
refs/heads/master from [~cpoerschke]
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=f7aed28 ]

SOLR-12464: Reduce Overseer.close() logging (for non-Overseer leaders)


> reduce Overseer.close() logging (for non-Overseer leaders)
> --
>
> Key: SOLR-12464
> URL: https://issues.apache.org/jira/browse/SOLR-12464
> Project: Solr
>  Issue Type: Task
>  Components: logging
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Attachments: SOLR-12464-ea.patch, SOLR-12464.patch
>
>
> ZkController.init creates an Overseer object:
> * If that object becomes overseer leader then its start method is called.
> * If the object does not become overseer leader then its close method is 
> called and that currently emits mildly confusing {{Overseer (id=null) 
> closing}} info logging -- confusing especially since this happens at node 
> startup.
> ZkController.init creates an Overseer object:
> * If that object becomes overseer leader then its start method is called and 
> (if assertions are enabled) ObjectReleaseTracker.track is called.
> * If the object does not become overseer leader then its close method is 
> called and (if assertions are enabled) ObjectReleaseTracker.release is called 
> despite ObjectReleaseTracker.track not having been called previously.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-9962) need to extend classes in org.apache.solr.client.solrj.io.stream.metrics package

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-9962?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-9962:
--
Fix Version/s: (was: 6.7)
   (was: 7.0)

> need to extend classes in org.apache.solr.client.solrj.io.stream.metrics 
> package
> 
>
> Key: SOLR-9962
> URL: https://issues.apache.org/jira/browse/SOLR-9962
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: Server
>Affects Versions: 6.3
>Reporter: radhakrishnan devarajan
>Priority: Trivial
>
> i want to extend the update(Tuple tuple) method in MaxMetric,. MinMetric, 
> SumMetric, MeanMetric classes.
> can you please make the below metioned variables and methods in the above 
> mentioned classes as protected so that it will be easy to extend
> variables
> ---
> longMax
> doubleMax
> columnName
> and 
> methods
> ---
> init



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-10415) Within solr-core, debug/trace level logging should use parameterized log messages

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-10415?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-10415:
---
Component/s: logging

> Within solr-core, debug/trace level logging should use parameterized log 
> messages
> -
>
> Key: SOLR-10415
> URL: https://issues.apache.org/jira/browse/SOLR-10415
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: logging
>Reporter: Michael Braun
>Priority: Trivial
>
> Noticed in several samplings of an active Solr that several debug statements 
> were taking decently measurable time because of the time of the .toString 
> even when the log.debug() statement would not output because it was 
> effectively INFO or higher. Using parameterized logging statements, ie 
> 'log.debug("Blah {}", o)' will avoid incurring that cost.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-11274) Unit test exception of TestEdisMaxSolrFeature

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-11274?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-11274:
---
Component/s: contrib - LTR

> Unit test exception of TestEdisMaxSolrFeature
> -
>
> Key: SOLR-11274
> URL: https://issues.apache.org/jira/browse/SOLR-11274
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: contrib - LTR
>Affects Versions: 6.6
>Reporter: chenmin
>Priority: Major
>
> ERROR :
> org.apache.solr.common.SolrException: ERROR: [doc=6] unknown field 
> 'description'!
> I found the unit test with the wrong configuration file (schema)
> Compiler Environment： eclipse,windows。



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12412) Leader should give up leadership when IndexWriter.tragedy occur

[Varun Thacker (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12412?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546974#comment-16546974
 ] 

Varun Thacker commented on SOLR-12412:
--

Hi Dat,

The Jira description reads  "When a leader meets some kind of unrecoverable 
exception (ie: CorruptedIndexException). The shard will go into the readable 
state and human has to intervene. In that case, it will be the best if the 
leader gives up its leadership and let other replicas become the leader."

 

But in the test we are asserting this?
{code:java}
assertEquals(leader.getName(), oldLeader.getName());{code}
 

I had a question that I posted yesterday , reposting it for reference :

testOtherReplicasAreNotActive() -> When there are two replicas , where are we 
actually checking if it becomes active or not after it has been started again? 
i.e after this statement should we be checking if it becomes active and fail 
the test?
{code:java}
if (otherReplicaJetty != null) {
// won't be able to do anything here, since this replica can't recovery from 
the leader
otherReplicaJetty.start();
}{code}

> Leader should give up leadership when IndexWriter.tragedy occur
> ---
>
> Key: SOLR-12412
> URL: https://issues.apache.org/jira/browse/SOLR-12412
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Cao Manh Dat
>Assignee: Cao Manh Dat
>Priority: Major
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12412.patch, SOLR-12412.patch, 
> jenkins-failure-2325.log
>
>
> When a leader meets some kind of unrecoverable exception (ie: 
> CorruptedIndexException). The shard will go into the readable state and human 
> has to intervene. In that case, it will be the best if the leader gives up 
> its leadership and let other replicas become the leader. 



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12559) FunctionQParser.FLAG_USE_FIELDNAME_SOURCE doesn't work when subparsers are involved

[Hoss Man (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12559?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546963#comment-16546963
 ] 

Hoss Man commented on SOLR-12559:
-

No trivially fix jumped out at me, because the "flags" used by FunctionQParser 
aren't an object property that can be set on the subprocessor ... they are only 
passed as a param directly to {{parseValueSource}}

> FunctionQParser.FLAG_USE_FIELDNAME_SOURCE doesn't work when subparsers are 
> involved
> ---
>
> Key: SOLR-12559
> URL: https://issues.apache.org/jira/browse/SOLR-12559
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: Facet Module
>Reporter: Hoss Man
>Priority: Major
>
> While working on a patch dealing with json facet syntax parsing, i was trying 
> to write a test verifying the equivalence of 2 json facet requests that 
> should be identical and discovered that when SOLR-10613 was implemented to 
> defer the parsing of field names via {{FieldNameValueSource}} the 
> implementation did not account for the invocation of sub parsers via param 
> references.
> specifically -- this json facet request will produce two AggValueSources 
> which are not {{equals()}}...
> {noformat}
> curl http://localhost:8983/solr/query -d 'q=*:*_field=foo_i&
> json.facet={
>   x : "min(foo_i)",
>   y : "min($my_field)"
> }'
> {noformat}
> "x" uses {{FieldNameValueSource}} while "y" directly uses an 
> {{IntValueSource}}
> (It's not immediately obvious to me if this currently causes any user visible 
> bugs or performance hicups, but it will definitely be problematic for users 
> once we add support for {{min(multivalued_field_i)}} )



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (SOLR-12559) FunctionQParser.FLAG_USE_FIELDNAME_SOURCE doesn't work when subparsers are involved

[Hoss Man (JIRA)]

Hoss Man created SOLR-12559:
---

 Summary: FunctionQParser.FLAG_USE_FIELDNAME_SOURCE doesn't work 
when subparsers are involved
 Key: SOLR-12559
 URL: https://issues.apache.org/jira/browse/SOLR-12559
 Project: Solr
  Issue Type: Bug
  Security Level: Public (Default Security Level. Issues are Public)
  Components: Facet Module
Reporter: Hoss Man


While working on a patch dealing with json facet syntax parsing, i was trying 
to write a test verifying the equivalence of 2 json facet requests that should 
be identical and discovered that when SOLR-10613 was implemented to defer the 
parsing of field names via {{FieldNameValueSource}} the implementation did not 
account for the invocation of sub parsers via param references.

specifically -- this json facet request will produce two AggValueSources which 
are not {{equals()}}...

{noformat}
curl http://localhost:8983/solr/query -d 'q=*:*_field=foo_i&
json.facet={
  x : "min(foo_i)",
  y : "min($my_field)"
}'
{noformat}

"x" uses {{FieldNameValueSource}} while "y" directly uses an {{IntValueSource}}

(It's not immediately obvious to me if this currently causes any user visible 
bugs or performance hicups, but it will definitely be problematic for users 
once we add support for {{min(multivalued_field_i)}} )



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_172) - Build # 2338 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/2338/
Java: 32bit/jdk1.8.0_172 -client -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  
org.apache.solr.handler.component.InfixSuggestersTest.testShutdownDuringBuild

Error Message:
junit.framework.AssertionFailedError: Unexpected wrapped exception type, 
expected CoreIsClosedException

Stack Trace:
java.util.concurrent.ExecutionException: junit.framework.AssertionFailedError: 
Unexpected wrapped exception type, expected CoreIsClosedException
at 
__randomizedtesting.SeedInfo.seed([8FFBE1FBBF9FD0F3:5076834481F68591]:0)
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at 
org.apache.solr.handler.component.InfixSuggestersTest.testShutdownDuringBuild(InfixSuggestersTest.java:130)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: junit.framework.AssertionFailedError: Unexpected wrapped 

[jira] [Updated] (SOLR-12524) CdcrBidirectionalTest.testBiDir() regularly fails

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12524?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-12524:
---
Component/s: Tests
 CDCR

> CdcrBidirectionalTest.testBiDir() regularly fails
> -
>
> Key: SOLR-12524
> URL: https://issues.apache.org/jira/browse/SOLR-12524
> Project: Solr
>  Issue Type: Test
>  Components: CDCR, Tests
>Reporter: Christine Poerschke
>Priority: Major
> Attachments: SOLR-12524.patch, SOLR-12524.patch
>
>
> e.g. from 
> https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4701/consoleText
> {code}
> [junit4] ERROR   20.4s J0 | CdcrBidirectionalTest.testBiDir <<<
> [junit4]> Throwable #1: 
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
> uncaught exception in thread: Thread[id=28371, 
> name=cdcr-replicator-11775-thread-1, state=RUNNABLE, 
> group=TGRP-CdcrBidirectionalTest]
> [junit4]> at 
> __randomizedtesting.SeedInfo.seed([CA5584AC7009CD50:8F8E744E68278112]:0)
> [junit4]> Caused by: java.lang.AssertionError
> [junit4]> at 
> __randomizedtesting.SeedInfo.seed([CA5584AC7009CD50]:0)
> [junit4]> at 
> org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
> [junit4]> at 
> org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:125)
> [junit4]> at 
> org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
> [junit4]> at 
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
> [junit4]> at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> [junit4]> at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> [junit4]> at java.lang.Thread.run(Thread.java:748)
> {code}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12343) JSON Field Facet refinement can return incorrect counts/stats for sorted buckets

[Lucene/Solr QA (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12343?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546927#comment-16546927
 ] 

Lucene/Solr QA commented on SOLR-12343:
---

| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green}  0m 
 0s{color} | {color:green} The patch appears to include 3 new or modified test 
files. {color} |
|| || || || {color:brown} master Compile Tests {color} ||
| {color:green}+1{color} | {color:green} compile {color} | {color:green}  2m 
21s{color} | {color:green} master passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} compile {color} | {color:green}  2m 
10s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green}  2m 
10s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Release audit (RAT) {color} | 
{color:green}  2m 18s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Check forbidden APIs {color} | 
{color:green}  2m 10s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Validate source patterns {color} | 
{color:green}  2m 10s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Validate ref guide {color} | 
{color:green}  2m 10s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red}179m 55s{color} 
| {color:red} core in the patch failed. {color} |
| {color:black}{color} | {color:black} {color} | {color:black}189m 14s{color} | 
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| Failed junit tests | solr.cloud.autoscaling.IndexSizeTriggerTest |
|   | solr.cloud.api.collections.ShardSplitTest |
|   | solr.cloud.autoscaling.sim.TestGenericDistributedQueue |
|   | solr.handler.component.InfixSuggestersTest |
\\
\\
|| Subsystem || Report/Notes ||
| JIRA Issue | SOLR-12343 |
| JIRA Patch URL | 
https://issues.apache.org/jira/secure/attachment/12931845/SOLR-12343.patch |
| Optional Tests |  compile  javac  unit  ratsources  checkforbiddenapis  
validatesourcepatterns  validaterefguide  |
| uname | Linux lucene1-us-west 3.13.0-88-generic #135-Ubuntu SMP Wed Jun 8 
21:10:42 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | ant |
| Personality | 
/home/jenkins/jenkins-slave/workspace/PreCommit-SOLR-Build/sourcedir/dev-tools/test-patch/lucene-solr-yetus-personality.sh
 |
| git revision | master / d730c8b |
| ant | version: Apache Ant(TM) version 1.9.3 compiled on April 8 2014 |
| Default Java | 1.8.0_172 |
| unit | 
https://builds.apache.org/job/PreCommit-SOLR-Build/144/artifact/out/patch-unit-solr_core.txt
 |
|  Test Results | 
https://builds.apache.org/job/PreCommit-SOLR-Build/144/testReport/ |
| modules | C: solr/core solr/solr-ref-guide U: solr |
| Console output | 
https://builds.apache.org/job/PreCommit-SOLR-Build/144/console |
| Powered by | Apache Yetus 0.7.0   http://yetus.apache.org |


This message was automatically generated.



> JSON Field Facet refinement can return incorrect counts/stats for sorted 
> buckets
> 
>
> Key: SOLR-12343
> URL: https://issues.apache.org/jira/browse/SOLR-12343
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Hoss Man
>Assignee: Yonik Seeley
>Priority: Major
> Attachments: SOLR-12343.patch, SOLR-12343.patch, SOLR-12343.patch, 
> SOLR-12343.patch, SOLR-12343.patch, SOLR-12343.patch, SOLR-12343.patch, 
> SOLR-12343.patch, SOLR-12343.patch, __incomplete_processEmpty_microfix.patch
>
>
> The way JSON Facet's simple refinement "re-sorts" buckets after refinement 
> can cause _refined_ buckets to be "bumped out" of the topN based on the 
> refined counts/stats depending on the sort - causing _unrefined_ buckets 
> originally discounted in phase#2 to bubble up into the topN and be returned 
> to clients *with inaccurate counts/stats*
> The simplest way to demonstrate this bug (in some data sets) is with a 
> {{sort: 'count asc'}} facet:
>  * assume shard1 returns termX & termY in phase#1 because they have very low 
> shard1 counts
>  ** but *not* returned at all by shard2, because these terms both have very 
> high shard2 counts.
>  * Assume termX has a slightly lower shard1 count then termY, such that:
>  ** termX "makes the cut" off for the limit=N topN buckets
>  ** termY does not make the cut, and is the "N+1" known bucket at the end of 
> phase#1
>  * termX then gets included in the 

[jira] [Commented] (SOLR-12489) Restore collection does not respect user specified replicationFactor

[Steve Rowe (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12489?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546910#comment-16546910
 ] 

Steve Rowe commented on SOLR-12489:
---

Policeman Jenkins found a reproducing seed for two test failures 
[https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/22467]; {{git 
bisect}} blames both on commit {{3d20e89}} on this issue:

{noformat}
Checking out Revision ae3929c3ed4adad9fa2842b32e9a7fb28699fef7 
(refs/remotes/origin/master)
[...]
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestLocalFSCloudBackupRestore -Dtests.method=test 
-Dtests.seed=BC266A8BBC96BD8C -Dtests.multiplier=3 -Dtests.slow=true 
-Dtests.locale=es-BR -Dtests.timezone=Asia/Kolkata -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 10.0s J0 | TestLocalFSCloudBackupRestore.test <<<
   [junit4]> Throwable #1: java.lang.AssertionError: expected: 
but was:
   [junit4]>at 
__randomizedtesting.SeedInfo.seed([BC266A8BBC96BD8C:34725551126AD074]:0)
   [junit4]>at 
org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:327)
   [junit4]>at 
org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:145)
   [junit4]>at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]>at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]>at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]>at 
java.base/java.lang.reflect.Method.invoke(Method.java:564)
   [junit4]>at java.base/java.lang.Thread.run(Thread.java:844)
[...]
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{shard_s=FST50, id=PostingsFormat(name=MockRandom)}, 
docValues:{_version_=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1295, 
maxMBSortInHeap=5.038487185282289, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@2ec58f02),
 locale=es-BR, timezone=Asia/Kolkata
   [junit4]   2> NOTE: Linux 4.15.0-24-generic amd64/Oracle Corporation 9.0.4 
(64-bit)/cpus=8,threads=1,free=181310592,total=518979584
{noformat}

{noformat}
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestHdfsCloudBackupRestore -Dtests.method=test 
-Dtests.seed=BC266A8BBC96BD8C -Dtests.multiplier=3 -Dtests.slow=true 
-Dtests.locale=es-GT -Dtests.timezone=America/Phoenix -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 11.0s J0 | TestHdfsCloudBackupRestore.test <<<
   [junit4]> Throwable #1: java.lang.AssertionError: expected: 
but was:
   [junit4]>at 
__randomizedtesting.SeedInfo.seed([BC266A8BBC96BD8C:34725551126AD074]:0)
   [junit4]>at 
org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:327)
   [junit4]>at 
org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:145)
   [junit4]>at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]>at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]>at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]>at 
java.base/java.lang.reflect.Method.invoke(Method.java:564)
   [junit4]>at java.base/java.lang.Thread.run(Thread.java:844)
{noformat}


> Restore collection does not respect user specified replicationFactor
> 
>
> Key: SOLR-12489
> URL: https://issues.apache.org/jira/browse/SOLR-12489
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Varun Thacker
>Assignee: Varun Thacker
>Priority: Major
>  Labels: Backup/Restore
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12489.patch
>
>
> When restoring a collection we can pass in the replicationFactor 
> However while restoring the collection we don't make use of this param and 
> end up using whatever is present  as the nrtReplicas key in the state.json
>  
> {code:java}
> int numNrtReplicas = getInt(message, NRT_REPLICAS, 
> backupCollectionState.getNumNrtReplicas(), 0);
> if (numNrtReplicas == 0) {
>   numNrtReplicas = getInt(message, REPLICATION_FACTOR, 
> backupCollectionState.getReplicationFactor(), 0);
> }{code}
> The tests didn't catch this as the create collection call from 

[jira] [Resolved] (LUCENE-8263) Add indexPctDeletedTarget as a parameter to TieredMergePolicy to control more aggressive merging

[Adrien Grand (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8263?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Adrien Grand resolved LUCENE-8263.
--
   Resolution: Fixed
Fix Version/s: 7.5
   master (8.0)

> Add indexPctDeletedTarget as a parameter to TieredMergePolicy to control more 
> aggressive merging
> 
>
> Key: LUCENE-8263
> URL: https://issues.apache.org/jira/browse/LUCENE-8263
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Erick Erickson
>Assignee: Erick Erickson
>Priority: Major
> Fix For: master (8.0), 7.5
>
> Attachments: LUCENE-8263.patch
>
>
> Spinoff of LUCENE-7976 to keep the two issues separate.
> The current TMP allows up to 50% deleted docs, which can be wasteful on large 
> indexes. This parameter will do more aggressive merging of segments with 
> deleted documents when the _total_ percentage of deleted docs in the entire 
> index exceeds it.
> Setting this to 50% should approximate current behavior. Setting it to 20% 
> caused the first cut at this to increase I/O roughly 10%. Setting it to 10% 
> caused about a 50% increase in I/O.
> I was conflating the two issues, so I'll change 7976 and comment out the bits 
> that reference this new parameter. After it's checked in we can bring this 
> back. That should be less work than reconstructing this later.
> Among the questions to be answered:
> 1> what should the default be? I propose 20% as it results in significantly 
> less space wasted and helps control heap usage for a modest increase in I/O.
> 2> what should the floor be? I propose 10% with _strong_ documentation 
> warnings about not setting it below 20%.
> 3> should there be two parameters? I think this was discussed somewhat in 
> 7976. The first cut at  this used this number for two purposes:
> 3a> the total percentage of deleted docs index-wide to trip this trigger
> 3b> the percentage of an _individual_ segment that had to be deleted if the 
> segment was over maxSegmentSize/2 bytes in order to be eligible for merging. 
> Empirically, using the same percentage for both caused the merging to hover 
> around the value specified for this parameter.
> My proposal for <3> would be to have the parameter do double-duty. Assuming 
> my preliminary results hold, you specify this parameter at, say, 20% and once 
> the index hits that % deleted docs it hovers right around there, even if 
> you've forceMerged earlier down to 1 segment. This seems in line with what 
> I'd expect and adding another parameter seems excessively complicated to no 
> good purpose. We could always add something like that later if we wanted.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8263) Add indexPctDeletedTarget as a parameter to TieredMergePolicy to control more aggressive merging

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8263?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546870#comment-16546870
 ] 

ASF subversion and git services commented on LUCENE-8263:
-

Commit 8875402694b42791e18da2d3cf44a4353c676e26 in lucene-solr's branch 
refs/heads/branch_7x from [~jpountz]
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=8875402 ]

LUCENE-8263: Replace TieredMergePolicy's reclaimDeletesWeight with 
deletesPctAllowed.


> Add indexPctDeletedTarget as a parameter to TieredMergePolicy to control more 
> aggressive merging
> 
>
> Key: LUCENE-8263
> URL: https://issues.apache.org/jira/browse/LUCENE-8263
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Erick Erickson
>Assignee: Erick Erickson
>Priority: Major
> Attachments: LUCENE-8263.patch
>
>
> Spinoff of LUCENE-7976 to keep the two issues separate.
> The current TMP allows up to 50% deleted docs, which can be wasteful on large 
> indexes. This parameter will do more aggressive merging of segments with 
> deleted documents when the _total_ percentage of deleted docs in the entire 
> index exceeds it.
> Setting this to 50% should approximate current behavior. Setting it to 20% 
> caused the first cut at this to increase I/O roughly 10%. Setting it to 10% 
> caused about a 50% increase in I/O.
> I was conflating the two issues, so I'll change 7976 and comment out the bits 
> that reference this new parameter. After it's checked in we can bring this 
> back. That should be less work than reconstructing this later.
> Among the questions to be answered:
> 1> what should the default be? I propose 20% as it results in significantly 
> less space wasted and helps control heap usage for a modest increase in I/O.
> 2> what should the floor be? I propose 10% with _strong_ documentation 
> warnings about not setting it below 20%.
> 3> should there be two parameters? I think this was discussed somewhat in 
> 7976. The first cut at  this used this number for two purposes:
> 3a> the total percentage of deleted docs index-wide to trip this trigger
> 3b> the percentage of an _individual_ segment that had to be deleted if the 
> segment was over maxSegmentSize/2 bytes in order to be eligible for merging. 
> Empirically, using the same percentage for both caused the merging to hover 
> around the value specified for this parameter.
> My proposal for <3> would be to have the parameter do double-duty. Assuming 
> my preliminary results hold, you specify this parameter at, say, 20% and once 
> the index hits that % deleted docs it hovers right around there, even if 
> you've forceMerged earlier down to 1 segment. This seems in line with what 
> I'd expect and adding another parameter seems excessively complicated to no 
> good purpose. We could always add something like that later if we wanted.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-master-Windows (64bit/jdk-10) - Build # 7423 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Windows/7423/
Java: 64bit/jdk-10 -XX:+UseCompressedOops -XX:+UseSerialGC

42 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: 
http://127.0.0.1:52280/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting 
response from server at: http://127.0.0.1:52280/collection1
at 
__randomizedtesting.SeedInfo.seed([57256029A76156DB:DF715FF3099D3B23]:0)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591)
at 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:213)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 

[jira] [Commented] (LUCENE-8263) Add indexPctDeletedTarget as a parameter to TieredMergePolicy to control more aggressive merging

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8263?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546854#comment-16546854
 ] 

ASF subversion and git services commented on LUCENE-8263:
-

Commit 8093c450c1fd22e2bfd5b2a4a1e45c6f95f16189 in lucene-solr's branch 
refs/heads/master from [~jpountz]
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=8093c45 ]

LUCENE-8263: Replace TieredMergePolicy's reclaimDeletesWeight with 
deletesPctAllowed.


> Add indexPctDeletedTarget as a parameter to TieredMergePolicy to control more 
> aggressive merging
> 
>
> Key: LUCENE-8263
> URL: https://issues.apache.org/jira/browse/LUCENE-8263
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Erick Erickson
>Assignee: Erick Erickson
>Priority: Major
> Attachments: LUCENE-8263.patch
>
>
> Spinoff of LUCENE-7976 to keep the two issues separate.
> The current TMP allows up to 50% deleted docs, which can be wasteful on large 
> indexes. This parameter will do more aggressive merging of segments with 
> deleted documents when the _total_ percentage of deleted docs in the entire 
> index exceeds it.
> Setting this to 50% should approximate current behavior. Setting it to 20% 
> caused the first cut at this to increase I/O roughly 10%. Setting it to 10% 
> caused about a 50% increase in I/O.
> I was conflating the two issues, so I'll change 7976 and comment out the bits 
> that reference this new parameter. After it's checked in we can bring this 
> back. That should be less work than reconstructing this later.
> Among the questions to be answered:
> 1> what should the default be? I propose 20% as it results in significantly 
> less space wasted and helps control heap usage for a modest increase in I/O.
> 2> what should the floor be? I propose 10% with _strong_ documentation 
> warnings about not setting it below 20%.
> 3> should there be two parameters? I think this was discussed somewhat in 
> 7976. The first cut at  this used this number for two purposes:
> 3a> the total percentage of deleted docs index-wide to trip this trigger
> 3b> the percentage of an _individual_ segment that had to be deleted if the 
> segment was over maxSegmentSize/2 bytes in order to be eligible for merging. 
> Empirically, using the same percentage for both caused the merging to hover 
> around the value specified for this parameter.
> My proposal for <3> would be to have the parameter do double-duty. Assuming 
> my preliminary results hold, you specify this parameter at, say, 20% and once 
> the index hits that % deleted docs it hovers right around there, even if 
> you've forceMerged earlier down to 1 segment. This seems in line with what 
> I'd expect and adding another parameter seems excessively complicated to no 
> good purpose. We could always add something like that later if we wanted.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-12344) SolrSlf4jReporter doesn't set MDC context

[Varun Thacker (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12344?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Varun Thacker updated SOLR-12344:
-
Description: 
I setup a slf4j reporter like this on master

solr.xml
{code:java}

  
1
UPDATE./update.requestTimes
update_logger
  
{code}
log4j2.xml
{code:java}




  

  

  %-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] 
%c; %m%n

  


  

  %-5p - %d{-MM-dd HH:mm:ss.SSS}; [%X{collection} %X{shard} 
%X{replica} %X{core}] %c; %m%n

  
  


  
  


  

  %-5p - %d{-MM-dd HH:mm:ss.SSS}; [%X{collection} %X{shard} 
%X{replica} %X{core}] %c; %m%n

  
  


  
  

  
  





  



  
  

  

{code}
The output I get from the solr_metric.log file is like this
{code:java}
INFO  - 2018-05-11 15:31:16.009; [   ] update_logger; type=TIMER, 
name=UPDATE./update.requestTimes, count=0, min=0.0, max=0.0, mean=0.0, 
stddev=0.0, median=0.0, p75=0.0, p95=0.0, p98=0.0, p99=0.0, p999=0.0, 
mean_rate=0.0, m1=0.0, m5=0.0, m15=0.0, rate_unit=events/second, 
duration_unit=milliseconds

INFO  - 2018-05-11 15:31:17.010; [   ] update_logger; type=TIMER, 
name=UPDATE./update.requestTimes, count=0, min=0.0, max=0.0, mean=0.0, 
stddev=0.0, median=0.0, p75=0.0, p95=0.0, p98=0.0, p99=0.0, p999=0.0, 
mean_rate=0.0, m1=0.0, m5=0.0, m15=0.0, rate_unit=events/second, 
duration_unit=milliseconds

INFO  - 2018-05-11 15:31:18.010; [   ] update_logger; type=TIMER, 
name=UPDATE./update.requestTimes, count=0, min=0.0, max=0.0, mean=0.0, 
stddev=0.0, median=0.0, p75=0.0, p95=0.0, p98=0.0, p99=0.0, p999=0.0, 
mean_rate=0.0, m1=0.0, m5=0.0, m15=0.0, rate_unit=events/second, 
duration_unit=milliseconds{code}
On a JVM which has multiple cores, this will become impossible to tell where 
it's coming from if MDC context is not set

  was:
I setup a slf4j reporter like this on mater

solr.xml
{code:java}

  
1
UPDATE./update.requestTimes
update_logger
  
{code}
log4j2.xml
{code:java}




  

  

  %-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] 
%c; %m%n

  


  

  %-5p - %d{-MM-dd HH:mm:ss.SSS}; [%X{collection} %X{shard} 
%X{replica} %X{core}] %c; %m%n

  
  


  
  


  

  %-5p - %d{-MM-dd HH:mm:ss.SSS}; [%X{collection} %X{shard} 
%X{replica} %X{core}] %c; %m%n

  
  


  
  

  
  





  



  
  

  

{code}
The output I get from the solr_metric.log file is like this


{code:java}
INFO  - 2018-05-11 15:31:16.009; [   ] update_logger; type=TIMER, 
name=UPDATE./update.requestTimes, count=0, min=0.0, max=0.0, mean=0.0, 
stddev=0.0, median=0.0, p75=0.0, p95=0.0, p98=0.0, p99=0.0, p999=0.0, 
mean_rate=0.0, m1=0.0, m5=0.0, m15=0.0, rate_unit=events/second, 
duration_unit=milliseconds

INFO  - 2018-05-11 15:31:17.010; [   ] update_logger; type=TIMER, 
name=UPDATE./update.requestTimes, count=0, min=0.0, max=0.0, mean=0.0, 
stddev=0.0, median=0.0, p75=0.0, p95=0.0, p98=0.0, p99=0.0, p999=0.0, 
mean_rate=0.0, m1=0.0, m5=0.0, m15=0.0, rate_unit=events/second, 
duration_unit=milliseconds

INFO  - 2018-05-11 15:31:18.010; [   ] update_logger; type=TIMER, 
name=UPDATE./update.requestTimes, count=0, min=0.0, max=0.0, mean=0.0, 
stddev=0.0, median=0.0, p75=0.0, p95=0.0, p98=0.0, p99=0.0, p999=0.0, 
mean_rate=0.0, m1=0.0, m5=0.0, m15=0.0, rate_unit=events/second, 
duration_unit=milliseconds{code}
On a JVM which has multiple cores, this will become impossible to tell where 
it's coming from if MDC context is not set


> SolrSlf4jReporter doesn't set MDC context
> -
>
> Key: SOLR-12344
> URL: https://issues.apache.org/jira/browse/SOLR-12344
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Varun Thacker
>Priority: Major
>
> I setup a slf4j reporter like this on master
> solr.xml
> {code:java}
> 
>class="org.apache.solr.metrics.reporters.SolrSlf4jReporter">
> 1
> UPDATE./update.requestTimes
> update_logger
>   
> {code}
> log4j2.xml
> {code:java}
> 
> 
> 
>   
> 
>   
> 
>   %-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} 
> %X{core}] %c; %m%n
> 
>   
> 
>  name="RollingFile"
> fileName="${sys:solr.log.dir}/solr.log"
> filePattern="${sys:solr.log.dir}/solr.log.%i" >
>   
> 
>   %-5p - %d{-MM-dd HH:mm:ss.SSS}; [%X{collection} %X{shard} 
> %X{replica} %X{core}] %c; 

[jira] [Commented] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Yonik Seeley (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546851#comment-16546851
 ] 

Yonik Seeley commented on SOLR-11598:
-

In general, we shouldn't have limits at all on stuff like this.  If the 
performance degradation and memory use is linear, there is no trap waiting to 
bite someone (except for the arbitrary limit itself).


> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
>   at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>   at 
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
>   at 
> 

[jira] [Updated] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Amrit Sarkar (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Amrit Sarkar updated SOLR-11598:

Attachment: SOLR-11598.patch

> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
>   at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>   at 
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
>   at 
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
>   at 
> 

[jira] [Commented] (SOLR-11598) Export Writer needs to support more than 4 Sort fields - Say 10, ideally it should not be bound at all, but 4 seems to really short sell the StreamRollup capabilities.

[Amrit Sarkar (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-11598?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546835#comment-16546835
 ] 

Amrit Sarkar commented on SOLR-11598:
-

Thanks Varun for re-arranging the classes, the ExportWriter class looks much 
better now.

I am attaching a patch, built on top of latest one uploaded on the Jira with 
extensive tests for testing 'n' sort fields with 1000s of documents for 
ExportWriter. Also, I made the maximum sort fields to 25, please feel free to 
change as necessary.

> Export Writer needs to support more than 4 Sort fields - Say 10, ideally it 
> should not be bound at all, but 4 seems to really short sell the StreamRollup 
> capabilities.
> ---
>
> Key: SOLR-11598
> URL: https://issues.apache.org/jira/browse/SOLR-11598
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: streaming expressions
>Affects Versions: 6.6.1, 7.0
>Reporter: Aroop
>Assignee: Varun Thacker
>Priority: Major
>  Labels: patch
> Attachments: SOLR-11598-6_6-streamtests, SOLR-11598-6_6.patch, 
> SOLR-11598-master.patch, SOLR-11598.patch, SOLR-11598.patch, 
> SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, SOLR-11598.patch, 
> streaming-export reports.xlsx
>
>
> I am a user of Streaming and I am currently trying to use rollups on an 10 
> dimensional document.
> I am unable to get correct results on this query as I am bounded by the 
> limitation of the export handler which supports only 4 sort fields.
> I do not see why this needs to be the case, as it could very well be 10 or 20.
> My current needs would be satisfied with 10, but one would want to ask why 
> can't it be any decent integer n, beyond which we know performance degrades, 
> but even then it should be caveat emptor.
> [~varunthacker] 
> Code Link:
> https://github.com/apache/lucene-solr/blob/19db1df81a18e6eb2cce5be973bf2305d606a9f8/solr/core/src/java/org/apache/solr/handler/ExportWriter.java#L455
> Error
> null:java.io.IOException: A max of 4 sorts can be specified
>   at 
> org.apache.solr.handler.ExportWriter.getSortDoc(ExportWriter.java:452)
>   at org.apache.solr.handler.ExportWriter.writeDocs(ExportWriter.java:228)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$1(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:664)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:333)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$null$2(ExportWriter.java:219)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:354)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:223)
>   at org.apache.solr.common.util.JavaBinCodec$1.put(JavaBinCodec.java:394)
>   at 
> org.apache.solr.handler.ExportWriter.lambda$write$3(ExportWriter.java:217)
>   at 
> org.apache.solr.common.util.JavaBinCodec.writeMap(JavaBinCodec.java:437)
>   at org.apache.solr.handler.ExportWriter.write(ExportWriter.java:215)
>   at org.apache.solr.core.SolrCore$3.write(SolrCore.java:2601)
>   at 
> org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49)
>   at 
> org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:809)
>   at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:538)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
>   at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
>   at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
>   at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>   at 
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
>   at 
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
>   at 
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
>   at 
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
>   

[jira] [Updated] (SOLR-12558) solr/core (private) logger renames

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12558?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-12558:
---
Attachment: SOLR-12558-part2.patch

> solr/core (private) logger renames
> --
>
> Key: SOLR-12558
> URL: https://issues.apache.org/jira/browse/SOLR-12558
> Project: Solr
>  Issue Type: Sub-task
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Attachments: SOLR-12558-part1.patch, SOLR-12558-part2.patch
>
>
> Will attach patches in two parts: part 1 to include renames involving 
> shadowing and part 2 with just the plain renames.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-12558) solr/core (private) logger renames

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12558?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-12558:
---
Attachment: SOLR-12558-part1.patch

> solr/core (private) logger renames
> --
>
> Key: SOLR-12558
> URL: https://issues.apache.org/jira/browse/SOLR-12558
> Project: Solr
>  Issue Type: Sub-task
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Attachments: SOLR-12558-part1.patch
>
>
> Will attach patches in two parts: part 1 to include renames involving 
> shadowing and part 2 with just the plain renames.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-12557) standardise solr/core (private) logger names

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12557?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke updated SOLR-12557:
---
Attachment: SOLR-12557.patch

> standardise solr/core (private) logger names
> 
>
> Key: SOLR-12557
> URL: https://issues.apache.org/jira/browse/SOLR-12557
> Project: Solr
>  Issue Type: Task
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Attachments: SOLR-12557.patch
>
>
> Standardise to {{log}} or {{LOG}} next for {{solr/core}} code, SOLR-12419 
> already standardised for {{solr/contrib}} code.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (SOLR-12558) solr/core (private) logger renames

[Christine Poerschke (JIRA)]

Christine Poerschke created SOLR-12558:
--

 Summary: solr/core (private) logger renames
 Key: SOLR-12558
 URL: https://issues.apache.org/jira/browse/SOLR-12558
 Project: Solr
  Issue Type: Sub-task
Reporter: Christine Poerschke
Assignee: Christine Poerschke


Will attach patches in two parts: part 1 to include renames involving shadowing 
and part 2 with just the plain renames.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (SOLR-12557) standardise solr/core (private) logger names

[Christine Poerschke (JIRA)]

Christine Poerschke created SOLR-12557:
--

 Summary: standardise solr/core (private) logger names
 Key: SOLR-12557
 URL: https://issues.apache.org/jira/browse/SOLR-12557
 Project: Solr
  Issue Type: Task
Reporter: Christine Poerschke
Assignee: Christine Poerschke


Standardise to {{log}} or {{LOG}} next for {{solr/core}} code, SOLR-12419 
already standardised for {{solr/contrib}} code.




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Resolved] (SOLR-12527) turn core/test TestCloudSearcherWarming.Config into test-framework/ConfigRequest

[Christine Poerschke (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-12527?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Christine Poerschke resolved SOLR-12527.

   Resolution: Fixed
Fix Version/s: 7.5
   master (8.0)

> turn core/test TestCloudSearcherWarming.Config into 
> test-framework/ConfigRequest
> 
>
> Key: SOLR-12527
> URL: https://issues.apache.org/jira/browse/SOLR-12527
> Project: Solr
>  Issue Type: Test
>Reporter: Christine Poerschke
>Assignee: Christine Poerschke
>Priority: Minor
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12527.patch
>
>
> Tests can use this class e.g. to add a custom component or handler to an 
> otherwise generic configset.
> [CustomHighlightComponentTest.java#L138-L171|https://github.com/apache/lucene-solr/blob/releases/lucene-solr/7.4.0/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java#L138-L171]
>  illustrates the approach. 
> https://lucene.apache.org/solr/guide/7_4/config-api.html is the Solr 
> Reference Guide's Config API section.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Solr and ZK timeout issues

[Shawn Heisey]

On 7/17/2018 4:05 AM, Dominique Bejean wrote:
I sent this question to solr-u...@lucene.apache.org 
 and I think the good one is 
dev@lucene.apache.org , so I resend it. 
Sorry for this error.


The solr-user list is more appropriate for your question than the dev 
list.  Because you're discussing source code, I know the dev list might 
seem more correct, but I promise you that it's not.  The user list is 
the correct place to discuss things that might be bugs.


Thanks,
Shawn


-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Explore alternative build systems - LUCENE-5755

[Shawn Heisey]

On 7/17/2018 3:35 AM, Diego Ceccarelli (BLOOMBERG/ LONDON) wrote:
Thanks Dawid, I agree it is a massive job, I was curious to know if 
there is interest and we can put together a group of people interested 
in doing the job.


If that work gives us a build system that is easier for everyone to 
understand and change, it would be worth pursuing.  If it doesn't make 
things easier, then I don't think we should change it.  Dependency 
management is only a small part of what the build system does.


I have no particular attachment to ant/ivy.  The current build system 
does work, and it would be important that any new build system be 
tortured to make sure it's more robustthan the ant/ivy build.


It would indeed be a big job.  If you tackle it, I would suggest setting 
it up so it exists in parallel with the ant build system for at least 
all of 8.x.  In the interests of stability, I don't think the work 
should affect branch_7x.


I really think that any significant work on the build system, whether we 
switch systems or not, should involve some detailed design and howto 
documentation, probably on the project wikis.


Thanks,
Shawn


-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Comment Edited] (LUCENE-8406) Make ByteBufferIndexInput public

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546729#comment-16546729
 ] 

Uwe Schindler edited comment on LUCENE-8406 at 7/17/18 2:57 PM:


I have no problem in making the class public, if the factory method is 
correctly documented and take an array of ByteBuffer. The unmapping an mmap 
logic is part of MMapDirectory and only available internally, so it's not risky 
to do this. It's also important to document ad add some checks to ensure that 
the slices need to have power-of-2 sizes (except the last).

But we should not make the BBGuard public, I think that can be hidden inside? 
BufferCleaner interface is fine, so somebody can hook in his own impl.


was (Author: thetaphi):
I have no problem in making the class public, if the factory method is 
correctly documented and take an array of ByteBuffer. The unmapping an mmap 
logic is part of MMapDirectory and only available internally, so it's not risky 
to do this.

But we should not make the BBGuard public, I think that can be hidden inside? 
BufferCleaner interface is fine, so somebody can hook in his own impl.

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Comment Edited] (LUCENE-8406) Make ByteBufferIndexInput public

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546729#comment-16546729
 ] 

Uwe Schindler edited comment on LUCENE-8406 at 7/17/18 2:56 PM:


I have no problem in making the class public, if the factory method is 
correctly documented and take an array of ByteBuffer. The unmapping an mmap 
logic is part of MMapDirectory and only available internally, so it's not risky 
to do this.

But we should not make the BBGuard public, I think that can be hidden inside? 
BufferCleaner interface is fine, so somebody can hook in his own impl.


was (Author: thetaphi):
I have no problem in making the class public, if the constructors are correctly 
documented and take an array of ByteBuffer. The unmapping an mmap logic is part 
of MMapDirectory and only available internally, so it's not risky to do this.

But we should not make the BBGuard public, I think that can be hidden inside? 
BufferCleaner interface is fine, so somebody can hook in his own impl.

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546734#comment-16546734
 ] 

Uwe Schindler commented on LUCENE-8406:
---

bq. Note the performance difference is an order of magnitude on this 32-CPU 
system (2 minutes vs. 9 seconds). The tiny performance difference between the 
implementation based on direct memory buffers vs. those acquired via 
ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
data via unsafe and the wrapped counterpart uses regular java array access (my 
best guess).

Should no longer be visible after Java 9 and enough warmup. Which version did 
you use?

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Uwe Schindler (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546729#comment-16546729
 ] 

Uwe Schindler commented on LUCENE-8406:
---

I have no problem in making the class public, if the constructors are correctly 
documented and take an array of ByteBuffer. The unmapping an mmap logic is part 
of MMapDirectory and only available internally, so it's not risky to do this.

But we should not make the BBGuard public, I think that can be hidden inside? 
BufferCleaner interface is fine, so somebody can hook in his own impl.

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Michael Braun (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546695#comment-16546695
 ] 

Michael Braun commented on LUCENE-8406:
---

Would love to have the performance of RAMDirectory improved if possible - when 
experimenting with Luwak (https://github.com/flaxsearch/luwak) CC 
[~romseygeek], there was noticeable performance degradation due to contention 
at the RAMFile level. Would the need for RAMFile be eliminated entirely with 
this?

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> One particular example here is RAMDirectory, which currently uses a custom 
> IndexInput implementation, which in turn reaches to RAMFile's synchronized 
> methods. This is the cause of quite dramatic congestions on multithreaded 
> systems. While we clearly discourage RAMDirectory from being used in 
> production environments, there really is no need for it to be slow. If 
> modified only slightly (to use ByteBuffer-based input), the performance is on 
> par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-7.x-Linux (64bit/jdk-10) - Build # 2337 - Still unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/2337/
Java: 64bit/jdk-10 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  
org.apache.solr.handler.component.InfixSuggestersTest.testShutdownDuringBuild

Error Message:
junit.framework.AssertionFailedError: Unexpected wrapped exception type, 
expected CoreIsClosedException

Stack Trace:
java.util.concurrent.ExecutionException: junit.framework.AssertionFailedError: 
Unexpected wrapped exception type, expected CoreIsClosedException
at 
__randomizedtesting.SeedInfo.seed([86E2389ED6A8CB02:596F5A21E8C19E60]:0)
at java.base/java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:191)
at 
org.apache.solr.handler.component.InfixSuggestersTest.testShutdownDuringBuild(InfixSuggestersTest.java:130)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Caused by: junit.framework.AssertionFailedError: Unexpected wrapped 

[jira] [Commented] (LUCENE-8306) Allow iteration over the term positions of a Match

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8306?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546660#comment-16546660
 ] 

David Smiley commented on LUCENE-8306:
--

I did a little review:
 * Shouldn't label() return the associated Query?  If not then how might a 
highlighter do anything useful with it?  The only thing I can imagine is simply 
recursively count the distinct label as a heuristic a match finding more or 
less terms than some other match.  Even if one could *also* do that with a 
Query, a Query seems intrinsically more useful rather than some opaque Object.
 * the changes to SloppyPhraseMatcher seemed like maybe you fixed bugs too? 
e.g. startOffset being idempotent, if I'm reading that correctly
 * In SloppyPhraseMatcher.getSubMatches, is it really necessary to build up 
this {{int[][] submatches = new int[phrasePositions.length][3];}} thing vs 
working on PhrasePositions directly?
 * Shouldn't TermMatchesIterator.getSubMatches return EMPTY? It's already a 
leaf.

> Allow iteration over the term positions of a Match
> --
>
> Key: LUCENE-8306
> URL: https://issues.apache.org/jira/browse/LUCENE-8306
> Project: Lucene - Core
>  Issue Type: New Feature
>Reporter: Alan Woodward
>Assignee: Alan Woodward
>Priority: Major
> Attachments: LUCENE-8306.patch, LUCENE-8306.patch, LUCENE-8306.patch
>
>
> For multi-term queries such as phrase queries, the matches API currently just 
> returns information about the span of the whole match.  It would be useful to 
> also expose information about the matching terms within the phrase.  The same 
> would apply to Spans and Interval queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12395) Typo in SignificantTermsQParserPlugin.NAME

[Christine Poerschke (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12395?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546641#comment-16546641
 ] 

Christine Poerschke commented on SOLR-12395:


bq. Should this issue perhaps also take care of the spelling errors in the 
class names and methods themselves. ...

Fair question. I had noticed the missing {{i}} in the 
{{SignifcantTerms(QParser|Collector)}} class names too but since the classes 
[currently|https://github.com/apache/lucene-solr/blob/releases/lucene-solr/7.4.0/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java]
 are private inner classes thought it okay to not include their fixing in the 
scope of this issue. But if anyone wanted to fix those spelling errors too, I'd 
be +1 to that.

> Typo in SignificantTermsQParserPlugin.NAME
> --
>
> Key: SOLR-12395
> URL: https://issues.apache.org/jira/browse/SOLR-12395
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: query parsers
>Affects Versions: 6.5, 7.3.1
>Reporter: Tobias Kässmann
>Assignee: Christine Poerschke
>Priority: Trivial
> Fix For: master (8.0), 7.5
>
> Attachments: SOLR-12395.patch, SOLR-12395.patch, SOLR-12395.patch, 
> SOLR-12395.patch
>
>
> I think there is a small typo in the {{SignificantTermsQParserPlugin}}:
> {code:java}
> public static final String NAME = "sigificantTerms";{code}
> should be:
> {code:java}
> public static final String NAME = "significantTerms";{code}
>  See the patch attached.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12524) CdcrBidirectionalTest.testBiDir() regularly fails

[Christine Poerschke (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12524?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546630#comment-16546630
 ] 

Christine Poerschke commented on SOLR-12524:


Thanks [~sarkaramr...@gmail.com] for attaching a patch for the issue!

Unfortunately I don't yet have enough experience with this area of the code to 
meaningfully review the patch -- could someone else help?

cc/fyi [~erickerickson] w.r.t. the Badapple report.

> CdcrBidirectionalTest.testBiDir() regularly fails
> -
>
> Key: SOLR-12524
> URL: https://issues.apache.org/jira/browse/SOLR-12524
> Project: Solr
>  Issue Type: Test
>Reporter: Christine Poerschke
>Priority: Major
> Attachments: SOLR-12524.patch, SOLR-12524.patch
>
>
> e.g. from 
> https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4701/consoleText
> {code}
> [junit4] ERROR   20.4s J0 | CdcrBidirectionalTest.testBiDir <<<
> [junit4]> Throwable #1: 
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
> uncaught exception in thread: Thread[id=28371, 
> name=cdcr-replicator-11775-thread-1, state=RUNNABLE, 
> group=TGRP-CdcrBidirectionalTest]
> [junit4]> at 
> __randomizedtesting.SeedInfo.seed([CA5584AC7009CD50:8F8E744E68278112]:0)
> [junit4]> Caused by: java.lang.AssertionError
> [junit4]> at 
> __randomizedtesting.SeedInfo.seed([CA5584AC7009CD50]:0)
> [junit4]> at 
> org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
> [junit4]> at 
> org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:125)
> [junit4]> at 
> org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
> [junit4]> at 
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
> [junit4]> at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> [junit4]> at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> [junit4]> at java.lang.Thread.run(Thread.java:748)
> {code}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8405) Remove TopHits.maxScore

[Michael McCandless (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8405?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546629#comment-16546629
 ] 

Michael McCandless commented on LUCENE-8405:


+1

> Remove TopHits.maxScore
> ---
>
> Key: LUCENE-8405
> URL: https://issues.apache.org/jira/browse/LUCENE-8405
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Adrien Grand
>Priority: Major
> Fix For: master (8.0)
>
> Attachments: LUCENE-8405.patch, LUCENE-8405.patch
>
>
> I would like to propose removing TopDocs.maxScore. The reasoning is that 
> either you are sorting by score and then its value is easy to access via the 
> score of the best hit. Or you sort by one or more fields and computing it is 
> wasteful:
>  - term frequencies and norms need to be read and decoded for every match
>  - scores need to be computed on every match
>  - early-termination optimizations are disabled
> It would be more efficient to collect hits twice: once with scores disabled 
> to get the top hits, and once to get the best score which would run 
> efficiently thanks to impacts and MAXSCORE, especially with a size of 1:
> {code:java}
> TopDocs topHits = searcher.search(query, 1);
> float maxScore = topHits.scoreDocs.length == 0 ? Float.NaN : 
> topHits.scoreDocs[0].score;
> {code}
> The {{doDocScores}} option of TopFieldCollector has drawbacks as well but at 
> least doesn't disable early-termination optimizations and doesn't require 
> scores to be computed on every hit.
> As this would be a significant breaking change, I'm targeting 8.0.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS-EA] Lucene-Solr-master-Linux (64bit/jdk-11-ea+22) - Build # 22469 - Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/22469/
Java: 64bit/jdk-11-ea+22 -XX:+UseCompressedOops -XX:+UseSerialGC

63 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DocValuesNotIndexedTest

Error Message:
Collection not found: dv_coll

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: dv_coll
at __randomizedtesting.SeedInfo.seed([852354B1C05285A3]:0)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:853)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at 
org.apache.solr.cloud.DocValuesNotIndexedTest.createCluster(DocValuesNotIndexedTest.java:154)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DocValuesNotIndexedTest

Error Message:
Collection not found: dv_coll

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: dv_coll
at __randomizedtesting.SeedInfo.seed([852354B1C05285A3]:0)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:853)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at 
org.apache.solr.cloud.DocValuesNotIndexedTest.createCluster(DocValuesNotIndexedTest.java:154)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 

[jira] [Updated] (LUCENE-8406) Make ByteBufferIndexInput public

[Dawid Weiss (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Dawid Weiss updated LUCENE-8406:

Description: 
The logic of handling byte buffers splits, their proper closing (cleaner) and 
all the trickery involved in slicing, cloning and proper exception handling is 
quite daunting. 

While ByteBufferIndexInput.newInstance(..) is public, the parent class 
ByteBufferIndexInput is not. I think we should make the parent class public to 
allow advanced users to make use of this (complex) piece of code to create 
IndexInput based on a sequence of ByteBuffers.

One particular example here is RAMDirectory, which currently uses a custom 
IndexInput implementation, which in turn reaches to RAMFile's synchronized 
methods. This is the cause of quite dramatic congestions on multithreaded 
systems. While we clearly discourage RAMDirectory from being used in production 
environments, there really is no need for it to be slow. If modified only 
slightly (to use ByteBuffer-based input), the performance is on par with 
FSDirectory. Here's a sample log comparing FSDirectory with RAMDirectory and 
the "modified" RAMDirectory making use of the ByteBuffer input:

{code}
14:26:40 INFO  console: FSDirectory index.
14:26:41 INFO  console: Opened with 299943 documents.
14:26:50 INFO  console: Finished: 8.820 s, 24 matches.

14:26:50 INFO  console: RAMDirectory index.
14:26:50 INFO  console: Opened with 299943 documents.
14:28:50 INFO  console: Finished: 2.012 min, 24 matches.

14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
14:28:50 INFO  console: Opened with 299943 documents.
14:29:00 INFO  console: Finished: 9.215 s, 24 matches.

14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
14:29:00 INFO  console: Opened with 299943 documents.
14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
{code}

Note the performance difference is an order of magnitude on this 32-CPU system 
(2 minutes vs. 9 seconds). The tiny performance difference between the 
implementation based on direct memory buffers vs. those acquired via 
ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
data via unsafe and the wrapped counterpart uses regular java array access (my 
best guess).


  was:
The logic of handling byte buffers splits, their proper closing (cleaner) and 
all the trickery involved in slicing, cloning and proper exception handling is 
quite daunting. 

While ByteBufferIndexInput.newInstance(..) is public, the parent class 
ByteBufferIndexInput is not. I think we should make the parent class public to 
allow advanced users to make use of this (complex) piece of code to create 
IndexInput based on a sequence of ByteBuffers.

The specific rationale I'm aiming at here is RAMDirectory, which currently uses 
a custom IndexInput implementation, which in turn reaches to RAMFile's 
synchronized methods. This is the cause of quite dramatic congestions on 
multithreaded systems. While we clearly discourage RAMDirectory from being used 
in production environments, there really is no need for it to be slow. If 
modified only slightly (to use ByteBuffer-based input), the performance is on 
par with FSDirectory. Here's a sample log comparing FSDirectory with 
RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer input:

{code}
14:26:40 INFO  console: FSDirectory index.
14:26:41 INFO  console: Opened with 299943 documents.
14:26:50 INFO  console: Finished: 8.820 s, 24 matches.

14:26:50 INFO  console: RAMDirectory index.
14:26:50 INFO  console: Opened with 299943 documents.
14:28:50 INFO  console: Finished: 2.012 min, 24 matches.

14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
14:28:50 INFO  console: Opened with 299943 documents.
14:29:00 INFO  console: Finished: 9.215 s, 24 matches.

14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
14:29:00 INFO  console: Opened with 299943 documents.
14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
{code}

Note the performance difference is an order of magnitude on this 32-CPU system 
(2 minutes vs. 9 seconds). The tiny performance difference between the 
implementation based on direct memory buffers vs. those acquired via 
ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
data via unsafe and the wrapped counterpart uses regular java array access (my 
best guess).


> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the 

[jira] [Commented] (LUCENE-8406) Make ByteBufferIndexInput public

[Dawid Weiss (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8406?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16546576#comment-16546576
 ] 

Dawid Weiss commented on LUCENE-8406:
-

Forgot about it: the public (experimental?) API would have to include making 
ByteBufferGuard and BufferCleaner public too (similar situation: public 
methods, hidden classes).

> Make ByteBufferIndexInput public
> 
>
> Key: LUCENE-8406
> URL: https://issues.apache.org/jira/browse/LUCENE-8406
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Minor
> Fix For: 6.7
>
>
> The logic of handling byte buffers splits, their proper closing (cleaner) and 
> all the trickery involved in slicing, cloning and proper exception handling 
> is quite daunting. 
> While ByteBufferIndexInput.newInstance(..) is public, the parent class 
> ByteBufferIndexInput is not. I think we should make the parent class public 
> to allow advanced users to make use of this (complex) piece of code to create 
> IndexInput based on a sequence of ByteBuffers.
> The specific rationale I'm aiming at here is RAMDirectory, which currently 
> uses a custom IndexInput implementation, which in turn reaches to RAMFile's 
> synchronized methods. This is the cause of quite dramatic congestions on 
> multithreaded systems. While we clearly discourage RAMDirectory from being 
> used in production environments, there really is no need for it to be slow. 
> If modified only slightly (to use ByteBuffer-based input), the performance is 
> on par with FSDirectory. Here's a sample log comparing FSDirectory with 
> RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer 
> input:
> {code}
> 14:26:40 INFO  console: FSDirectory index.
> 14:26:41 INFO  console: Opened with 299943 documents.
> 14:26:50 INFO  console: Finished: 8.820 s, 24 matches.
> 14:26:50 INFO  console: RAMDirectory index.
> 14:26:50 INFO  console: Opened with 299943 documents.
> 14:28:50 INFO  console: Finished: 2.012 min, 24 matches.
> 14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
> 14:28:50 INFO  console: Opened with 299943 documents.
> 14:29:00 INFO  console: Finished: 9.215 s, 24 matches.
> 14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
> 14:29:00 INFO  console: Opened with 299943 documents.
> 14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
> {code}
> Note the performance difference is an order of magnitude on this 32-CPU 
> system (2 minutes vs. 9 seconds). The tiny performance difference between the 
> implementation based on direct memory buffers vs. those acquired via 
> ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
> data via unsafe and the wrapped counterpart uses regular java array access 
> (my best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (LUCENE-8406) Make ByteBufferIndexInput public

[Dawid Weiss (JIRA)]

Dawid Weiss created LUCENE-8406:
---

 Summary: Make ByteBufferIndexInput public
 Key: LUCENE-8406
 URL: https://issues.apache.org/jira/browse/LUCENE-8406
 Project: Lucene - Core
  Issue Type: Improvement
Reporter: Dawid Weiss
Assignee: Dawid Weiss
 Fix For: 6.7


The logic of handling byte buffers splits, their proper closing (cleaner) and 
all the trickery involved in slicing, cloning and proper exception handling is 
quite daunting. 

While ByteBufferIndexInput.newInstance(..) is public, the parent class 
ByteBufferIndexInput is not. I think we should make the parent class public to 
allow advanced users to make use of this (complex) piece of code to create 
IndexInput based on a sequence of ByteBuffers.

The specific rationale I'm aiming at here is RAMDirectory, which currently uses 
a custom IndexInput implementation, which in turn reaches to RAMFile's 
synchronized methods. This is the cause of quite dramatic congestions on 
multithreaded systems. While we clearly discourage RAMDirectory from being used 
in production environments, there really is no need for it to be slow. If 
modified only slightly (to use ByteBuffer-based input), the performance is on 
par with FSDirectory. Here's a sample log comparing FSDirectory with 
RAMDirectory and the "modified" RAMDirectory making use of the ByteBuffer input:

{code}
14:26:40 INFO  console: FSDirectory index.
14:26:41 INFO  console: Opened with 299943 documents.
14:26:50 INFO  console: Finished: 8.820 s, 24 matches.

14:26:50 INFO  console: RAMDirectory index.
14:26:50 INFO  console: Opened with 299943 documents.
14:28:50 INFO  console: Finished: 2.012 min, 24 matches.

14:28:50 INFO  console: RAMDirectory2 index (wrapped byte[] buffers).
14:28:50 INFO  console: Opened with 299943 documents.
14:29:00 INFO  console: Finished: 9.215 s, 24 matches.

14:29:00 INFO  console: RAMDirectory2 index (direct memory buffers).
14:29:00 INFO  console: Opened with 299943 documents.
14:29:08 INFO  console: Finished: 8.817 s, 24 matches.
{code}

Note the performance difference is an order of magnitude on this 32-CPU system 
(2 minutes vs. 9 seconds). The tiny performance difference between the 
implementation based on direct memory buffers vs. those acquired via 
ByteBuffer.wrap(byte[]) is due to the fact that direct buffers access their 
data via unsafe and the wrapped counterpart uses regular java array access (my 
best guess).



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1587 - Still unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1587/

5 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: 
http://127.0.0.1:43937/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting 
response from server at: http://127.0.0.1:43937/collection1
at 
__randomizedtesting.SeedInfo.seed([17B862331B5703D3:9FEC5DE9B5AB6E2B]:0)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591)
at 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:213)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at