Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.6-Linux/63/ Java: 64bit/jdk-9-ea+181 -XX:+UseCompressedOops -XX:+UseSerialGC --illegal-access=deny
15 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestHdfsCloudBackupRestore Error Message: Stack Trace: java.lang.ExceptionInInitializerError at __randomizedtesting.SeedInfo.seed([FE0A9A5BF89307F0]:0) at org.apache.hadoop.util.StringUtils.<clinit>(StringUtils.java:80) at org.apache.hadoop.conf.Configuration.getTrimmedStringCollection(Configuration.java:1868) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getStorageDirs(FSNamesystem.java:1374) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getNamespaceDirs(FSNamesystem.java:1355) at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:935) at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:811) at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:742) at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:612) at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:104) at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:63) at org.apache.solr.cloud.TestHdfsCloudBackupRestore.setupClass(TestHdfsCloudBackupRestore.java:100) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) Caused by: java.lang.StringIndexOutOfBoundsException: begin 0, end 3, length 1 at java.base/java.lang.String.checkBoundsBeginEnd(String.java:3116) at java.base/java.lang.String.substring(String.java:1885) at org.apache.hadoop.util.Shell.<clinit>(Shell.java:51) ... 34 more FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: Error starting up MiniSolrCloudCluster Stack Trace: java.lang.Exception: Error starting up MiniSolrCloudCluster at __randomizedtesting.SeedInfo.seed([FE0A9A5BF89307F0]:0) at org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:575) at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:255) at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:199) at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:126) at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.startup(TestSolrCloudWithDelegationTokens.java:70) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) Suppressed: java.lang.NoClassDefFoundError: Could not initialize class org.apache.hadoop.util.StringUtils at org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1437) at org.apache.hadoop.security.token.delegation.web.DelegationTokenManager.<init>(DelegationTokenManager.java:115) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.initTokenManager(DelegationTokenAuthenticationHandler.java:148) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.init(DelegationTokenAuthenticationHandler.java:118) at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler.init(HttpParamDelegationTokenPlugin.java:185) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.initializeAuthHandler(AuthenticationFilter.java:238) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.initializeAuthHandler(DelegationTokenAuthenticationFilter.java:209) at org.apache.solr.security.DelegationTokenKerberosFilter.initializeAuthHandler(DelegationTokenKerberosFilter.java:148) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.init(AuthenticationFilter.java:227) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.init(DelegationTokenAuthenticationFilter.java:175) at org.apache.solr.security.DelegationTokenKerberosFilter.init(DelegationTokenKerberosFilter.java:77) at org.apache.solr.security.HttpParamDelegationTokenPlugin.init(HttpParamDelegationTokenPlugin.java:142) at org.apache.solr.core.CoreContainer.initializeAuthenticationPlugin(CoreContainer.java:346) at org.apache.solr.core.CoreContainer.reloadSecurityProperties(CoreContainer.java:627) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:491) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) ... 1 more Suppressed: java.lang.NoClassDefFoundError: Could not initialize class org.apache.hadoop.util.StringUtils at org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1437) at org.apache.hadoop.security.token.delegation.web.DelegationTokenManager.<init>(DelegationTokenManager.java:115) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.initTokenManager(DelegationTokenAuthenticationHandler.java:148) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.init(DelegationTokenAuthenticationHandler.java:118) at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler.init(HttpParamDelegationTokenPlugin.java:185) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.initializeAuthHandler(AuthenticationFilter.java:238) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.initializeAuthHandler(DelegationTokenAuthenticationFilter.java:209) at org.apache.solr.security.DelegationTokenKerberosFilter.initializeAuthHandler(DelegationTokenKerberosFilter.java:148) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.init(AuthenticationFilter.java:227) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter.init(DelegationTokenAuthenticationFilter.java:175) at org.apache.solr.security.DelegationTokenKerberosFilter.init(DelegationTokenKerberosFilter.java:77) at org.apache.solr.security.HttpParamDelegationTokenPlugin.init(HttpParamDelegationTokenPlugin.java:142) at org.apache.solr.core.CoreContainer.initializeAuthenticationPlugin(CoreContainer.java:346) at org.apache.solr.core.CoreContainer.reloadSecurityProperties(CoreContainer.java:627) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:491) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) ... 1 more FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: Stack Trace: java.lang.NullPointerException at __randomizedtesting.SeedInfo.seed([FE0A9A5BF89307F0]:0) at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.shutdown(TestSolrCloudWithDelegationTokens.java:87) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: ObjectTracker found 5 object(s) that were not released!!! [ZkController, SolrZkClient, SolrZkClient, ZkController, Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:526) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) at org.apache.solr.cloud.ZkController.init(ZkController.java:688) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) Stack Trace: java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [ZkController, SolrZkClient, SolrZkClient, ZkController, Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:526) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) at org.apache.solr.cloud.ZkController.init(ZkController.java:688) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:485) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:388) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:251) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base/java.lang.Thread.run(Thread.java:844) at __randomizedtesting.SeedInfo.seed([FE0A9A5BF89307F0]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302) at jdk.internal.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: 49 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens: 1) Thread[id=15749, name=zkCallback-2608-thread-4, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 2) Thread[id=15755, name=Curator-Framework-0-SendThread(127.0.0.1:41239), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 3) Thread[id=15710, name=org.eclipse.jetty.server.session.HashSessionManager@4ce9cadcTimer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 4) Thread[id=15748, name=zkCallback-2608-thread-3, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 5) Thread[id=15708, name=Scheduler-2042477173, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 6) Thread[id=15718, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.base@9/java.lang.Thread.run(Thread.java:844) 7) Thread[id=15737, name=Curator-ConnectionStateManager-0, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062) at java.base@9/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:417) at app//org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:245) at app//org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43) at app//org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:111) at java.base@9/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 8) Thread[id=15734, name=OverseerStateUpdate-98507008406388740-127.0.0.1:34691_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:399) at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:132) at java.base@9/java.lang.Thread.run(Thread.java:844) 9) Thread[id=15707, name=qtp1103209341-15707, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.base@9/java.lang.Thread.run(Thread.java:844) 10) Thread[id=15744, name=Curator-Framework-0, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.DelayQueue.take(DelayQueue.java:229) at java.base@9/java.util.concurrent.DelayQueue.take(DelayQueue.java:77) at app//org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:780) at app//org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:62) at app//org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:257) at java.base@9/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 11) Thread[id=15701, name=qtp1103209341-15701-acceptor-0@4a82c9ec-ServerConnector@4e9cd610{HTTP/1.1,[http/1.1]}{127.0.0.1:34691}, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at java.base@9/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:424) at java.base@9/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:252) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 12) Thread[id=15693, name=qtp1103209341-15693, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 13) Thread[id=15700, name=qtp2032368766-15700-acceptor-0@6e96ea9e-ServerConnector@2b7659d0{HTTP/1.1,[http/1.1]}{127.0.0.1:39009}, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at java.base@9/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:424) at java.base@9/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:252) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 14) Thread[id=15719, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.base@9/java.lang.Thread.run(Thread.java:844) 15) Thread[id=15696, name=qtp2032368766-15696, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 16) Thread[id=15753, name=Curator-Framework-0-SendThread(127.0.0.1:41239), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 17) Thread[id=15733, name=zkCallback-2610-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 18) Thread[id=15725, name=jetty-launcher-2598-thread-2-SendThread(127.0.0.1:41239), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 19) Thread[id=15722, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.base@9/java.lang.Thread.run(Thread.java:844) 20) Thread[id=15694, name=qtp2032368766-15694, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 21) Thread[id=15699, name=qtp1103209341-15699, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 22) Thread[id=15704, name=qtp2032368766-15704, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.base@9/java.lang.Thread.run(Thread.java:844) 23) Thread[id=15735, name=OverseerCollectionConfigSetProcessor-98507008406388740-127.0.0.1:34691_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.base@9/java.lang.Thread.run(Thread.java:844) 24) Thread[id=15703, name=qtp1103209341-15703, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.base@9/java.lang.Thread.run(Thread.java:844) 25) Thread[id=15730, name=zkCallback-2608-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 26) Thread[id=15695, name=qtp1103209341-15695, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 27) Thread[id=15732, name=jetty-launcher-2598-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062) at java.base@9/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 28) Thread[id=15705, name=qtp1103209341-15705, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.base@9/java.lang.Thread.run(Thread.java:844) 29) Thread[id=15706, name=qtp2032368766-15706, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.base@9/java.lang.Thread.run(Thread.java:844) 30) Thread[id=15738, name=Curator-ConnectionStateManager-0, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062) at java.base@9/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:417) at app//org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:245) at app//org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43) at app//org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:111) at java.base@9/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 31) Thread[id=15731, name=jetty-launcher-2598-thread-1-SendThread(127.0.0.1:41239), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 32) Thread[id=15756, name=Curator-Framework-0-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062) at java.base@9/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 33) Thread[id=15698, name=qtp2032368766-15698, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 34) Thread[id=15702, name=qtp2032368766-15702, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.base@9/java.lang.Thread.run(Thread.java:844) 35) Thread[id=15743, name=Curator-Framework-0, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.DelayQueue.take(DelayQueue.java:229) at java.base@9/java.util.concurrent.DelayQueue.take(DelayQueue.java:77) at app//org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:780) at app//org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:62) at app//org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:257) at java.base@9/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 36) Thread[id=15754, name=Curator-Framework-0-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062) at java.base@9/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 37) Thread[id=15697, name=qtp1103209341-15697, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 38) Thread[id=15752, name=zkCallback-2610-thread-3, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 39) Thread[id=15745, name=pool-10-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 40) Thread[id=15724, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.base@9/java.lang.Thread.run(Thread.java:844) 41) Thread[id=15709, name=Scheduler-402484184, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 42) Thread[id=15692, name=qtp2032368766-15692, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@9/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@9/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@9/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at java.base@9/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at app//org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at app//org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.base@9/java.lang.Thread.run(Thread.java:844) 43) Thread[id=15747, name=zkCallback-2608-thread-2, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 44) Thread[id=15711, name=org.eclipse.jetty.server.session.HashSessionManager@7b03f1e9Timer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 45) Thread[id=15751, name=zkCallback-2610-thread-2, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 46) Thread[id=15750, name=zkCallback-2608-thread-5, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 47) Thread[id=15736, name=OverseerHdfsCoreFailoverThread-98507008406388740-127.0.0.1:34691_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.base@9/java.lang.Thread.run(Thread.java:844) 48) Thread[id=15746, name=pool-9-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2104) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1131) at java.base@9/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:848) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1092) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 49) Thread[id=15726, name=jetty-launcher-2598-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062) at java.base@9/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 49 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens: 1) Thread[id=15749, name=zkCallback-2608-thread-4, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.base@ [...truncated too long message...] lrCore org.apache.solr.core.SolrCore@66978e7a [junit4] 2> 165866 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception [junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df79d6d63001e, likely client has closed socket [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 165866 INFO (coreCloseExecutor-865-thread-2) [ ] o.a.s.c.SolrCore [testcollection_shard4_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@395d99d7 [junit4] 2> 165866 INFO (coreCloseExecutor-865-thread-1) [ ] o.a.s.c.SolrCore [falsepredicate_shard4_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@175b8d7d [junit4] 2> 165866 INFO (zkCallback-772-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3) [junit4] 2> 165866 INFO (zkCallback-800-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3) [junit4] 2> 165866 INFO (zkCallback-806-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3) [junit4] 2> 165866 INFO (coreCloseExecutor-864-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.falsepredicate.shard2.replica1 [junit4] 2> 165866 INFO (coreCloseExecutor-866-thread-1) [ ] o.a.s.c.SolrCore [currentstate_shard1_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@75124deb [junit4] 2> 165866 INFO (coreCloseExecutor-864-thread-2) [ ] o.a.s.c.SolrCore [testcollection_shard3_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@3dbc2416 [junit4] 2> 165866 INFO (coreCloseExecutor-864-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testcollection.shard3.replica1 [junit4] 2> 165866 INFO (coreCloseExecutor-864-thread-3) [ ] o.a.s.c.SolrCore [waitforstate_shard1_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@7492749b [junit4] 2> 165867 INFO (coreCloseExecutor-866-thread-2) [ ] o.a.s.c.SolrCore [falsepredicate_shard3_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@6d3011d6 [junit4] 2> 165867 INFO (coreCloseExecutor-865-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.falsepredicate.shard4.replica1 [junit4] 2> 165867 INFO (coreCloseExecutor-866-thread-3) [ ] o.a.s.c.SolrCore [testcollection_shard2_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@7ad96d00 [junit4] 2> 165867 INFO (coreCloseExecutor-866-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.currentstate.shard1.replica1 [junit4] 2> 165867 INFO (coreCloseExecutor-864-thread-4) [ ] o.a.s.c.SolrCore [stateformat1_shard1_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@6d396d41 [junit4] 2> 165867 INFO (coreCloseExecutor-864-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.waitforstate.shard1.replica1 [junit4] 2> 165867 INFO (coreCloseExecutor-865-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testcollection.shard4.replica1 [junit4] 2> 165867 INFO (coreCloseExecutor-866-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.falsepredicate.shard3.replica1 [junit4] 2> 165867 INFO (coreCloseExecutor-866-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testcollection.shard2.replica1 [junit4] 2> 165868 INFO (coreCloseExecutor-864-thread-4) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.stateformat1.shard1.replica1 [junit4] 2> 165869 INFO (jetty-closer-760-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98507022389149703-127.0.0.1:42507_solr-n_0000000001) closing [junit4] 2> 165869 INFO (jetty-closer-760-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@55fc416a{/solr,null,UNAVAILABLE} [junit4] 2> 165869 INFO (OverseerStateUpdate-98507022389149703-127.0.0.1:42507_solr-n_0000000001) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:42507_solr [junit4] 2> 165870 INFO (zkCallback-772-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1) [junit4] 2> 165870 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception [junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df79d6d630007, likely client has closed socket [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 167366 WARN (zkCallback-813-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 167366 INFO (jetty-closer-760-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3631c199{/solr,null,UNAVAILABLE} [junit4] 2> 167369 WARN (zkCallback-806-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 167369 INFO (jetty-closer-760-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@33245afa{/solr,null,UNAVAILABLE} [junit4] 2> 167370 WARN (zkCallback-772-thread-3) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 167370 INFO (jetty-closer-760-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@164241e0{/solr,null,UNAVAILABLE} [junit4] 2> 167371 ERROR (SUITE-TestCollectionStateWatchers-seed#[C41B21FB55DEB02C]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 167371 INFO (SUITE-TestCollectionStateWatchers-seed#[C41B21FB55DEB02C]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39989 39989 [junit4] 2> 172391 INFO (Thread-248) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39989 39989 [junit4] 2> 172391 WARN (Thread-248) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 8 /solr/aliases.json [junit4] 2> 7 /solr/security.json [junit4] 2> 7 /solr/configs/config [junit4] 2> 3 /solr/collections/stateformat1/state.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 12 /solr/collections/waitforstate/state.json [junit4] 2> 8 /solr/clusterprops.json [junit4] 2> 8 /solr/clusterstate.json [junit4] 2> 7 /solr/collections/falsepredicate/state.json [junit4] 2> 7 /solr/collections/testcollection/state.json [junit4] 2> 3 /solr/collections/stateformat1/state.json [junit4] 2> 2 /solr/collections/currentstate/state.json [junit4] 2> 2 /solr/overseer_elect/election/98507022389149703-127.0.0.1:42507_solr-n_0000000001 [junit4] 2> 2 /solr/collections/delayed/state.json [junit4] 2> 2 /solr/collections/tobedeleted/state.json [junit4] 2> 2 /solr/overseer_elect/election/98507022389149708-127.0.0.1:42007_solr-n_0000000002 [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 8 /solr/collections [junit4] 2> 5 /solr/live_nodes [junit4] 2> 2 /solr/overseer/queue [junit4] 2> 2 /solr/overseer/collection-queue-work [junit4] 2> [junit4] 2> 172391 INFO (SUITE-TestCollectionStateWatchers-seed#[C41B21FB55DEB02C]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore [junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.TestCollectionStateWatchers_C41B21FB55DEB02C-001 [junit4] 2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=5, maxDocsPerChunk=4, blockSize=461), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=5, blockSize=461)), sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=fr-TD, timezone=Atlantic/Stanley [junit4] 2> NOTE: Linux 4.10.0-27-generic amd64/Oracle Corporation 9 (64-bit)/cpus=8,threads=1,free=303595992,total=518979584 [junit4] 2> NOTE: All tests run in this JVM: [AndEvaluatorTest, PowerEvaluatorTest, LessThanEvaluatorTest, SolrExceptionTest, TestRetryUtil, TestJavaBinCodec, StreamExpressionParserTest, SolrExampleXMLTest, NoOpResponseParserTest, ModifiableSolrParamsTest, GreaterThanEqualToEvaluatorTest, SolrSchemalessExampleTest, LargeVolumeJettyTest, BasicHttpSolrClientTest, TestBatchUpdate, TestConfigSetAdminRequest, SolrExampleJettyTest, FacetFieldTest, ArcTangentEvaluatorTest, TestHash, CloudSolrClientMultiConstructorTest, TangentEvaluatorTest, JdbcTest, LargeVolumeBinaryJettyTest, JettyWebappTest, LargeVolumeEmbeddedTest, TestSolrProperties, SolrParamTest, TermsResponseTest, TestEmbeddedSolrServer, TestCoreAdmin, GetByIdTest, CloudSolrClientCacheTest, CloudSolrClientTest, GraphExpressionTest, StreamingTest, TestDelegationTokenResponse, TestSuggesterResponse, TestCollectionStateWatchers] [junit4] Completed [130/130 (1!)] on J2 in 63.57s, 9 tests, 1 failure <<< FAILURES! [...truncated 35552 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org