Build: https://ci-builds.apache.org/job/Solr/job/Solr-TestIntegration-10.x/210/
No tests ran. Build Log: [...truncated 424 lines...] # ERROR: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update # # org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update # at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:506) # at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302) # at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:466) # at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:442) # at org.apache.solr.cli.PostTool.commit(PostTool.java:778) # at org.apache.solr.cli.PostTool.execute(PostTool.java:346) # at org.apache.solr.cli.PostTool.runImpl(PostTool.java:320) # at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81) # at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117) # Caused by: java.net.ConnectException: Connection refused # at java.base/sun.nio.ch.Net.pollConnect(Native Method) # at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:682) # at java.base/sun.nio.ch.SocketChannelImpl.finishTimedConnect(SocketChannelImpl.java:1191) # at java.base/sun.nio.ch.SocketChannelImpl.blockingConnect(SocketChannelImpl.java:1233) # at java.base/sun.nio.ch.SocketAdaptor.connect(SocketAdaptor.java:102) # at org.eclipse.jetty.io.ClientConnector.connect(ClientConnector.java:467) # at org.eclipse.jetty.io.Transport$Socket.connect(Transport.java:175) # at org.eclipse.jetty.http2.client.HTTP2Client.connect(HTTP2Client.java:486) # at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:140) # at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:128) # at org.eclipse.jetty.client.HttpClient$1.connect(HttpClient.java:579) # at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:556) # at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:552) # at org.eclipse.jetty.util.SocketAddressResolver$Async.lambda$resolve$0(SocketAddressResolver.java:186) # at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408) # at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) # at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) # at java.base/java.lang.Thread.run(Thread.java:1583) # -- # # Please find the SOLR_HOME snapshot for failed test #12 at: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/failure-snapshots/81-12 # Last output: # Posting files to [base] url http://localhost:35851/solr/monitors_verbose/update using content-type application/xml... # POSTing file monitor.xml to [base] # PostTool: WARNING: IOException while reading response: java.net.SocketException: Unexpected end of file from server # 1 files indexed. # COMMITting Solr index changes to http://localhost:35851/solr/monitors_verbose/update... # # ERROR: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update # # org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://localhost:35851/solr/monitors_verbose/update # at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:506) # at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302) # at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:466) # at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:442) # at org.apache.solr.cli.PostTool.commit(PostTool.java:778) # at org.apache.solr.cli.PostTool.execute(PostTool.java:346) # at org.apache.solr.cli.PostTool.runImpl(PostTool.java:320) # at org.apache.solr.cli.ToolBase.runTool(ToolBase.java:81) # at org.apache.solr.cli.SolrCLI.main(SolrCLI.java:117) # Caused by: java.net.ConnectException: Connection refused # at java.base/sun.nio.ch.Net.pollConnect(Native Method) # at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:682) # at java.base/sun.nio.ch.SocketChannelImpl.finishTimedConnect(SocketChannelImpl.java:1191) # at java.base/sun.nio.ch.SocketChannelImpl.blockingConnect(SocketChannelImpl.java:1233) # at java.base/sun.nio.ch.SocketAdaptor.connect(SocketAdaptor.java:102) # at org.eclipse.jetty.io.ClientConnector.connect(ClientConnector.java:467) # at org.eclipse.jetty.io.Transport$Socket.connect(Transport.java:175) # at org.eclipse.jetty.http2.client.HTTP2Client.connect(HTTP2Client.java:486) # at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:140) # at org.eclipse.jetty.http2.client.transport.HttpClientTransportOverHTTP2.connect(HttpClientTransportOverHTTP2.java:128) # at org.eclipse.jetty.client.HttpClient$1.connect(HttpClient.java:579) # at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:556) # at org.eclipse.jetty.client.HttpClient$1.succeeded(HttpClient.java:552) # at org.eclipse.jetty.util.SocketAddressResolver$Async.lambda$resolve$0(SocketAddressResolver.java:186) # at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408) # at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) # at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) # at java.base/java.lang.Thread.run(Thread.java:1583) ok 82 post solr log into solr via cli # in 9182 ms # Test failed - capturing Docker diagnostics # === docker ps (summary) === NAMES STATUS IMAGE PORTS solr-node3 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp solr-node2 Up 17 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp solr-node1 Up 29 seconds apache/solr-nightly:9.10.0-SNAPSHOT-slim 8983/tcp # === Docker logs for solr-node1 === 2026-05-15T16:22:30.332019868Z *** [WARN] *** Your open file limit is currently 1024. 2026-05-15T16:22:30.332055727Z It should be set to 65000 to avoid operational disruption. 2026-05-15T16:22:30.332059171Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh 2026-05-15T16:22:30.332289820Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag. 2026-05-15T16:22:30.332301390Z 2026-05-15T16:22:30.344612183Z Java 17 detected. Enabled workaround for SOLR-16463 2026-05-15T16:22:30.355404909Z [0.001s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system. 2026-05-15T16:22:30.451832404Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true 2026-05-15T16:22:30.587796270Z WARNING: A command line option has enabled the Security Manager 2026-05-15T16:22:30.587826508Z WARNING: The Security Manager is deprecated and will be removed in a future release 2026-05-15T16:22:32.825865424Z 2026-05-15 16:22:32.820 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8 2026-05-15T16:22:33.432818995Z 2026-05-15 16:22:33.432 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner 2026-05-15T16:22:33.498393783Z 2026-05-15 16:22:33.497 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory 2026-05-15T16:22:33.515433082Z 2026-05-15 16:22:33.514 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted] 2026-05-15T16:22:33.515505812Z 2026-05-15 16:22:33.515 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8983 2026-05-15T16:22:33.515619397Z 2026-05-15 16:22:33.515 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim 2026-05-15T16:22:33.515734575Z 2026-05-15 16:22:33.515 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-15T16:22:33.515259460Z 2026-05-15T16:22:33.519506623Z 2026-05-15 16:22:33.519 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log 2026-05-15T16:22:33.588966659Z 2026-05-15 16:22:33.588 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home) 2026-05-15T16:22:33.644303327Z 2026-05-15 16:22:33.643 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default 2026-05-15T16:22:33.644480725Z 2026-05-15 16:22:33.643 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml 2026-05-15T16:22:33.738250761Z 2026-05-15 16:22:33.737 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib] 2026-05-15T16:22:35.370673308Z 2026-05-15 16:22:35.369 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled. 2026-05-15T16:22:36.326510569Z 2026-05-15 16:22:36.325 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure. 2026-05-15T16:22:36.342769143Z 2026-05-15 16:22:36.342 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Zookeeper configuration not found in /var/solr/data, using built-in default 2026-05-15T16:22:36.342788116Z 2026-05-15 16:22:36.342 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServerProps Reading configuration from: /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/zoo.cfg 2026-05-15T16:22:36.349055630Z 2026-05-15 16:22:36.348 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port 9983, listening on host 0.0.0.0 2026-05-15T16:22:36.349070472Z 2026-05-15 16:22:36.348 WARN (main) [c: s: r: x: t:] o.a.s.c.SolrZkServer Embedded Zookeeper is not recommended in production environments. See Reference Guide for details. 2026-05-15T16:22:36.455618719Z 2026-05-15 16:22:36.455 WARN (embeddedZkServer) [c: s: r: x: t:] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0. 2026-05-15T16:22:36.852077130Z 2026-05-15 16:22:36.851 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=172.18.0.2:9983 2026-05-15T16:22:36.884846121Z 2026-05-15 16:22:36.884 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates. 2026-05-15T16:22:36.962866269Z 2026-05-15 16:22:36.962 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper 2026-05-15T16:22:36.993262401Z 2026-05-15 16:22:36.992 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected 2026-05-15T16:22:36.993356233Z 2026-05-15 16:22:36.992 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper 2026-05-15T16:22:37.130452877Z 2026-05-15 16:22:37.129 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html 2026-05-15T16:22:37.170519256Z 2026-05-15 16:22:37.169 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates. 2026-05-15T16:22:37.183729619Z 2026-05-15 16:22:37.183 INFO (main) [c: s: r: x: t:] o.a.s.c.OverseerElectionContext I am going to be the leader solr-node1:8983_solr 2026-05-15T16:22:37.193527714Z 2026-05-15 16:22:37.193 INFO (main) [c: s: r: x: t:] o.a.s.c.Overseer Overseer (id=72059903898681344-solr-node1:8983_solr-n_0000000000) starting 2026-05-15T16:22:37.321246167Z 2026-05-15 16:22:37.320 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node1:8983_solr 2026-05-15T16:22:37.321288763Z 2026-05-15 16:22:37.320 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.Overseer Starting to work on the main queue : solr-node1:8983_solr 2026-05-15T16:22:37.385536647Z 2026-05-15 16:22:37.385 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) 2026-05-15T16:22:37.521333263Z 2026-05-15 16:22:37.520 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info 2026-05-15T16:22:38.209333359Z 2026-05-15 16:22:38.208 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data 2026-05-15T16:22:38.543424974Z 2026-05-15 16:22:38.542 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available. 2026-05-15T16:22:38.946832537Z 2026-05-15 16:22:38.946 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer 2026-05-15T16:22:39.370410083Z 2026-05-15 16:22:39.369 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp} 2026-05-15T16:22:39.506170003Z 2026-05-15 16:22:39.504 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_15.request.log 2026-05-15T16:22:39.510388831Z 2026-05-15 16:22:39.509 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8983} 2026-05-15T16:22:39.513045396Z 2026-05-15 16:22:39.512 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @9158ms 2026-05-15T16:22:40.275660827Z 2026-05-15 16:22:40.274 INFO (qtp985324122-29-solr-node1-1) [c: s: r: x: t:solr-node1-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=105 2026-05-15T16:22:41.010523561Z 2026-05-15 16:22:41.010 INFO (qtp985324122-96-solr-node1-2) [c: s: r: x: t:solr-node1-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=19 2026-05-15T16:22:45.600929194Z 2026-05-15 16:22:45.600 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3) 2026-05-15T16:22:52.893223793Z 2026-05-15 16:22:52.892 INFO (qtp985324122-99-solr-node1-3) [c: s: r: x: t:solr-node1-3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=44 2026-05-15T16:22:53.037932384Z 2026-05-15 16:22:53.037 INFO (qtp985324122-99-solr-node1-4) [c: s: r: x: t:solr-node1-4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=54 2026-05-15T16:22:53.076630017Z 2026-05-15 16:22:53.076 INFO (qtp985324122-29-solr-node1-5) [c: s: r: x: t:solr-node1-5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=5 2026-05-15T16:22:53.705372375Z 2026-05-15 16:22:53.704 INFO (qtp985324122-99-solr-node1-6) [c: s: r: x: t:solr-node1-6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=20 2026-05-15T16:22:53.757319061Z 2026-05-15 16:22:53.756 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.a.c.CreateCollectionCmd Create collection test-collection 2026-05-15T16:22:53.807528657Z 2026-05-15 16:22:53.807 INFO (OverseerThreadFactory-17-thread-1) [c:test-collection s: r: x: t:] o.a.s.c.c.ZkStateReader /collections/test-collection/state.json is deleted, stop watching children 2026-05-15T16:22:53.832036203Z 2026-05-15 16:22:53.831 INFO (zkCallback-11-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/test-collection/state.json zxid: 173] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:53.957511863Z 2026-05-15 16:22:53.957 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() { 2026-05-15T16:22:53.957534019Z "core":"test-collection_shard3_replica_n1", 2026-05-15T16:22:53.957536735Z "node_name":"solr-node1:8983_solr", 2026-05-15T16:22:53.957538894Z "base_url":"http://solr-node1:8983/solr";, 2026-05-15T16:22:53.957540960Z "collection":"test-collection", 2026-05-15T16:22:53.957543834Z "shard":"shard3", 2026-05-15T16:22:53.957545820Z "state":"down", 2026-05-15T16:22:53.957547763Z "type":"NRT", 2026-05-15T16:22:53.957549751Z "operation":"ADDREPLICA", 2026-05-15T16:22:53.957552031Z "waitForFinalState":"false"} 2026-05-15T16:22:53.985968399Z 2026-05-15 16:22:53.984 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() { 2026-05-15T16:22:53.985985569Z "core":"test-collection_shard2_replica_n2", 2026-05-15T16:22:53.985988213Z "node_name":"solr-node3:8985_solr", 2026-05-15T16:22:53.985990416Z "base_url":"http://solr-node3:8985/solr";, 2026-05-15T16:22:53.985992723Z "collection":"test-collection", 2026-05-15T16:22:53.985994733Z "shard":"shard2", 2026-05-15T16:22:53.985996631Z "state":"down", 2026-05-15T16:22:53.985998780Z "type":"NRT", 2026-05-15T16:22:53.986000764Z "operation":"ADDREPLICA", 2026-05-15T16:22:53.986002719Z "waitForFinalState":"false"} 2026-05-15T16:22:53.994703900Z 2026-05-15 16:22:53.993 INFO (OverseerStateUpdate-72059903898681344-solr-node1:8983_solr-n_0000000000) [c: s: r: x: t:] o.a.s.c.o.SliceMutator createReplica() { 2026-05-15T16:22:53.994722569Z "core":"test-collection_shard1_replica_n4", 2026-05-15T16:22:53.994735197Z "node_name":"solr-node2:8984_solr", 2026-05-15T16:22:53.994737463Z "base_url":"http://solr-node2:8984/solr";, 2026-05-15T16:22:53.994739486Z "collection":"test-collection", 2026-05-15T16:22:53.994741498Z "shard":"shard1", 2026-05-15T16:22:53.994743947Z "state":"down", 2026-05-15T16:22:53.994745845Z "type":"NRT", 2026-05-15T16:22:53.994747740Z "operation":"ADDREPLICA", 2026-05-15T16:22:53.994749756Z "waitForFinalState":"false"} 2026-05-15T16:22:54.107911835Z 2026-05-15 16:22:54.107 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 185] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:54.255534627Z 2026-05-15 16:22:54.255 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node3&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&shard=shard3&wt=javabin 2026-05-15T16:22:54.366823096Z 2026-05-15 16:22:54.366 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0 2026-05-15T16:22:54.504918310Z 2026-05-15 16:22:54.504 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config 2026-05-15T16:22:55.658754750Z 2026-05-15 16:22:55.658 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id 2026-05-15T16:22:55.658777902Z 2026-05-15 16:22:55.658 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard3_replica_n1' using configuration from configset techproducts, trusted=true 2026-05-15T16:22:55.770700732Z 2026-05-15 16:22:55.769 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard3_replica_n1], dataDir=[/var/solr/data/test-collection_shard3_replica_n1/data/] 2026-05-15T16:22:58.143710376Z 2026-05-15 16:22:58.143 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer 2026-05-15T16:22:58.196652992Z 2026-05-15 16:22:58.195 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog 2026-05-15T16:22:58.203258487Z 2026-05-15 16:22:58.202 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 2026-05-15T16:22:58.204529589Z 2026-05-15 16:22:58.204 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 2026-05-15T16:22:58.204608672Z 2026-05-15 16:22:58.204 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms; 2026-05-15T16:22:58.307232260Z 2026-05-15 16:22:58.306 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts 2026-05-15T16:22:58.311217027Z 2026-05-15 16:22:58.310 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts 2026-05-15T16:22:58.313999624Z 2026-05-15 16:22:58.313 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml 2026-05-15T16:22:58.322001602Z 2026-05-15 16:22:58.321 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest 2026-05-15T16:22:58.335702328Z 2026-05-15 16:22:58.335 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal} 2026-05-15T16:22:58.342792888Z 2026-05-15 16:22:58.342 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms 2026-05-15T16:22:58.351152768Z 2026-05-15 16:22:58.350 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done. 2026-05-15T16:22:58.351167419Z 2026-05-15 16:22:58.350 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default 2026-05-15T16:22:58.359303493Z 2026-05-15 16:22:58.359 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard3_replica_n1 solr-node1-8 core_node3 solr-node1:8983_solr test-collection shard3) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms 2026-05-15T16:22:58.371447450Z 2026-05-15 16:22:58.370 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard3 to Terms{values={core_node3=0}, version=0} for registerTerm 2026-05-15T16:22:58.375740912Z 2026-05-15 16:22:58.375 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard3 2026-05-15T16:22:58.398782212Z 2026-05-15 16:22:58.398 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. 2026-05-15T16:22:58.398799780Z 2026-05-15 16:22:58.398 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync 2026-05-15T16:22:58.398803388Z 2026-05-15 16:22:58.398 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ 2026-05-15T16:22:58.400671114Z 2026-05-15 16:22:58.400 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me 2026-05-15T16:22:58.401625863Z 2026-05-15 16:22:58.401 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ has no replicas 2026-05-15T16:22:58.404199682Z 2026-05-15 16:22:58.403 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard3/leader after winning as /collections/test-collection/leader_elect/shard3/election/72059903898681344-core_node3-n_0000000000 2026-05-15T16:22:58.410814074Z 2026-05-15 16:22:58.410 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node1:8983/solr/test-collection_shard3_replica_n1/ shard3 2026-05-15T16:22:58.418915697Z 2026-05-15 16:22:58.418 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard3 r:core_node3 x:test-collection_shard3_replica_n1 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary 2026-05-15T16:22:58.431216967Z 2026-05-15 16:22:58.430 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=techproducts&newCollection=true&name=test-collection_shard3_replica_n1&action=CREATE&numShards=3&collection=test-collection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4177 2026-05-15T16:22:58.447113266Z 2026-05-15 16:22:58.446 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas 2026-05-15T16:22:58.526561925Z 2026-05-15 16:22:58.526 INFO (zkCallback-11-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:58.529088541Z 2026-05-15 16:22:58.528 INFO (qtp985324122-101-solr-node1-7) [c:test-collection s: r: x: t:solr-node1-7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=techproducts&name=test-collection&nrtReplicas=1&action=CREATE&numShards=3&wt=json&version=2.2} status=0 QTime=4803 # === Docker inspect for solr-node1 === { "Name": "/solr-node1", "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 1457614, "ExitCode": 0, "Error": "", "StartedAt": "2026-05-15T16:22:29.243323028Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Ports": { "8983/tcp": null } } # === Docker logs for solr-node2 === 2026-05-15T16:22:41.905028544Z *** [WARN] *** Your open file limit is currently 1024. 2026-05-15T16:22:41.905055235Z It should be set to 65000 to avoid operational disruption. 2026-05-15T16:22:41.905058311Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh 2026-05-15T16:22:41.905405233Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag. 2026-05-15T16:22:41.905414380Z 2026-05-15T16:22:41.916773279Z Java 17 detected. Enabled workaround for SOLR-16463 2026-05-15T16:22:41.927055447Z [0.002s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system. 2026-05-15T16:22:41.987521199Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true 2026-05-15T16:22:42.028451217Z WARNING: A command line option has enabled the Security Manager 2026-05-15T16:22:42.028469962Z WARNING: The Security Manager is deprecated and will be removed in a future release 2026-05-15T16:22:43.431569216Z 2026-05-15 16:22:43.428 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8 2026-05-15T16:22:43.710302531Z 2026-05-15 16:22:43.709 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner 2026-05-15T16:22:43.741491860Z 2026-05-15 16:22:43.741 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory 2026-05-15T16:22:43.750735583Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted] 2026-05-15T16:22:43.751084812Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8984 2026-05-15T16:22:43.751095116Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim 2026-05-15T16:22:43.751097761Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-15T16:22:43.750534382Z 2026-05-15T16:22:43.753704303Z 2026-05-15 16:22:43.753 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log 2026-05-15T16:22:43.796934258Z 2026-05-15 16:22:43.796 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home) 2026-05-15T16:22:43.848423068Z 2026-05-15 16:22:43.847 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes 2026-05-15T16:22:44.010063620Z 2026-05-15 16:22:44.009 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper 2026-05-15T16:22:44.032428278Z 2026-05-15 16:22:44.032 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected 2026-05-15T16:22:44.032546986Z 2026-05-15 16:22:44.032 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper 2026-05-15T16:22:44.032677206Z 2026-05-15 16:22:44.032 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes 2026-05-15T16:22:44.147731736Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper) 2026-05-15T16:22:44.147994179Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default 2026-05-15T16:22:44.148101993Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml 2026-05-15T16:22:44.201809785Z 2026-05-15 16:22:44.201 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib] 2026-05-15T16:22:45.311567614Z 2026-05-15 16:22:45.310 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled. 2026-05-15T16:22:45.467617852Z 2026-05-15 16:22:45.467 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure. 2026-05-15T16:22:45.469052756Z 2026-05-15 16:22:45.467 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983 2026-05-15T16:22:45.477902421Z 2026-05-15 16:22:45.477 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates. 2026-05-15T16:22:45.483193337Z 2026-05-15 16:22:45.482 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper 2026-05-15T16:22:45.486055340Z 2026-05-15 16:22:45.485 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected 2026-05-15T16:22:45.486067556Z 2026-05-15 16:22:45.485 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper 2026-05-15T16:22:45.527769716Z 2026-05-15 16:22:45.527 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html 2026-05-15T16:22:45.537995338Z 2026-05-15 16:22:45.537 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) 2026-05-15T16:22:45.560924181Z 2026-05-15 16:22:45.560 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates. 2026-05-15T16:22:45.577611649Z 2026-05-15 16:22:45.577 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node2:8984_solr as DOWN 2026-05-15T16:22:45.589533356Z 2026-05-15 16:22:45.588 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node2:8984_solr 2026-05-15T16:22:45.602136236Z 2026-05-15 16:22:45.601 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3) 2026-05-15T16:22:45.664474950Z 2026-05-15 16:22:45.663 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info 2026-05-15T16:22:46.068796963Z 2026-05-15 16:22:46.068 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data 2026-05-15T16:22:46.339950000Z 2026-05-15 16:22:46.339 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available. 2026-05-15T16:22:46.635498919Z 2026-05-15 16:22:46.635 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer 2026-05-15T16:22:46.792433280Z 2026-05-15 16:22:46.791 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp} 2026-05-15T16:22:46.798014409Z 2026-05-15 16:22:46.797 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_15.request.log 2026-05-15T16:22:46.801769735Z 2026-05-15 16:22:46.801 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8984} 2026-05-15T16:22:46.803150758Z 2026-05-15 16:22:46.802 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4877ms 2026-05-15T16:22:47.067886583Z 2026-05-15 16:22:47.066 INFO (qtp985324122-27-solr-node2-1) [c: s: r: x: t:solr-node2-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=44 2026-05-15T16:22:47.143134589Z 2026-05-15 16:22:47.142 INFO (qtp985324122-55-solr-node2-2) [c: s: r: x: t:solr-node2-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=13 2026-05-15T16:22:54.254149485Z 2026-05-15 16:22:54.253 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node6&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&shard=shard1&wt=javabin 2026-05-15T16:22:54.325512012Z 2026-05-15 16:22:54.325 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0 2026-05-15T16:22:54.427761192Z 2026-05-15 16:22:54.427 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config 2026-05-15T16:22:55.026501743Z 2026-05-15 16:22:55.025 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id 2026-05-15T16:22:55.026519387Z 2026-05-15 16:22:55.026 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard1_replica_n4' using configuration from configset techproducts, trusted=true 2026-05-15T16:22:55.075155350Z 2026-05-15 16:22:55.074 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard1_replica_n4], dataDir=[/var/solr/data/test-collection_shard1_replica_n4/data/] 2026-05-15T16:22:55.720372206Z 2026-05-15 16:22:55.719 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer 2026-05-15T16:22:55.749705878Z 2026-05-15 16:22:55.749 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog 2026-05-15T16:22:55.759399830Z 2026-05-15 16:22:55.758 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 2026-05-15T16:22:55.760531433Z 2026-05-15 16:22:55.759 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 2026-05-15T16:22:55.760597758Z 2026-05-15 16:22:55.760 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms; 2026-05-15T16:22:55.815611644Z 2026-05-15 16:22:55.815 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts 2026-05-15T16:22:55.819783634Z 2026-05-15 16:22:55.819 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts 2026-05-15T16:22:55.822389201Z 2026-05-15 16:22:55.822 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml 2026-05-15T16:22:55.824797847Z 2026-05-15 16:22:55.824 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest 2026-05-15T16:22:55.841841491Z 2026-05-15 16:22:55.841 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxQueryFrequency=0.01, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4} 2026-05-15T16:22:55.850462069Z 2026-05-15 16:22:55.850 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms 2026-05-15T16:22:55.856821993Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done. 2026-05-15T16:22:55.856852501Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default 2026-05-15T16:22:55.863961753Z 2026-05-15 16:22:55.863 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard1_replica_n4 solr-node1-8 core_node6 solr-node2:8984_solr test-collection shard1) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms 2026-05-15T16:22:55.888285784Z 2026-05-15 16:22:55.887 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard1 to Terms{values={core_node6=0}, version=0} for registerTerm 2026-05-15T16:22:55.889578988Z 2026-05-15 16:22:55.889 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard1 2026-05-15T16:22:55.914642969Z 2026-05-15 16:22:55.914 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. 2026-05-15T16:22:55.914799994Z 2026-05-15 16:22:55.914 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync 2026-05-15T16:22:55.914812069Z 2026-05-15 16:22:55.914 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ 2026-05-15T16:22:55.916158538Z 2026-05-15 16:22:55.915 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me 2026-05-15T16:22:55.916518899Z 2026-05-15 16:22:55.916 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ has no replicas 2026-05-15T16:22:55.917750925Z 2026-05-15 16:22:55.917 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard1/leader after winning as /collections/test-collection/leader_elect/shard1/election/72059903898681348-core_node6-n_0000000000 2026-05-15T16:22:55.937144288Z 2026-05-15 16:22:55.936 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node2:8984/solr/test-collection_shard1_replica_n4/ shard1 2026-05-15T16:22:56.044490181Z 2026-05-15 16:22:56.044 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 218] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:56.048152616Z 2026-05-15 16:22:56.047 INFO (qtp985324122-28-solr-node1-8) [c:test-collection s:shard1 r:core_node6 x:test-collection_shard1_replica_n4 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary 2026-05-15T16:22:56.058576648Z 2026-05-15 16:22:56.058 INFO (qtp985324122-28-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=techproducts&newCollection=true&name=test-collection_shard1_replica_n4&action=CREATE&numShards=3&collection=test-collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1806 2026-05-15T16:22:56.161561369Z 2026-05-15 16:22:56.161 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 222] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:58.416031078Z 2026-05-15 16:22:58.415 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:58.526717514Z 2026-05-15 16:22:58.526 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3]) # === Docker inspect for solr-node2 === { "Name": "/solr-node2", "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 1458025, "ExitCode": 0, "Error": "", "StartedAt": "2026-05-15T16:22:41.194129254Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Ports": { "8983/tcp": null } } # === Docker logs for solr-node3 === 2026-05-15T16:22:41.904992740Z *** [WARN] *** Your open file limit is currently 1024. 2026-05-15T16:22:41.905029126Z It should be set to 65000 to avoid operational disruption. 2026-05-15T16:22:41.905032451Z If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh 2026-05-15T16:22:41.905280439Z Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag. 2026-05-15T16:22:41.905289945Z 2026-05-15T16:22:41.916173928Z Java 17 detected. Enabled workaround for SOLR-16463 2026-05-15T16:22:41.928941200Z [0.004s][warning][pagesize] UseLargePages disabled, no large pages configured and available on the system. 2026-05-15T16:22:41.987585643Z CompileCommand: exclude com/github/benmanes/caffeine/cache/BoundedLocalCache.put bool exclude = true 2026-05-15T16:22:42.028382506Z WARNING: A command line option has enabled the Security Manager 2026-05-15T16:22:42.028410428Z WARNING: The Security Manager is deprecated and will be removed in a future release 2026-05-15T16:22:43.431146643Z 2026-05-15 16:22:43.428 INFO (main) [c: s: r: x: t:] o.e.j.s.Server jetty-10.0.26; built: 2025-08-14T15:56:56.454Z; git: 65a55c55a68c56d5f2e8232164439bd6e76e53d1; jvm 17.0.16+8 2026-05-15T16:22:43.716281283Z 2026-05-15 16:22:43.715 WARN (main) [c: s: r: x: t:] o.e.j.u.DeprecationWarning Using @Deprecated Class org.eclipse.jetty.servlet.listener.ELContextCleaner 2026-05-15T16:22:43.741702523Z 2026-05-15 16:22:43.741 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory 2026-05-15T16:22:43.751200653Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider ___ _ Welcome to Apache Solr™ version 9.10.0-SNAPSHOT 1edeb3ac6a97186b4ae7de511196a1769de7960e [snapshot build, details omitted] 2026-05-15T16:22:43.751214402Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider / __| ___| |_ _ Starting in cloud mode on port 8985 2026-05-15T16:22:43.751217022Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_| Install dir: /opt/solr-9.10.0-SNAPSHOT-slim 2026-05-15T16:22:43.751220037Z 2026-05-15 16:22:43.750 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider |___/\___/_|_| Start time: 2026-05-15T16:22:43.750600842Z 2026-05-15T16:22:43.753837301Z 2026-05-15 16:22:43.753 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr started with "-XX:+CrashOnOutOfMemoryError" that will crash on any OutOfMemoryError exception. The cause of the OOME will be logged in the crash file at the following path: /var/solr/logs/jvm_crash_1.log 2026-05-15T16:22:43.796967800Z 2026-05-15 16:22:43.796 INFO (main) [c: s: r: x: t:] o.a.s.s.CoreContainerProvider Solr Home: /var/solr/data (source: system property: solr.solr.home) 2026-05-15T16:22:43.849355227Z 2026-05-15 16:22:43.848 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes 2026-05-15T16:22:44.010230583Z 2026-05-15 16:22:44.009 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 30000ms for client to connect to ZooKeeper 2026-05-15T16:22:44.031521582Z 2026-05-15 16:22:44.030 INFO (zkCallback-1-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected 2026-05-15T16:22:44.031597535Z 2026-05-15 16:22:44.031 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper 2026-05-15T16:22:44.031774325Z 2026-05-15 16:22:44.031 WARN (main) [c: s: r: x: t:] o.a.s.c.c.SolrZkClient Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes 2026-05-15T16:22:44.147466027Z 2026-05-15 16:22:44.146 INFO (main) [c: s: r: x: t:] o.a.s.c.NodeConfig Loading solr.xml from SolrHome (not found in ZooKeeper) 2026-05-15T16:22:44.148006931Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig solr.xml not found in SOLR_HOME, using built-in default 2026-05-15T16:22:44.148156319Z 2026-05-15 16:22:44.147 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrXmlConfig Loading solr.xml from /opt/solr-9.10.0-SNAPSHOT-slim/server/solr/solr.xml 2026-05-15T16:22:44.201781177Z 2026-05-15 16:22:44.201 INFO (main) [c: s: r: x: t:] o.a.s.c.SolrResourceLoader Added 1 libs to classloader, from paths: [/opt/solr-9.10.0-SNAPSHOT-slim/lib] 2026-05-15T16:22:45.231740946Z 2026-05-15 16:22:45.231 INFO (main) [c: s: r: x: t:] o.a.s.u.t.SimplePropagator Always-on trace id generation enabled. 2026-05-15T16:22:45.458334600Z 2026-05-15 16:22:45.457 WARN (main) [c: s: r: x: t:] o.a.s.u.StartupLoggingUtils Jetty request logging enabled. Will retain logs for last 3 days. See chapter "Configuring Logging" in reference guide for how to configure. 2026-05-15T16:22:45.460592331Z 2026-05-15 16:22:45.457 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkContainer Zookeeper client=solr-node1:9983 2026-05-15T16:22:45.473610977Z 2026-05-15 16:22:45.473 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates. 2026-05-15T16:22:45.480792438Z 2026-05-15 16:22:45.480 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Waiting up to 15000ms for client to connect to ZooKeeper 2026-05-15T16:22:45.485303569Z 2026-05-15 16:22:45.484 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager zkClient has connected 2026-05-15T16:22:45.485367701Z 2026-05-15 16:22:45.485 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper 2026-05-15T16:22:45.527724460Z 2026-05-15 16:22:45.527 WARN (main) [c: s: r: x: t:] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/solr/latest/deployment-guide/zookeeper-access-control.html 2026-05-15T16:22:45.538101513Z 2026-05-15 16:22:45.537 INFO (main) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) 2026-05-15T16:22:45.562225201Z 2026-05-15 16:22:45.561 INFO (main) [c: s: r: x: t:] o.a.s.c.DistributedClusterStateUpdater Creating DistributedClusterStateUpdater with useDistributedStateUpdate=false. Solr will be using Overseer based cluster state updates. 2026-05-15T16:22:45.577653693Z 2026-05-15 16:22:45.577 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Publish node=solr-node3:8985_solr as DOWN 2026-05-15T16:22:45.588305608Z 2026-05-15 16:22:45.587 INFO (main) [c: s: r: x: t:] o.a.s.c.ZkController Register node as live in ZooKeeper: /live_nodes/solr-node3:8985_solr 2026-05-15T16:22:45.601901849Z 2026-05-15 16:22:45.601 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3) 2026-05-15T16:22:45.664521474Z 2026-05-15 16:22:45.663 WARN (main) [c: s: r: x: t:] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info 2026-05-15T16:22:46.068796015Z 2026-05-15 16:22:46.068 INFO (main) [c: s: r: x: t:] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /var/solr/data 2026-05-15T16:22:46.340104135Z 2026-05-15 16:22:46.339 WARN (main) [c: s: r: x: t:] o.g.j.m.i.MessagingBinders A class jakarta.activation.DataSource for a default provider MessageBodyWriter<jakarta.activation.DataSource> was not found. The provider is not available. 2026-05-15T16:22:46.635087411Z 2026-05-15 16:22:46.634 INFO (main) [c: s: r: x: t:] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer 2026-05-15T16:22:46.791615765Z 2026-05-15 16:22:46.790 INFO (main) [c: s: r: x: t:] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@71f67a79{solr-jetty-context.xml,/solr,file:///opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp/,AVAILABLE}{/opt/solr-9.10.0-SNAPSHOT-slim/server/solr-webapp/webapp} 2026-05-15T16:22:46.797985721Z 2026-05-15 16:22:46.797 INFO (main) [c: s: r: x: t:] o.e.j.s.RequestLogWriter Opened /var/solr/logs/2026_05_15.request.log 2026-05-15T16:22:46.801766587Z 2026-05-15 16:22:46.801 INFO (main) [c: s: r: x: t:] o.e.j.s.AbstractConnector Started ServerConnector@61884cb1{HTTP/1.1, (http/1.1, h2c)}{0.0.0.0:8985} 2026-05-15T16:22:46.803125023Z 2026-05-15 16:22:46.802 INFO (main) [c: s: r: x: t:] o.e.j.s.Server Started Server@350ec41e{STARTING}[10.0.26,sto=0] @4878ms 2026-05-15T16:22:49.612040234Z 2026-05-15 16:22:49.603 INFO (qtp985324122-27-solr-node3-1) [c: s: r: x: t:solr-node3-1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/info/system params={wt=javabin&version=2} status=0 QTime=78 2026-05-15T16:22:49.680327943Z 2026-05-15 16:22:49.679 INFO (qtp985324122-56-solr-node3-2) [c: s: r: x: t:solr-node3-2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=20 2026-05-15T16:22:54.269223436Z 2026-05-15 16:22:54.267 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=techproducts&newCollection=true&collection=test-collection&version=2&replicaType=NRT&coreNodeName=core_node5&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&shard=shard2&wt=javabin 2026-05-15T16:22:54.346633296Z 2026-05-15 16:22:54.346 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.12.0 2026-05-15T16:22:54.426420431Z 2026-05-15 16:22:54.425 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Schema name=default-config 2026-05-15T16:22:55.035639611Z 2026-05-15 16:22:55.035 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.IndexSchema Loaded schema default-config/1.7 with uniqueid field id 2026-05-15T16:22:55.035663652Z 2026-05-15 16:22:55.035 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.CoreContainer Creating SolrCore 'test-collection_shard2_replica_n2' using configuration from configset techproducts, trusted=true 2026-05-15T16:22:55.074968152Z 2026-05-15 16:22:55.074 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Opening new SolrCore at [/var/solr/data/test-collection_shard2_replica_n2], dataDir=[/var/solr/data/test-collection_shard2_replica_n2/data/] 2026-05-15T16:22:55.722256218Z 2026-05-15 16:22:55.721 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer 2026-05-15T16:22:55.749593210Z 2026-05-15 16:22:55.749 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog 2026-05-15T16:22:55.759394531Z 2026-05-15 16:22:55.758 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 2026-05-15T16:22:55.760493244Z 2026-05-15 16:22:55.759 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 2026-05-15T16:22:55.760550216Z 2026-05-15 16:22:55.760 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms; 2026-05-15T16:22:55.814991171Z 2026-05-15 16:22:55.814 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/techproducts 2026-05-15T16:22:55.818155168Z 2026-05-15 16:22:55.817 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/techproducts 2026-05-15T16:22:55.820079156Z 2026-05-15 16:22:55.819 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/techproducts/managed-schema.xml 2026-05-15T16:22:55.822513890Z 2026-05-15 16:22:55.822 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest 2026-05-15T16:22:55.841798598Z 2026-05-15 16:22:55.841 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, accuracy=0.5, maxQueryFrequency=0.01} 2026-05-15T16:22:55.849625774Z 2026-05-15 16:22:55.849 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms 2026-05-15T16:22:55.856881812Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.QuerySenderListener QuerySenderListener done. 2026-05-15T16:22:55.856923791Z 2026-05-15 16:22:55.856 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default 2026-05-15T16:22:55.863215517Z 2026-05-15 16:22:55.862 INFO (searcherExecutor-22-thread-1-processing-test-collection_shard2_replica_n2 solr-node1-8 core_node5 solr-node3:8985_solr test-collection shard2) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms 2026-05-15T16:22:55.888899138Z 2026-05-15 16:22:55.888 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkShardTerms Successful update of terms at /collections/test-collection/terms/shard2 to Terms{values={core_node5=0}, version=0} for registerTerm 2026-05-15T16:22:55.890224232Z 2026-05-15 16:22:55.889 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/test-collection/leaders/shard2 2026-05-15T16:22:55.914776531Z 2026-05-15 16:22:55.914 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. 2026-05-15T16:22:55.914783890Z 2026-05-15 16:22:55.914 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync 2026-05-15T16:22:55.914786688Z 2026-05-15 16:22:55.914 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync replicas to http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ 2026-05-15T16:22:55.916627658Z 2026-05-15 16:22:55.916 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me 2026-05-15T16:22:55.917459848Z 2026-05-15 16:22:55.917 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.SyncStrategy http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ has no replicas 2026-05-15T16:22:55.918495297Z 2026-05-15 16:22:55.918 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/test-collection/leaders/shard2/leader after winning as /collections/test-collection/leader_elect/shard2/election/72059903898681347-core_node5-n_0000000000 2026-05-15T16:22:55.933367104Z 2026-05-15 16:22:55.932 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://solr-node3:8985/solr/test-collection_shard2_replica_n2/ shard2 2026-05-15T16:22:56.044210593Z 2026-05-15 16:22:56.043 INFO (zkCallback-13-thread-1) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 218] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:56.048066330Z 2026-05-15 16:22:56.047 INFO (qtp985324122-58-solr-node1-8) [c:test-collection s:shard2 r:core_node5 x:test-collection_shard2_replica_n2 t:solr-node1-8] o.a.s.c.ZkController I am the leader, no recovery necessary 2026-05-15T16:22:56.058347277Z 2026-05-15 16:22:56.058 INFO (qtp985324122-58-solr-node1-8) [c: s: r: x: t:solr-node1-8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=techproducts&newCollection=true&name=test-collection_shard2_replica_n2&action=CREATE&numShards=3&collection=test-collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1806 2026-05-15T16:22:56.162026545Z 2026-05-15 16:22:56.161 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 222] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:58.415922758Z 2026-05-15 16:22:58.415 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 237] for collection [test-collection] has occurred - updating... (live nodes size: [3]) 2026-05-15T16:22:58.526597510Z 2026-05-15 16:22:58.526 INFO (zkCallback-13-thread-2) [c: s: r: x: t:] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test-collection/state.json zxid: 243] for collection [test-collection] has occurred - updating... (live nodes size: [3]) # === Docker inspect for solr-node3 === { "Name": "/solr-node3", "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 1458155, "ExitCode": 0, "Error": "", "StartedAt": "2026-05-15T16:22:41.414999113Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Ports": { "8983/tcp": null } } # Docker artifacts saved to: /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build/test-output/docker not ok 83 Docker SolrCloud rolling upgrade # in 33135 ms # (in test file test/test_rolling_upgrade.bats, line 118) # `wait_for 30 1 docker exec solr-node1 solr healthcheck -c test-collection' failed with status 127 # 9.10.0-SNAPSHOT-slim: Pulling from apache/solr-nightly # Digest: sha256:15c8155ceffe1042f37c77bb537aa531804a3510702ff87f2af5fedf523f4203 # Status: Image is up to date for apache/solr-nightly:9.10.0-SNAPSHOT-slim # docker.io/apache/solr-nightly:9.10.0-SNAPSHOT-slim # 10.0.0-SNAPSHOT-slim: Pulling from apache/solr-nightly # Digest: sha256:7b9bb30f4dc4d340f52255f6cab0a7e3bc0bf4e6c8f5b0d5e0744356ef86b458 # Status: Image is up to date for apache/solr-nightly:10.0.0-SNAPSHOT-slim # docker.io/apache/solr-nightly:10.0.0-SNAPSHOT-slim # 241765af35e954f0d0c4b276a2481823836dba20b6a8794cc2cb31d24c6c3c4f # solr-data1 # solr-data2 # solr-data3 # Starting solr-node1 with embedded ZooKeeper # 7b256613583cc2d5ad747092de8aef692fbbf9929bb95c80141cf9108506f438 # Starting solr-node2 connected to first node's ZooKeeper # 578e8e5144cb6661b822ec468ce1c04a3734d993d1d08f33a220883567834858 # Started solr-node3 connected to first node's ZooKeeper # 8d6ce433f0356088d4b73bf98a3f5a28d2cc219a2eabf74e632d0d22271d8851 # Creating a Collection # Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983. # Neither --zk-host or --solr-url parameters provided so assuming solr url is http://localhost:8983. # WARN - 2026-05-15 16:22:53.291; org.apache.solr.common.cloud.SolrZkClient; Using default ZkCredentialsInjector. ZkCredentialsInjector is not secure, it creates an empty list of credentials which leads to 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes # WARN - 2026-05-15 16:22:53.386; org.apache.solr.common.cloud.SolrZkClient; Using default ZkACLProvider. DefaultZkACLProvider is not secure, it creates 'OPEN_ACL_UNSAFE' ACLs to Zookeeper nodes # WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. "/solr"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [http://solr-node1:8983/solr] to [http://solr-node1:8983/]. # Created collection 'test-collection' with 3 shard(s), 1 replica(s) with config-set 'techproducts' # Checking collection health # /home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/test/test_rolling_upgrade.bats: line 118: wait_for: command not found # solr-node1 # solr-node2 # solr-node3 # solr-node1 # solr-node2 # solr-node3 # solr-data1 # solr-data2 # solr-data3 # solrcloud-test ok 84 allowPaths - backup # in 15187 ms ok 85 snapshot lifecycle # in 4713 ms ok 86 snapshot list # in 3297 ms ok 87 snapshot describe # in 3298 ms ok 88 start solr with ssl # in 15955 ms ok 89 use different hostname when not checking peer-name # in 24414 ms ok 90 start solr with ssl and auth # in 14272 ms ok 91 start solr with client truststore and security manager # in 16089 ms ok 92 start solr with mTLS needed # in 45604 ms ok 93 start solr with mTLS wanted # in 34906 ms ok 94 test keystore reload # in 57897 ms ok 95 SOLR-11740 check 'solr stop' connection # in 15770 ms ok 96 stop command for single port # in 18112 ms ok 97 check stop command doesn't hang # in 213 ms ok 98 SOLR-16976 solr starts with remote JMX enabled # in 8244 ms ok 99 deprecated system properties converted to modern properties # in 194561 ms ok 100 start with custom jetty options # in 8003 ms ok 101 bootstrapping a configset # in 9158 ms ok 102 status detects locally running solr # in 9388 ms ok 103 status with --solr-url from user # in 7918 ms ok 104 status with --port from user # in 7983 ms ok 105 multiple connection options are prevented # in 843 ms ok 106 status with invalid --solr-url from user # in 1222 ms ok 107 status with --short format # in 195718 ms ok 108 searching solr via locally executed streaming expression # in 1708 ms ok 109 searching solr via remotely executed streaming expression # in 1270 ms ok 110 variable interpolation # in 1664 ms ok 111 searching solr without credentials fails with error # in 1112 ms ok 112 --version returns Solr version # in 724 ms ok 113 version as direct tool call still runs # in 722 ms ok 114 short help # in 762 ms ok 115 short help is inferred # in 754 ms ok 116 long help # in 762 ms ok 117 running subcommands with zk is prevented # in 702 ms ok 118 listing out files # in 2156 ms ok 119 connecting to solr via various solr urls and zk hosts # in 7698 ms ok 120 copying files around # in 10885 ms ok 121 upconfig # in 2575 ms ok 122 SOLR-12429 test upconfig fails with symlink # in 1182 ms ok 123 downconfig # in 1183 ms ok 124 bin/solr zk cp gets 'solrhome' from '--solr-home' command line option # in 1432 ms ok 125 env var ZK_HOST is honored # in 3160 ms ok 126 Cleanup # in 1841 ms > Task :solr:packaging:integrationTests FAILED FAILURE: Build failed with an exception. * Where: Build file '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/build.gradle' line: 339 * What went wrong: Execution failed for task ':solr:packaging:integrationTests'. > Process 'command > '/home/jenkins/jenkins-agent/workspace/Solr/Solr-TestIntegration-10.x/solr/packaging/.gradle/node/node_modules/bats/bin/bats'' > finished with non-zero exit value 1 * Try: > Run with --stacktrace option to get the stack trace. > Run with --info or --debug option to get more log output. > Get more help at https://help.gradle.org. Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0. You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins. For more on this, please refer to https://docs.gradle.org/8.10/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation. BUILD FAILED in 29m 33s 128 actionable tasks: 128 executed Publishing Build Scan to Develocity... https://develocity.apache.org/s/vi4h6xumkait4 Build step 'Invoke Gradle script' changed build result to FAILURE Build step 'Invoke Gradle script' marked build as failure Archiving artifacts Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
