FWIW: I'm seeing a bunch of test failures that look suspiciously related to this commit .... i'll post details in jira.
: Date: Wed, 21 Jun 2017 15:26:00 -0000 : From: da...@apache.org : Reply-To: dev@lucene.apache.org : To: comm...@lucene.apache.org : Subject: [1/2] lucene-solr:master: SOLR-8256: Set legacyCloud=false as default : : Repository: lucene-solr : Updated Branches: : refs/heads/master eff583ee8 -> 8e9d685a4 : : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java : ---------------------------------------------------------------------- : diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java : index 2d327a2..f6abb54 100644 : --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java : +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java : @@ -29,7 +29,6 @@ import java.util.List; : import java.util.Locale; : import java.util.Map; : import java.util.Set; : -import java.util.concurrent.ExecutorService; : import java.util.concurrent.TimeUnit; : import java.util.concurrent.TimeoutException; : import java.util.concurrent.atomic.AtomicInteger; : @@ -47,13 +46,11 @@ import org.apache.solr.common.cloud.SolrZkClient; : import org.apache.solr.common.cloud.ZkNodeProps; : import org.apache.solr.common.cloud.ZkStateReader; : import org.apache.solr.common.params.CollectionParams; : -import org.apache.solr.common.util.ExecutorUtil; : import org.apache.solr.common.util.Utils; : import org.apache.solr.core.CloudConfig; : import org.apache.solr.handler.component.HttpShardHandlerFactory; : import org.apache.solr.update.UpdateShardHandler; : import org.apache.solr.update.UpdateShardHandlerConfig; : -import org.apache.solr.util.DefaultSolrThreadFactory; : import org.apache.zookeeper.CreateMode; : import org.apache.zookeeper.KeeperException; : import org.apache.zookeeper.KeeperException.NoNodeException; : @@ -131,8 +128,20 @@ public class OverseerTest extends SolrTestCaseJ4 { : zkStateReader.close(); : zkClient.close(); : } : - : - public String publishState(String collection, String coreName, String coreNodeName, Replica.State stateName, int numShards) : + : + public void createCollection(String collection, int numShards) throws KeeperException, InterruptedException { : + : + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), : + "name", collection, : + ZkStateReader.REPLICATION_FACTOR, "1", : + ZkStateReader.NUM_SHARDS_PROP, numShards+"", : + "createNodeSet", ""); : + DistributedQueue q = Overseer.getStateUpdateQueue(zkClient); : + q.offer(Utils.toJSON(m)); : + : + } : + : + public String publishState(String collection, String coreName, String coreNodeName, String shard, Replica.State stateName, int numShards) : throws KeeperException, InterruptedException, IOException { : if (stateName == null) { : ElectionContext ec = electionContext.remove(coreName); : @@ -144,22 +153,23 @@ public class OverseerTest extends SolrTestCaseJ4 { : ZkStateReader.CORE_NAME_PROP, coreName, : ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName, : ZkStateReader.COLLECTION_PROP, collection); : - DistributedQueue q = Overseer.getStateUpdateQueue(zkClient); : - q.offer(Utils.toJSON(m)); : - return null; : + DistributedQueue q = Overseer.getStateUpdateQueue(zkClient); : + q.offer(Utils.toJSON(m)); : + return null; : } else { : ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : - ZkStateReader.STATE_PROP, stateName.toString(), : - ZkStateReader.NODE_NAME_PROP, nodeName, : - ZkStateReader.CORE_NAME_PROP, coreName, : - ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName, : - ZkStateReader.COLLECTION_PROP, collection, : - ZkStateReader.NUM_SHARDS_PROP, Integer.toString(numShards), : - ZkStateReader.BASE_URL_PROP, "http://"; + nodeName + "/solr/"); : + ZkStateReader.STATE_PROP, stateName.toString(), : + ZkStateReader.NODE_NAME_PROP, nodeName, : + ZkStateReader.CORE_NAME_PROP, coreName, : + ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName, : + ZkStateReader.COLLECTION_PROP, collection, : + ZkStateReader.SHARD_ID_PROP, shard, : + ZkStateReader.NUM_SHARDS_PROP, Integer.toString(numShards), : + ZkStateReader.BASE_URL_PROP, "http://"; + nodeName + "/solr/"); : DistributedQueue q = Overseer.getStateUpdateQueue(zkClient); : q.offer(Utils.toJSON(m)); : } : - : + : if (collection.length() > 0) { : for (int i = 0; i < 120; i++) { : String shardId = getShardId(collection, coreNodeName); : @@ -193,7 +203,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : } : return null; : } : - : + : private String getShardId(String collection, String coreNodeName) { : Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(collection); : if (slices != null) { : @@ -257,7 +267,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : server.run(); : AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost()); : AbstractZkTestCase.makeSolrZkNode(server.getZkHost()); : - : + : zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT); : ZkController.createClusterZkNodes(zkClient); : : @@ -265,24 +275,32 @@ public class OverseerTest extends SolrTestCaseJ4 { : : ZkStateReader reader = new ZkStateReader(zkClient); : reader.createClusterStateWatchersAndUpdate(); : - : + : zkController = new MockZKController(server.getZkAddress(), "127.0.0.1"); : : final int numShards=6; : - : + : + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), : + "name", COLLECTION, : + ZkStateReader.REPLICATION_FACTOR, "1", : + ZkStateReader.NUM_SHARDS_PROP, "3", : + "createNodeSet", ""); : + DistributedQueue q = Overseer.getStateUpdateQueue(zkClient); : + q.offer(Utils.toJSON(m)); : + : for (int i = 0; i < numShards; i++) { : - assertNotNull("shard got no id?", zkController.publishState(COLLECTION, "core" + (i+1), "node" + (i+1), Replica.State.ACTIVE, 3)); : + assertNotNull("shard got no id?", zkController.publishState(COLLECTION, "core" + (i+1), "node" + (i+1), "shard"+((i%3)+1), Replica.State.ACTIVE, 3)); : } : final Map<String,Replica> rmap = reader.getClusterState().getSlice(COLLECTION, "shard1").getReplicasMap(); : assertEquals(rmap.toString(), 2, rmap.size()); : assertEquals(rmap.toString(), 2, reader.getClusterState().getSlice(COLLECTION, "shard2").getReplicasMap().size()); : assertEquals(rmap.toString(), 2, reader.getClusterState().getSlice(COLLECTION, "shard3").getReplicasMap().size()); : - : + : //make sure leaders are in cloud state : assertNotNull(reader.getLeaderUrl(COLLECTION, "shard1", 15000)); : assertNotNull(reader.getLeaderUrl(COLLECTION, "shard2", 15000)); : assertNotNull(reader.getLeaderUrl(COLLECTION, "shard3", 15000)); : - : + : } finally { : close(zkClient); : if (zkController != null) { : @@ -319,9 +337,10 @@ public class OverseerTest extends SolrTestCaseJ4 { : zkController = new MockZKController(server.getZkAddress(), "127.0.0.1"); : : final int numShards=3; : - : + zkController.createCollection(COLLECTION, 3); : for (int i = 0; i < numShards; i++) { : - assertNotNull("shard got no id?", zkController.publishState(COLLECTION, "core" + (i+1), "node" + (i+1), Replica.State.ACTIVE, 3)); : + assertNotNull("shard got no id?", zkController.publishState(COLLECTION, "core" + (i+1), : + "node" + (i+1), "shard"+((i%3)+1) , Replica.State.ACTIVE, 3)); : } : : assertEquals(1, reader.getClusterState().getSlice(COLLECTION, "shard1").getReplicasMap().size()); : @@ -335,12 +354,14 @@ public class OverseerTest extends SolrTestCaseJ4 { : : // publish a bad queue item : String emptyCollectionName = ""; : - zkController.publishState(emptyCollectionName, "core0", "node0", Replica.State.ACTIVE, 1); : - zkController.publishState(emptyCollectionName, "core0", "node0", null, 1); : - : + zkController.publishState(emptyCollectionName, "core0", "node0", "shard1", Replica.State.ACTIVE, 1); : + zkController.publishState(emptyCollectionName, "core0", "node0", "shard1", null, 1); : + : + zkController.createCollection("collection2", 3); : // make sure the Overseer is still processing items : for (int i = 0; i < numShards; i++) { : - assertNotNull("shard got no id?", zkController.publishState("collection2", "core" + (i + 1), "node" + (i + 1), Replica.State.ACTIVE, 3)); : + assertNotNull("shard got no id?", zkController.publishState("collection2", : + "core" + (i + 1), "node" + (i + 1),"shard"+((i%3)+1), Replica.State.ACTIVE, 3)); : } : : assertEquals(1, reader.getClusterState().getSlice("collection2", "shard1").getReplicasMap().size()); : @@ -361,147 +382,6 @@ public class OverseerTest extends SolrTestCaseJ4 { : server.shutdown(); : } : } : - : - @Test : - public void testShardAssignmentBigger() throws Exception { : - String zkDir = createTempDir("zkData").toFile().getAbsolutePath(); : - : - final int nodeCount = random().nextInt(TEST_NIGHTLY ? 50 : 10)+(TEST_NIGHTLY ? 50 : 10)+1; //how many simulated nodes (num of threads) : - final int coreCount = random().nextInt(TEST_NIGHTLY ? 100 : 11)+(TEST_NIGHTLY ? 100 : 11)+1; //how many cores to register : - final int sliceCount = random().nextInt(TEST_NIGHTLY ? 20 : 5)+1; //how many slices : - : - ZkTestServer server = new ZkTestServer(zkDir); : - : - SolrZkClient zkClient = null; : - ZkStateReader reader = null; : - SolrZkClient overseerClient = null; : - : - final MockZKController[] controllers = new MockZKController[nodeCount]; : - final ExecutorService[] nodeExecutors = new ExecutorService[nodeCount]; : - try { : - server.run(); : - AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost()); : - AbstractZkTestCase.makeSolrZkNode(server.getZkHost()); : - : - zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT); : - ZkController.createClusterZkNodes(zkClient); : - : - overseerClient = electNewOverseer(server.getZkAddress()); : - : - reader = new ZkStateReader(zkClient); : - reader.createClusterStateWatchersAndUpdate(); : - : - for (int i = 0; i < nodeCount; i++) { : - controllers[i] = new MockZKController(server.getZkAddress(), "node" + i); : - } : - for (int i = 0; i < nodeCount; i++) { : - nodeExecutors[i] = ExecutorUtil.newMDCAwareFixedThreadPool(1, new DefaultSolrThreadFactory("testShardAssignment")); : - } : - : - final String[] ids = new String[coreCount]; : - //register total of coreCount cores : - for (int i = 0; i < coreCount; i++) { : - final int slot = i; : - : - nodeExecutors[i % nodeCount].submit((Runnable) () -> { : - : - final String coreName = "core" + slot; : - : - try { : - ids[slot] = controllers[slot % nodeCount].publishState(COLLECTION, coreName, "node" + slot, Replica.State.ACTIVE, sliceCount); : - } catch (Throwable e) { : - e.printStackTrace(); : - fail("register threw exception:" + e.getClass()); : - } : - }); : - } : - : - for (int i = 0; i < nodeCount; i++) { : - nodeExecutors[i].shutdown(); : - } : - : - for (int i = 0; i < nodeCount; i++) { : - while (!nodeExecutors[i].awaitTermination(100, TimeUnit.MILLISECONDS)); : - } : - : - // make sure all cores have been assigned a id in cloudstate : - int cloudStateSliceCount = 0; : - for (int i = 0; i < 40; i++) { : - cloudStateSliceCount = 0; : - ClusterState state = reader.getClusterState(); : - final Map<String,Slice> slices = state.getSlicesMap(COLLECTION); : - if (slices != null) { : - for (String name : slices.keySet()) { : - cloudStateSliceCount += slices.get(name).getReplicasMap().size(); : - } : - if (coreCount == cloudStateSliceCount) break; : - } : - : - Thread.sleep(200); : - } : - assertEquals("Unable to verify all cores have been assigned an id in cloudstate", : - coreCount, cloudStateSliceCount); : - : - // make sure all cores have been returned an id : - int assignedCount = 0; : - for (int i = 0; i < 240; i++) { : - assignedCount = 0; : - for (int j = 0; j < coreCount; j++) { : - if (ids[j] != null) { : - assignedCount++; : - } : - } : - if (coreCount == assignedCount) { : - break; : - } : - Thread.sleep(1000); : - } : - assertEquals("Unable to verify all cores have been returned an id", : - coreCount, assignedCount); : - : - final HashMap<String, AtomicInteger> counters = new HashMap<>(); : - for (int i = 1; i < sliceCount+1; i++) { : - counters.put("shard" + i, new AtomicInteger()); : - } : - : - for (int i = 0; i < coreCount; i++) { : - final AtomicInteger ai = counters.get(ids[i]); : - assertNotNull("could not find counter for shard:" + ids[i], ai); : - ai.incrementAndGet(); : - } : - : - for (String counter: counters.keySet()) { : - int count = counters.get(counter).intValue(); : - int expectedCount = coreCount / sliceCount; : - int min = expectedCount - 1; : - int max = expectedCount + 1; : - if (count < min || count > max) { : - fail("Unevenly assigned shard ids, " + counter + " had " + count : - + ", expected: " + min + "-" + max); : - } : - } : - : - //make sure leaders are in cloud state : - for (int i = 0; i < sliceCount; i++) { : - assertNotNull(reader.getLeaderUrl(COLLECTION, "shard" + (i + 1), 15000)); : - } : - : - } finally { : - close(zkClient); : - close(overseerClient); : - close(reader); : - for (int i = 0; i < controllers.length; i++) : - if (controllers[i] != null) { : - controllers[i].close(); : - } : - server.shutdown(); : - for (int i = 0; i < nodeCount; i++) { : - if (nodeExecutors[i] != null) { : - nodeExecutors[i].shutdownNow(); : - } : - } : - } : - } : : //wait until collections are available : private void waitForCollections(ZkStateReader stateReader, String... collections) throws InterruptedException, KeeperException { : @@ -545,11 +425,19 @@ public class OverseerTest extends SolrTestCaseJ4 { : overseerClient = electNewOverseer(server.getZkAddress()); : : DistributedQueue q = Overseer.getStateUpdateQueue(zkClient); : - : - ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : + : + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), : + "name", COLLECTION, : + ZkStateReader.REPLICATION_FACTOR, "1", : + ZkStateReader.NUM_SHARDS_PROP, "1", : + "createNodeSet", ""); : + q.offer(Utils.toJSON(m)); : + : + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : ZkStateReader.NODE_NAME_PROP, "node1", : ZkStateReader.COLLECTION_PROP, COLLECTION, : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.CORE_NAME_PROP, "core1", : ZkStateReader.ROLES_PROP, "", : ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString()); : @@ -557,15 +445,14 @@ public class OverseerTest extends SolrTestCaseJ4 { : q.offer(Utils.toJSON(m)); : : waitForCollections(reader, COLLECTION); : - : - assertSame(reader.getClusterState().toString(), Replica.State.RECOVERING, : - reader.getClusterState().getSlice(COLLECTION, "shard1").getReplica("core_node1").getState()); : + verifyReplicaStatus(reader, "collection1", "shard1", "core_node1", Replica.State.RECOVERING); : : //publish node state (active) : m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : ZkStateReader.NODE_NAME_PROP, "node1", : ZkStateReader.COLLECTION_PROP, COLLECTION, : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.CORE_NAME_PROP, "core1", : ZkStateReader.ROLES_PROP, "", : ZkStateReader.STATE_PROP, Replica.State.ACTIVE.toString()); : @@ -634,7 +521,8 @@ public class OverseerTest extends SolrTestCaseJ4 { : overseerClient = electNewOverseer(server.getZkAddress()); : : Thread.sleep(1000); : - mockController.publishState(COLLECTION, core, core_node, : + mockController.createCollection(COLLECTION, 1); : + mockController.publishState(COLLECTION, core, core_node, "shard1", : Replica.State.RECOVERING, numShards); : : waitForCollections(reader, COLLECTION); : @@ -642,7 +530,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : : int version = getClusterStateVersion(zkClient); : : - mockController.publishState(COLLECTION, core, core_node, Replica.State.ACTIVE, : + mockController.publishState(COLLECTION, core, core_node, "shard1", Replica.State.ACTIVE, : numShards); : : while (version == getClusterStateVersion(zkClient)); : @@ -652,7 +540,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : overseerClient.close(); : Thread.sleep(1000); // wait for overseer to get killed : : - mockController.publishState(COLLECTION, core, core_node, : + mockController.publishState(COLLECTION, core, core_node, "shard1", : Replica.State.RECOVERING, numShards); : version = getClusterStateVersion(zkClient); : : @@ -667,7 +555,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : assertEquals(shard+" replica count does not match", 1, reader.getClusterState() : .getSlice(COLLECTION, shard).getReplicasMap().size()); : version = getClusterStateVersion(zkClient); : - mockController.publishState(COLLECTION, core, core_node, null, numShards); : + mockController.publishState(COLLECTION, core, core_node, "shard1", null, numShards); : while (version == getClusterStateVersion(zkClient)); : Thread.sleep(500); : assertTrue(COLLECTION +" should remain after removal of the last core", // as of SOLR-5209 core removal does not cascade to remove the slice and collection : @@ -723,7 +611,8 @@ public class OverseerTest extends SolrTestCaseJ4 { : overseerElector.setup(ec); : overseerElector.joinElection(ec, false); : : - mockController.publishState(COLLECTION, "core1", "core_node1", Replica.State.ACTIVE, 1); : + mockController.createCollection(COLLECTION, 1); : + mockController.publishState(COLLECTION, "core1", "core_node1", "shard1", Replica.State.ACTIVE, 1); : : assertNotNull(overseer.getStats()); : assertTrue((overseer.getStats().getSuccessCount(OverseerAction.STATE.toLower())) > 0); : @@ -819,16 +708,17 @@ public class OverseerTest extends SolrTestCaseJ4 { : for (int i = 0; i < atLeast(4); i++) { : killCounter.incrementAndGet(); //for each round allow 1 kill : mockController = new MockZKController(server.getZkAddress(), "node1"); : - mockController.publishState(COLLECTION, "core1", "node1", Replica.State.ACTIVE,1); : + mockController.createCollection(COLLECTION, 1); : + mockController.publishState(COLLECTION, "core1", "node1", "shard1", Replica.State.ACTIVE,1); : if(mockController2!=null) { : mockController2.close(); : mockController2 = null; : } : - mockController.publishState(COLLECTION, "core1", "node1",Replica.State.RECOVERING,1); : + mockController.publishState(COLLECTION, "core1", "node1","shard1", Replica.State.RECOVERING,1); : mockController2 = new MockZKController(server.getZkAddress(), "node2"); : - mockController.publishState(COLLECTION, "core1", "node1", Replica.State.ACTIVE,1); : + mockController.publishState(COLLECTION, "core1", "node1","shard1", Replica.State.ACTIVE,1); : verifyShardLeader(reader, COLLECTION, "shard1", "core1"); : - mockController2.publishState(COLLECTION, "core4", "node2", Replica.State.ACTIVE ,1); : + mockController2.publishState(COLLECTION, "core4", "node2", "shard1", Replica.State.ACTIVE ,1); : mockController.close(); : mockController = null; : verifyShardLeader(reader, COLLECTION, "shard1", "core4"); : @@ -874,7 +764,8 @@ public class OverseerTest extends SolrTestCaseJ4 { : : overseerClient = electNewOverseer(server.getZkAddress()); : : - mockController.publishState(COLLECTION, "core1", "core_node1", Replica.State.RECOVERING, 1); : + mockController.createCollection(COLLECTION, 1); : + mockController.publishState(COLLECTION, "core1", "core_node1", "shard1", Replica.State.RECOVERING, 1); : : waitForCollections(reader, "collection1"); : : @@ -885,7 +776,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : int version = getClusterStateVersion(controllerClient); : : mockController = new MockZKController(server.getZkAddress(), "node1"); : - mockController.publishState(COLLECTION, "core1", "core_node1", Replica.State.RECOVERING, 1); : + mockController.publishState(COLLECTION, "core1", "core_node1","shard1", Replica.State.RECOVERING, 1); : : while (version == reader.getClusterState().getZkClusterStateVersion()) { : Thread.sleep(100); : @@ -915,47 +806,6 @@ public class OverseerTest extends SolrTestCaseJ4 { : } : : @Test : - public void testPlaceholders() throws Exception { : - String zkDir = createTempDir("zkData").toFile().getAbsolutePath(); : - : - ZkTestServer server = new ZkTestServer(zkDir); : - : - SolrZkClient controllerClient = null; : - SolrZkClient overseerClient = null; : - ZkStateReader reader = null; : - MockZKController mockController = null; : - : - try { : - server.run(); : - controllerClient = new SolrZkClient(server.getZkAddress(), TIMEOUT); : - : - AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost()); : - AbstractZkTestCase.makeSolrZkNode(server.getZkHost()); : - ZkController.createClusterZkNodes(controllerClient); : - : - reader = new ZkStateReader(controllerClient); : - reader.createClusterStateWatchersAndUpdate(); : - : - mockController = new MockZKController(server.getZkAddress(), "node1"); : - : - overseerClient = electNewOverseer(server.getZkAddress()); : - : - mockController.publishState(COLLECTION, "core1", "node1", Replica.State.RECOVERING, 12); : - : - waitForCollections(reader, COLLECTION); : - : - assertEquals("Slicecount does not match", 12, reader.getClusterState().getSlices(COLLECTION).size()); : - : - } finally { : - close(overseerClient); : - close(mockController); : - close(controllerClient); : - close(reader); : - server.shutdown(); : - } : - } : - : - @Test : @Ignore : public void testPerformance() throws Exception { : String zkDir = createTempDir("OverseerTest.testPerformance").toFile().getAbsolutePath(); : @@ -1113,10 +963,17 @@ public class OverseerTest extends SolrTestCaseJ4 { : reader.createClusterStateWatchersAndUpdate(); : //prepopulate work queue with some items to emulate previous overseer died before persisting state : DistributedQueue queue = Overseer.getInternalWorkQueue(zkClient, new Overseer.Stats()); : - ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : + : + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), : + "name", COLLECTION, : + ZkStateReader.REPLICATION_FACTOR, "1", : + ZkStateReader.NUM_SHARDS_PROP, "1", : + "createNodeSet", ""); : + queue.offer(Utils.toJSON(m)); : + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : ZkStateReader.NODE_NAME_PROP, "node1", : - ZkStateReader.SHARD_ID_PROP, "s1", : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.COLLECTION_PROP, COLLECTION, : ZkStateReader.CORE_NAME_PROP, "core1", : ZkStateReader.ROLES_PROP, "", : @@ -1125,7 +982,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : ZkStateReader.NODE_NAME_PROP, "node1", : - ZkStateReader.SHARD_ID_PROP, "s1", : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.COLLECTION_PROP, COLLECTION, : ZkStateReader.CORE_NAME_PROP, "core2", : ZkStateReader.ROLES_PROP, "", : @@ -1139,7 +996,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : ZkStateReader.NODE_NAME_PROP, "node1", : - ZkStateReader.SHARD_ID_PROP, "s1", : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.COLLECTION_PROP, COLLECTION, : ZkStateReader.CORE_NAME_PROP, "core3", : ZkStateReader.ROLES_PROP, "", : @@ -1147,12 +1004,12 @@ public class OverseerTest extends SolrTestCaseJ4 { : queue.offer(Utils.toJSON(m)); : : for(int i=0;i<100;i++) { : - Slice s = reader.getClusterState().getSlice(COLLECTION, "s1"); : + Slice s = reader.getClusterState().getSlice(COLLECTION, "shard1"); : if(s!=null && s.getReplicasMap().size()==3) break; : Thread.sleep(100); : } : - assertNotNull(reader.getClusterState().getSlice(COLLECTION, "s1")); : - assertEquals(3, reader.getClusterState().getSlice(COLLECTION, "s1").getReplicasMap().size()); : + assertNotNull(reader.getClusterState().getSlice(COLLECTION, "shard1")); : + assertEquals(3, reader.getClusterState().getSlice(COLLECTION, "shard1").getReplicasMap().size()); : } finally { : close(overseerClient); : close(zkClient); : @@ -1188,8 +1045,17 @@ public class OverseerTest extends SolrTestCaseJ4 { : : DistributedQueue q = Overseer.getStateUpdateQueue(zkClient); : : - ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : + : + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), : + "name", "c1", : + ZkStateReader.REPLICATION_FACTOR, "1", : + ZkStateReader.NUM_SHARDS_PROP, "1", : + "createNodeSet", ""); : + q.offer(Utils.toJSON(m)); : + : + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.NODE_NAME_PROP, "node1", : ZkStateReader.COLLECTION_PROP, "c1", : ZkStateReader.CORE_NAME_PROP, "core1", : @@ -1203,6 +1069,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : : m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.NODE_NAME_PROP, "node1", : ZkStateReader.COLLECTION_PROP, "c1", : ZkStateReader.CORE_NAME_PROP, "core1", : @@ -1214,6 +1081,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : : m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : + ZkStateReader.SHARD_ID_PROP, "shard1", : ZkStateReader.NODE_NAME_PROP, "node1", : ZkStateReader.COLLECTION_PROP, "c1", : ZkStateReader.CORE_NAME_PROP, "core1", : @@ -1379,6 +1247,7 @@ public class OverseerTest extends SolrTestCaseJ4 { : final int N = (numReplicas-rr)*numShards + ss; : ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), : ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr";, : + ZkStateReader.SHARD_ID_PROP, "shard"+ss, : ZkStateReader.NODE_NAME_PROP, "node"+N, : ZkStateReader.COLLECTION_PROP, COLLECTION, : ZkStateReader.CORE_NAME_PROP, "core"+N, : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java : ---------------------------------------------------------------------- : diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java : index d7fae92..edbeb50 100644 : --- a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java : +++ b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java : @@ -83,7 +83,7 @@ public class ReplaceNodeTest extends SolrCloudTestCase { : new CollectionAdminRequest.ReplaceNode(node2bdecommissioned, emptyNode).processAsync("000", cloudClient); : CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("000"); : boolean success = false; : - for (int i = 0; i < 200; i++) { : + for (int i = 0; i < 300; i++) { : CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); : if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { : success = true; : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java : ---------------------------------------------------------------------- : diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java : index c5e35c3..78e4128 100644 : --- a/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java : +++ b/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java : @@ -20,6 +20,8 @@ import java.io.File; : : import org.apache.solr.client.solrj.SolrClient; : import org.apache.solr.client.solrj.embedded.JettySolrRunner; : +import org.apache.solr.client.solrj.request.CollectionAdminRequest; : +import org.apache.solr.common.cloud.Replica; : import org.junit.BeforeClass; : import org.junit.Test; : : @@ -60,6 +62,16 @@ public class ShardRoutingCustomTest extends AbstractFullDistribZkTestBase { : jettyDir.mkdirs(); : setupJettySolrHome(jettyDir); : JettySolrRunner j = createJetty(jettyDir, createTempDir().toFile().getAbsolutePath(), "shardA", "solrconfig.xml", null); : + assertEquals(0, CollectionAdminRequest : + .createCollection(DEFAULT_COLLECTION, 1, 1) : + .setStateFormat(Integer.parseInt(getStateFormat())) : + .setCreateNodeSet("") : + .process(cloudClient).getStatus()); : + assertTrue(CollectionAdminRequest : + .addReplicaToShard(collection,"shard1") : + .setNode(j.getNodeName()) : + .setType(useTlogReplicas()? Replica.Type.TLOG: Replica.Type.NRT) : + .process(cloudClient).isSuccess()); : jettys.add(j); : SolrClient client = createNewSolrClient(j.getLocalPort()); : clients.add(client); : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java : ---------------------------------------------------------------------- : diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java : index b77389b..c51904f 100644 : --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java : +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java : @@ -21,8 +21,9 @@ import org.apache.solr.SolrTestCaseJ4.SuppressSSL; : import org.apache.solr.client.solrj.SolrClient; : import org.apache.solr.client.solrj.SolrQuery; : import org.apache.solr.client.solrj.SolrServerException; : +import org.apache.solr.client.solrj.embedded.JettySolrRunner; : import org.apache.solr.client.solrj.impl.HttpSolrClient; : -import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; : +import org.apache.solr.client.solrj.request.CollectionAdminRequest; : import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; : import org.apache.solr.common.SolrInputDocument; : import org.apache.solr.common.cloud.Replica; : @@ -32,12 +33,12 @@ import org.apache.solr.common.cloud.ZkStateReader; : import org.apache.solr.common.params.ModifiableSolrParams; : import org.apache.solr.common.params.SolrParams; : import org.apache.solr.common.util.ExecutorUtil; : +import org.apache.solr.core.SolrCore; : import org.apache.solr.update.DirectUpdateHandler2; : import org.apache.solr.util.DefaultSolrThreadFactory; : import org.apache.solr.util.TimeOut; : import org.junit.Test; : : -import java.io.File; : import java.io.IOException; : import java.util.Collection; : import java.util.Random; : @@ -108,37 +109,28 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : final String coreName1 = collection+"_1"; : final String coreName2 = collection+"_2"; : : - // create one leader and one replica : - Create createCmd = new Create(); : - createCmd.setCoreName(coreName1); : - createCmd.setCollection(collection); : - String coreDataDir = createTempDir().toFile().getAbsolutePath(); : - createCmd.setDataDir(getDataDir(coreDataDir)); : - createCmd.setNumShards(numShards); : - : - SolrClient client = clients.get(0); : - String url1 = getBaseUrl(client); : - : - try (HttpSolrClient adminClient = getHttpSolrClient(url1)) { : - adminClient.setConnectionTimeout(15000); : - adminClient.setSoTimeout(60000); : - adminClient.request(createCmd); : + assertEquals(0, CollectionAdminRequest.createCollection(collection, numShards, 1) : + .setCreateNodeSet("") : + .process(cloudClient).getStatus()); : + assertTrue(CollectionAdminRequest.addReplicaToShard(collection, "shard1") : + .setCoreName(coreName1) : + .setNode(jettys.get(0).getNodeName()) : + .process(cloudClient).isSuccess()); : : - createCmd = new Create(); : - createCmd.setCoreName(coreName2); : - createCmd.setCollection(collection); : - coreDataDir = createTempDir().toFile().getAbsolutePath(); : - createCmd.setDataDir(getDataDir(coreDataDir)); : + assertTrue(CollectionAdminRequest.addReplicaToShard(collection, "shard2") : + .setCoreName(coreName2) : + .setNode(jettys.get(0).getNodeName()) : + .process(cloudClient).isSuccess()); : : - adminClient.request(createCmd); : : - // does not mean they are active and up yet :* : - waitForRecoveriesToFinish(collection, false); : + // does not mean they are active and up yet :* : + waitForRecoveriesToFinish(collection, false); : : - final boolean unloadInOrder = random().nextBoolean(); : - final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2); : - final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1); : + final boolean unloadInOrder = random().nextBoolean(); : + final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2); : + final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1); : : + try (HttpSolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) { : // now unload one of the two : Unload unloadCmd = new Unload(false); : unloadCmd.setCoreName(unloadCmdCoreName1); : @@ -163,42 +155,26 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : * @throws Exception on any problem : */ : private void testCoreUnloadAndLeaders() throws Exception { : - File tmpDir = createTempDir().toFile(); : - : - String core1DataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection1" + "_1n"; : + JettySolrRunner jetty1 = jettys.get(0); : : - // create a new collection collection : - SolrClient client = clients.get(0); : - String url1 = getBaseUrl(client); : - try (HttpSolrClient adminClient = getHttpSolrClient(url1)) { : - adminClient.setConnectionTimeout(15000); : - adminClient.setSoTimeout(60000); : - : - Create createCmd = new Create(); : - createCmd.setCoreName("unloadcollection1"); : - createCmd.setCollection("unloadcollection"); : - createCmd.setNumShards(1); : - createCmd.setDataDir(getDataDir(core1DataDir)); : - adminClient.request(createCmd); : - } : + assertEquals(0, CollectionAdminRequest : + .createCollection("unloadcollection", 1,1) : + .setCreateNodeSet(jetty1.getNodeName()) : + .process(cloudClient).getStatus()); : ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader(); : : zkStateReader.forceUpdateCollection("unloadcollection"); : : int slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); : assertEquals(1, slices); : - : - client = clients.get(1); : - String url2 = getBaseUrl(client); : - try (HttpSolrClient adminClient = getHttpSolrClient(url2)) { : - : - Create createCmd = new Create(); : - createCmd.setCoreName("unloadcollection2"); : - createCmd.setCollection("unloadcollection"); : - String core2dataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection1" + "_2n"; : - createCmd.setDataDir(getDataDir(core2dataDir)); : - adminClient.request(createCmd); : - } : + SolrCore solrCore = jetty1.getCoreContainer().getCore("unloadcollection_shard1_replica1"); : + String core1DataDir = solrCore.getDataDir(); : + solrCore.close(); : + : + assertTrue(CollectionAdminRequest : + .addReplicaToShard("unloadcollection", "shard1") : + .setNode(jettys.get(1).getNodeName()) : + .process(cloudClient).isSuccess()); : zkStateReader.forceUpdateCollection("unloadcollection"); : slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); : assertEquals(1, slices); : @@ -225,24 +201,17 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : } : } : : - // create another replica for our collection : - client = clients.get(2); : - String url3 = getBaseUrl(client); : - try (HttpSolrClient adminClient = getHttpSolrClient(url3)) { : - Create createCmd = new Create(); : - createCmd.setCoreName("unloadcollection3"); : - createCmd.setCollection("unloadcollection"); : - String core3dataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection" + "_3n"; : - createCmd.setDataDir(getDataDir(core3dataDir)); : - adminClient.request(createCmd); : - } : - : + assertTrue(CollectionAdminRequest : + .addReplicaToShard("unloadcollection", "shard1") : + .setNode(jettys.get(2).getNodeName()) : + .process(cloudClient).isSuccess()); : + : waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); : - : + : // so that we start with some versions when we reload... : DirectUpdateHandler2.commitOnClose = false; : - : - try (HttpSolrClient addClient = getHttpSolrClient(url3 + "/unloadcollection3")) { : + : + try (HttpSolrClient addClient = getHttpSolrClient(jettys.get(2).getBaseUrl() + "/unloadcollection_shard1_replica3")) { : addClient.setConnectionTimeout(30000); : : // add a few docs : @@ -276,11 +245,11 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : fail("Leader never changed"); : } : } : - : + : // ensure there is a leader : zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); : - : - try (HttpSolrClient addClient = getHttpSolrClient(url2 + "/unloadcollection2")) { : + : + try (HttpSolrClient addClient = getHttpSolrClient(jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2")) { : addClient.setConnectionTimeout(30000); : addClient.setSoTimeout(90000); : : @@ -291,23 +260,14 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : addClient.add(doc1); : } : } : - : - // create another replica for our collection : - client = clients.get(3); : - String url4 = getBaseUrl(client); : - try (HttpSolrClient adminClient = getHttpSolrClient(url4)) { : - adminClient.setConnectionTimeout(15000); : - adminClient.setSoTimeout(30000); : : - Create createCmd = new Create(); : - createCmd.setCoreName("unloadcollection4"); : - createCmd.setCollection("unloadcollection"); : - String core4dataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection" + "_4n"; : - createCmd.setDataDir(getDataDir(core4dataDir)); : - adminClient.request(createCmd); : - } : + assertTrue(CollectionAdminRequest : + .addReplicaToShard("unloadcollection", "shard1") : + .setNode(jettys.get(3).getNodeName()) : + .process(cloudClient).isSuccess()); : + : waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); : - : + : // unload the leader again : leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1"); : try (HttpSolrClient collectionClient = getHttpSolrClient(leaderProps.getBaseUrl())) { : @@ -326,29 +286,22 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : fail("Leader never changed"); : } : } : - : + : zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); : - : - : + : // set this back : DirectUpdateHandler2.commitOnClose = true; : - : - // bring the downed leader back as replica : - try (HttpSolrClient adminClient = getHttpSolrClient(leaderProps.getBaseUrl())) { : - adminClient.setConnectionTimeout(15000); : - adminClient.setSoTimeout(30000); : + assertTrue(CollectionAdminRequest : + .addReplicaToShard("unloadcollection", "shard1") : + .setDataDir(core1DataDir) : + .setNode(leaderProps.getNodeName()) : + .process(cloudClient).isSuccess()); : : - Create createCmd = new Create(); : - createCmd.setCoreName(leaderProps.getCoreName()); : - createCmd.setCollection("unloadcollection"); : - createCmd.setDataDir(getDataDir(core1DataDir)); : - adminClient.request(createCmd); : - } : waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); : : long found1, found3; : - : - try (HttpSolrClient adminClient = getHttpSolrClient(url2 + "/unloadcollection")) { : + : + try (HttpSolrClient adminClient = getHttpSolrClient(jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2")) { : adminClient.setConnectionTimeout(15000); : adminClient.setSoTimeout(30000); : adminClient.commit(); : @@ -356,7 +309,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : q.set("distrib", false); : found1 = adminClient.query(q).getResults().getNumFound(); : } : - try (HttpSolrClient adminClient = getHttpSolrClient(url3 + "/unloadcollection")) { : + try (HttpSolrClient adminClient = getHttpSolrClient(jettys.get(2).getBaseUrl() + "/unloadcollection_shard1_replica3")) { : adminClient.setConnectionTimeout(15000); : adminClient.setSoTimeout(30000); : adminClient.commit(); : @@ -365,7 +318,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { : found3 = adminClient.query(q).getResults().getNumFound(); : } : : - try (HttpSolrClient adminClient = getHttpSolrClient(url4 + "/unloadcollection")) { : + try (HttpSolrClient adminClient = getHttpSolrClient(jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4")) { : adminClient.setConnectionTimeout(15000); : adminClient.setSoTimeout(30000); : adminClient.commit(); : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java : ---------------------------------------------------------------------- : diff --git a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java : index 1839881..162584c 100644 : --- a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java : +++ b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java : @@ -109,7 +109,7 @@ public class TestConfigReload extends AbstractFullDistribZkTestBase { : assertTrue(newStat.getVersion() > stat.getVersion()); : log.info("new_version "+ newStat.getVersion()); : Integer newVersion = newStat.getVersion(); : - long maxTimeoutSeconds = 20; : + long maxTimeoutSeconds = 60; : DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection("collection1"); : List<String> urls = new ArrayList<>(); : for (Slice slice : coll.getSlices()) { : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java : ---------------------------------------------------------------------- : diff --git a/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java b/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java : index 3c7a90e..2427f19 100644 : --- a/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java : +++ b/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java : @@ -62,7 +62,7 @@ public class TestCloudManagedSchema extends AbstractFullDistribZkTestBase { : NamedList namedListResponse = client.request(request); : client.setBaseURL(previousBaseURL); // Restore baseURL : NamedList status = (NamedList)namedListResponse.get("status"); : - NamedList collectionStatus = (NamedList)status.get("collection1"); : + NamedList collectionStatus = (NamedList)status.getVal(0); : String collectionSchema = (String)collectionStatus.get(CoreAdminParams.SCHEMA); : // Make sure the upgrade to managed schema happened : assertEquals("Schema resource name differs from expected name", "managed-schema", collectionSchema); : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java : ---------------------------------------------------------------------- : diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java : index ed5b622..740de34 100644 : --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java : +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java : @@ -1360,6 +1360,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse> : protected String collection; : protected String shard; : protected String node; : + protected String coreName; : protected String routeKey; : protected String instanceDir; : protected String dataDir; : @@ -1426,6 +1427,11 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse> : return this; : } : : + public AddReplica setCoreName(String coreName) { : + this.coreName = coreName; : + return this; : + } : + : @Override : public SolrParams getParams() { : ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); : @@ -1446,6 +1452,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse> : if (dataDir != null) { : params.add("dataDir", dataDir); : } : + if (coreName != null) { : + params.add("name", coreName); : + } : if (type != null) { : params.add(ZkStateReader.REPLICA_TYPE, type.name()); : } : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java : ---------------------------------------------------------------------- : diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java : index ed778a6..c46d346 100644 : --- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java : +++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java : @@ -1116,20 +1116,7 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 { : protected void setupJettySolrHome(File jettyHome) throws IOException { : seedSolrHome(jettyHome); : : - Properties coreProperties = new Properties(); : - coreProperties.setProperty("name", "collection1"); : - coreProperties.setProperty("shard", "${shard:}"); : - coreProperties.setProperty("collection", "${collection:collection1}"); : - coreProperties.setProperty("config", "${solrconfig:solrconfig.xml}"); : - coreProperties.setProperty("schema", "${schema:schema.xml}"); : - coreProperties.setProperty("coreNodeName", "${coreNodeName:}"); : - coreProperties.setProperty("replicaType", "${replicaType:}"); : - : - writeCoreProperties(jettyHome.toPath().resolve("cores").resolve("collection1"), coreProperties, "collection1"); : - : - // <core name="collection1" instanceDir="collection1" shard="${shard:}" : - // collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}" : - //coreNodeName="${coreNodeName:}"/> : + Files.createDirectories(jettyHome.toPath().resolve("cores").resolve("collection1")); : } : : } : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java : ---------------------------------------------------------------------- : diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java : index 7141eed..b7c4162 100644 : --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java : +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java : @@ -25,6 +25,7 @@ import org.apache.commons.io.FileUtils; : import org.apache.solr.BaseDistributedSearchTestCase; : import org.apache.solr.client.solrj.embedded.JettySolrRunner; : import org.apache.solr.client.solrj.impl.CloudSolrClient; : +import org.apache.solr.client.solrj.request.CollectionAdminRequest; : import org.apache.solr.common.cloud.ClusterState; : import org.apache.solr.common.cloud.DocCollection; : import org.apache.solr.common.cloud.Replica; : @@ -98,20 +99,15 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes : File controlHome = new File(new File(getSolrHome()).getParentFile(), "control" + homeCount.incrementAndGet()); : FileUtils.copyDirectory(new File(getSolrHome()), controlHome); : setupJettySolrHome(controlHome); : - : - System.setProperty("collection", "control_collection"); : - String numShardsS = System.getProperty(ZkStateReader.NUM_SHARDS_PROP); : - System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "1"); : - controlJetty = createJetty(controlHome, null); // let the shardId default to shard1 : - System.clearProperty("collection"); : - if(numShardsS != null) { : - System.setProperty(ZkStateReader.NUM_SHARDS_PROP, numShardsS); : - } else { : - System.clearProperty(ZkStateReader.NUM_SHARDS_PROP); : - } : : + controlJetty = createJetty(controlHome, null); // let the shardId default to shard1 : controlClient = createNewSolrClient(controlJetty.getLocalPort()); : : + assertTrue(CollectionAdminRequest : + .createCollection("control_collection", 1, 1) : + .setCreateNodeSet(controlJetty.getNodeName()) : + .process(controlClient).isSuccess()); : + : StringBuilder sb = new StringBuilder(); : for (int i = 1; i <= numShards; i++) { : if (sb.length() > 0) sb.append(','); : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java : ---------------------------------------------------------------------- : diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java : index cf6f2e1..f3d9d8a 100644 : --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java : +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java : @@ -62,7 +62,6 @@ import org.apache.solr.common.cloud.ClusterState; : import org.apache.solr.common.cloud.DocCollection; : import org.apache.solr.common.cloud.Replica; : import org.apache.solr.common.cloud.Slice; : -import org.apache.solr.common.cloud.SolrZkClient; : import org.apache.solr.common.cloud.ZkCoreNodeProps; : import org.apache.solr.common.cloud.ZkNodeProps; : import org.apache.solr.common.cloud.ZkStateReader; : @@ -234,9 +233,15 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes : System.clearProperty("urlScheme"); : try (ZkStateReader zkStateReader = new ZkStateReader(zkServer.getZkAddress(), : AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT)) { : - zkStateReader.getZkClient().create(ZkStateReader.CLUSTER_PROPS, : - Utils.toJSON(Collections.singletonMap("urlScheme", "https")), : - CreateMode.PERSISTENT, true); : + try { : + zkStateReader.getZkClient().create(ZkStateReader.CLUSTER_PROPS, : + Utils.toJSON(Collections.singletonMap("urlScheme", "https")), : + CreateMode.PERSISTENT, true); : + } catch (KeeperException.NodeExistsException e) { : + ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData(ZkStateReader.CLUSTER_PROPS, : + null, null, true)); : + zkStateReader.getZkClient().setData(ZkStateReader.CLUSTER_PROPS, Utils.toJSON(props.plus("urlScheme", "https")), true); : + } : } : } : if (useTlogReplicas()) { : @@ -300,48 +305,31 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes : @Override : protected void createServers(int numServers) throws Exception { : : - System.setProperty("collection", "control_collection"); : - : - // we want hashes by default for the control, so set to 1 shard as opposed to leaving unset : - String oldNumShards = System.getProperty(ZkStateReader.NUM_SHARDS_PROP); : - System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "1"); : - : - try { : - : - File controlJettyDir = createTempDir("control").toFile(); : - setupJettySolrHome(controlJettyDir); : - : - controlJetty = createJetty(controlJettyDir, useJettyDataDir ? getDataDir(testDir : - + "/control/data") : null); // don't pass shard name... let it default to : - // "shard1" : - : - : - controlClient = createNewSolrClient(controlJetty.getLocalPort()); : - : - if (sliceCount <= 0) { : - // for now, just create the cloud client for the control if we don't : - // create the normal cloud client. : - // this can change if more tests need it. : - controlClientCloud = createCloudClient("control_collection"); : - controlClientCloud.connect(); : - waitForCollection(controlClientCloud.getZkStateReader(), : - "control_collection", 0); : - // NOTE: we are skipping creation of the chaos monkey by returning here : - cloudClient = controlClientCloud; // temporary - some code needs/uses : - // cloudClient : - return; : - } : - : - } finally { : - System.clearProperty("collection"); : - if (oldNumShards != null) { : - System.setProperty(ZkStateReader.NUM_SHARDS_PROP, oldNumShards); : - } else { : - System.clearProperty(ZkStateReader.NUM_SHARDS_PROP); : + File controlJettyDir = createTempDir("control").toFile(); : + setupJettySolrHome(controlJettyDir); : + controlJetty = createJetty(controlJettyDir, useJettyDataDir ? getDataDir(testDir : + + "/control/data") : null); : + try (SolrClient client = createCloudClient("control_collection")) { : + assertEquals(0, CollectionAdminRequest : + .createCollection("control_collection", 1, 1) : + .setCreateNodeSet(controlJetty.getNodeName()) : + .process(client).getStatus()); : } : + controlClient = new HttpSolrClient.Builder(controlJetty.getBaseUrl() + "/control_collection").build(); : + if (sliceCount <= 0) { : + // for now, just create the cloud client for the control if we don't : + // create the normal cloud client. : + // this can change if more tests need it. : + controlClientCloud = createCloudClient("control_collection"); : + controlClientCloud.connect(); : + waitForCollection(controlClientCloud.getZkStateReader(), : + "control_collection", 0); : + // NOTE: we are skipping creation of the chaos monkey by returning here : + cloudClient = controlClientCloud; // temporary - some code needs/uses : + // cloudClient : + return; : } : : - : initCloud(); : : createJettys(numServers); : @@ -390,24 +378,13 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes : List<SolrClient> clients = new ArrayList<>(); : StringBuilder sb = new StringBuilder(); : : - if ("2".equals(getStateFormat())) { : - log.info("Creating " + DEFAULT_COLLECTION + " with stateFormat=2"); : - SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), : - AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT); : - Overseer.getStateUpdateQueue(zkClient).offer( : - Utils.toJSON(Utils.makeMap(Overseer.QUEUE_OPERATION, : - CollectionParams.CollectionAction.CREATE.toLower(), : - "name", DEFAULT_COLLECTION, : - "numShards", String.valueOf(sliceCount), : - DocCollection.STATE_FORMAT, getStateFormat(), : - ZkStateReader.NRT_REPLICAS, useTlogReplicas()?"0":"1", : - ZkStateReader.TLOG_REPLICAS, useTlogReplicas()?"1":"0", : - ZkStateReader.PULL_REPLICAS, String.valueOf(getPullReplicaCount())))); : - zkClient.close(); : - } : + assertEquals(0, CollectionAdminRequest : + .createCollection(DEFAULT_COLLECTION, sliceCount, 1) : + .setStateFormat(Integer.parseInt(getStateFormat())) : + .setCreateNodeSet("") : + .process(cloudClient).getStatus()); : : - int numPullReplicas = getPullReplicaCount() * sliceCount; : - : + int numOtherReplicas = numJettys - getPullReplicaCount() * sliceCount; : for (int i = 1; i <= numJettys; i++) { : if (sb.length() > 0) sb.append(','); : int cnt = this.jettyIntCntr.incrementAndGet(); : @@ -417,20 +394,37 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes : jettyDir.mkdirs(); : setupJettySolrHome(jettyDir); : JettySolrRunner j; : - : - if (numPullReplicas > 0) { : - numPullReplicas--; : + : + if (numOtherReplicas > 0) { : + numOtherReplicas--; : + if (useTlogReplicas()) { : + log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.TLOG); : + j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" : + + cnt) : null, null, "solrconfig.xml", null, Replica.Type.TLOG); : + assertTrue(CollectionAdminRequest : + .addReplicaToShard(DEFAULT_COLLECTION, "shard"+((i%sliceCount)+1)) : + .setNode(j.getNodeName()) : + .setType(Replica.Type.TLOG) : + .process(cloudClient).isSuccess()); : + } else { : + log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.NRT); : + j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" : + + cnt) : null, null, "solrconfig.xml", null, null); : + assertTrue(CollectionAdminRequest : + .addReplicaToShard(DEFAULT_COLLECTION, "shard"+((i%sliceCount)+1)) : + .setNode(j.getNodeName()) : + .setType(Replica.Type.NRT) : + .process(cloudClient).isSuccess()); : + } : + } else { : log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.PULL); : j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" : + cnt) : null, null, "solrconfig.xml", null, Replica.Type.PULL); : - } else if (useTlogReplicas()) { : - log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.TLOG); : - j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" : - + cnt) : null, null, "solrconfig.xml", null, Replica.Type.TLOG); : - } else { : - log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.NRT); : - j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" : - + cnt) : null, null, "solrconfig.xml", null, null); : + assertTrue(CollectionAdminRequest : + .addReplicaToShard(DEFAULT_COLLECTION, "shard"+((i%sliceCount)+1)) : + .setNode(j.getNodeName()) : + .setType(Replica.Type.PULL) : + .process(cloudClient).isSuccess()); : } : jettys.add(j); : SolrClient client = createNewSolrClient(j.getLocalPort()); : : http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e9d685a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractZkTestCase.java : ---------------------------------------------------------------------- : diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractZkTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractZkTestCase.java : index 4c63bfb..7461c4c 100644 : --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractZkTestCase.java : +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractZkTestCase.java : @@ -19,6 +19,7 @@ package org.apache.solr.cloud; : import org.apache.solr.SolrTestCaseJ4; : import org.apache.solr.common.cloud.SolrZkClient; : import org.apache.solr.common.cloud.ZkNodeProps; : +import org.apache.solr.common.cloud.ZkStateReader; : import org.apache.solr.common.util.Utils; : import org.apache.zookeeper.CreateMode; : import org.junit.AfterClass; : @@ -28,6 +29,7 @@ import org.slf4j.LoggerFactory; : : import java.io.File; : import java.lang.invoke.MethodHandles; : +import java.nio.charset.StandardCharsets; : import java.util.HashMap; : import java.util.Map; : : @@ -97,7 +99,9 @@ public abstract class AbstractZkTestCase extends SolrTestCaseJ4 { : zkClient.makePath("/collections/collection1/shards", CreateMode.PERSISTENT, true); : zkClient.makePath("/collections/control_collection", Utils.toJSON(zkProps), CreateMode.PERSISTENT, true); : zkClient.makePath("/collections/control_collection/shards", CreateMode.PERSISTENT, true); : - : + // this workaround is acceptable until we remove legacyCloud because we just init a single core here : + String defaultClusterProps = "{\""+ZkStateReader.LEGACY_CLOUD+"\":\"true\"}"; : + zkClient.makePath(ZkStateReader.CLUSTER_PROPS, defaultClusterProps.getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, true); : // for now, always upload the config and schema to the canonical names : putConfig("conf1", zkClient, solrhome, config, "solrconfig.xml"); : putConfig("conf1", zkClient, solrhome, schema, "schema.xml"); : : -Hoss http://www.lucidworks.com/ --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org
