Author: yonik
Date: Tue Jan 19 16:52:19 2010
New Revision: 900841
URL: http://svn.apache.org/viewvc?rev=900841&view=rev
Log:
draft - distrib search uses CloudState
Modified:
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java
Modified:
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
URL:
http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
---
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
(original)
+++
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
Tue Jan 19 16:52:19 2010
@@ -39,6 +39,10 @@
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.*;
import org.apache.solr.util.SolrPluginUtils;
+import org.apache.solr.cloud.CloudState;
+import org.apache.solr.cloud.Slice;
+import org.apache.solr.cloud.ZkNodeProps;
+
import java.io.IOException;
import java.net.URL;
@@ -104,14 +108,78 @@
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
- // TODO: temporary... this should go in a different component.
+ // TODO: temporary... this should go in a different component, or in
SearchHandler
+ rb.isDistrib = params.getBool("distrib",false);
String shards = params.get(ShardParams.SHARDS);
- // nocommit : get shards based on ZooKeeper info, using load balancing
- // shards =
req.getCore().getCoreDescriptor().getCoreContainer().getZooKeeperController().getSearchNodes();
- if (shards != null) {
- List<String> lst = StrUtils.splitSmart(shards, ",", true);
- rb.shards = lst.toArray(new String[lst.size()]);
+
+ // for back compat, a shards param with URLs like localhost:8983/solr will
mean that this
+ // search is distributed.
+ boolean hasShardURL = shards != null && shards.charAt('/') > 0;
+ rb.isDistrib = hasShardURL | rb.isDistrib;
+
+ if (rb.isDistrib) {
+ CloudState cloudState = null;
+
+ if (shards != null) {
+ List<String> lst = StrUtils.splitSmart(shards, ",", true);
+ rb.shards = lst.toArray(new String[lst.size()]);
+ rb.slices = new String[rb.shards.length];
+ for (int i=0; i<rb.shards.length; i++) {
+ if (rb.shards[i].indexOf('/') < 0) {
+ // this is a logical shard
+ rb.slices[i] = rb.shards[i];
+ rb.shards[i] = null;
+ }
+ }
+ } else {
+ // we weren't provided with a list of slices to query, so find the
list that will cover the complete index
+
+ cloudState =
req.getCore().getCoreDescriptor().getCoreContainer().getZooKeeperController().getCloudInfo();
+
+ // TODO: EXAMPLE
+ rb.slices = new String[]{"shard1","shard2"};
+ rb.shards = new String[rb.shards.length];
+ }
+
+ //
+ // Map slices to shards
+ //
+ for (int i=0; i<rb.shards.length; i++) {
+ if (rb.shards[i] == null) {
+ if (cloudState == null) {
+ cloudState =
req.getCore().getCoreDescriptor().getCoreContainer().getZooKeeperController().getCloudInfo();
+ }
+ String sliceStr = rb.slices[i];
+ Slice slice = cloudState.getSlice(sliceStr);
+
+ if (slice==null) {
+ // TODO: we could treat this as "all servers down" for a slice if
partial results are enabled.
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no
such slice: " + sliceStr);
+ }
+
+ Map<String, ZkNodeProps> sliceShards = slice.getShards();
+
+ // For now, recreate the | delimited list of equivalent servers
+ StringBuilder sliceShardsStr = new StringBuilder();
+ boolean first = true;
+ for (ZkNodeProps nodeProps : sliceShards.values()) {
+ if (first) {
+ first = false;
+ } else {
+ sliceShardsStr.append('|');
+ }
+ String url = nodeProps.get("url");
+ if (url.startsWith("http://";))
+ url = url.substring(7);
+ sliceShardsStr.append(url);
+ }
+
+ rb.shards[i] = sliceShardsStr.toString();
+ }
+ }
}
+
+
String shards_rows = params.get(ShardParams.SHARDS_ROWS);
if(shards_rows != null) {
rb.shards_rows = Integer.parseInt(shards_rows);
Modified:
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
URL:
http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
---
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
(original)
+++
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
Tue Jan 19 16:52:19 2010
@@ -94,7 +94,9 @@
public int stage; // What stage is this current request at?
//The address of the Shard
+ boolean isDistrib; // is this a distributed search?
public String[] shards;
+ public String[] slices; // the optional logical ids of the shards
public int shards_rows = -1;
public int shards_start = -1;
public List<ShardRequest> outgoing; // requests to be sent
Modified:
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
URL:
http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
---
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
(original)
+++
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
Tue Jan 19 16:52:19 2010
@@ -189,7 +189,7 @@
subt.stop();
}
- if (rb.shards == null) {
+ if (!rb.isDistrib) {
// a normal non-distributed request
// The semantics of debugging vs not debugging are different enough that
@@ -466,6 +466,13 @@
// no need to set the response parser as binary is the default
// req.setResponseParser(new BinaryResponseParser());
+ // if there are no shards available for a slice, urls.size()==0
+ if (urls.size()==0) {
+ // TODO: what's the right error code here? We should use the same
thing when
+ // all of the servers for a shard are down.
+ throw new
SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "no servers hosting
shard");
+ }
+
if (urls.size() <= 1) {
String url = urls.get(0);
srsp.setShardAddress(url);
Modified:
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java
URL:
http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
---
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java
(original)
+++
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java
Tue Jan 19 16:52:19 2010
@@ -41,8 +41,7 @@
public int purpose; // the purpose of this request
public String[] shards; // the shards this request should be sent to, null
for all
-// TODO: how to request a specific shard address?
-
+ public String[] slices; // the slice of the index requested for each
public ModifiableSolrParams params;
