http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java deleted file mode 100644 index bff6c6a..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.beans; - -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrInputDocument; - -import java.lang.reflect.*; -import java.util.*; -import java.util.regex.Pattern; -import java.util.concurrent.ConcurrentHashMap; -import java.nio.ByteBuffer; - -/** - * A class to map objects to and from solr documents. - * - * - * @since solr 1.3 - */ -public class DocumentObjectBinder { - - private final Map<Class, List<DocField>> infocache = new ConcurrentHashMap<>(); - - public DocumentObjectBinder() { - } - - public <T> List<T> getBeans(Class<T> clazz, SolrDocumentList solrDocList) { - List<DocField> fields = getDocFields(clazz); - List<T> result = new ArrayList<>(solrDocList.size()); - - for (SolrDocument sdoc : solrDocList) { - result.add(getBean(clazz, fields, sdoc)); - } - return result; - } - - public <T> T getBean(Class<T> clazz, SolrDocument solrDoc) { - return getBean(clazz, null, solrDoc); - } - - private <T> T getBean(Class<T> clazz, List<DocField> fields, SolrDocument solrDoc) { - if (fields == null) { - fields = getDocFields(clazz); - } - - try { - T obj = clazz.newInstance(); - for (DocField docField : fields) { - docField.inject(obj, solrDoc); - } - return obj; - } catch (Exception e) { - throw new BindingException("Could not instantiate object of " + clazz, e); - } - } - - public SolrInputDocument toSolrInputDocument(Object obj) { - List<DocField> fields = getDocFields(obj.getClass()); - if (fields.isEmpty()) { - throw new BindingException("class: " + obj.getClass() + " does not define any fields."); - } - - SolrInputDocument doc = new SolrInputDocument(); - for (DocField field : fields) { - if (field.dynamicFieldNamePatternMatcher != null && - field.get(obj) != null && - field.isContainedInMap) { - Map<String, Object> mapValue = (Map<String, Object>) field.get(obj); - - for (Map.Entry<String, Object> e : mapValue.entrySet()) { - doc.setField(e.getKey(), e.getValue(), 1.0f); - } - } else { - if (field.child != null) { - addChild(obj, field, doc); - } else { - doc.setField(field.name, field.get(obj), 1.0f); - } - } - } - return doc; - } - - private void addChild(Object obj, DocField field, SolrInputDocument doc) { - Object val = field.get(obj); - if (val == null) return; - if (val instanceof Collection) { - Collection collection = (Collection) val; - for (Object o : collection) { - SolrInputDocument child = toSolrInputDocument(o); - doc.addChildDocument(child); - } - } else if (val.getClass().isArray()) { - Object[] objs = (Object[]) val; - for (Object o : objs) doc.addChildDocument(toSolrInputDocument(o)); - } else { - doc.addChildDocument(toSolrInputDocument(val)); - } - } - - private List<DocField> getDocFields(Class clazz) { - List<DocField> fields = infocache.get(clazz); - if (fields == null) { - synchronized(infocache) { - infocache.put(clazz, fields = collectInfo(clazz)); - } - } - return fields; - } - - private List<DocField> collectInfo(Class clazz) { - List<DocField> fields = new ArrayList<>(); - Class superClazz = clazz; - List<AccessibleObject> members = new ArrayList<>(); - - while (superClazz != null && superClazz != Object.class) { - members.addAll(Arrays.asList(superClazz.getDeclaredFields())); - members.addAll(Arrays.asList(superClazz.getDeclaredMethods())); - superClazz = superClazz.getSuperclass(); - } - boolean childFieldFound = false; - for (AccessibleObject member : members) { - if (member.isAnnotationPresent(Field.class)) { - member.setAccessible(true); - DocField df = new DocField(member); - if (df.child != null) { - if (childFieldFound) - throw new BindingException(clazz.getName() + " cannot have more than one Field with child=true"); - childFieldFound = true; - } - fields.add(df); - } - } - return fields; - } - - private class DocField { - private Field annotation; - private String name; - private java.lang.reflect.Field field; - private Method setter; - private Method getter; - private Class type; - private boolean isArray; - private boolean isList; - private List<DocField> child; - - /* - * dynamic fields may use a Map based data structure to bind a given field. - * if a mapping is done using, "Map<String, List<String>> foo", <code>isContainedInMap</code> - * is set to <code>TRUE</code> as well as <code>isList</code> is set to <code>TRUE</code> - */ - private boolean isContainedInMap; - private Pattern dynamicFieldNamePatternMatcher; - - public DocField(AccessibleObject member) { - if (member instanceof java.lang.reflect.Field) { - field = (java.lang.reflect.Field) member; - } else { - setter = (Method) member; - } - annotation = member.getAnnotation(Field.class); - storeName(annotation); - storeType(); - - // Look for a matching getter - if (setter != null) { - String gname = setter.getName(); - if (gname.startsWith("set")) { - gname = "get" + gname.substring(3); - try { - getter = setter.getDeclaringClass().getMethod(gname, (Class[]) null); - } catch (Exception ex) { - // no getter -- don't worry about it... - if (type == Boolean.class) { - gname = "is" + setter.getName().substring(3); - try { - getter = setter.getDeclaringClass().getMethod(gname, (Class[]) null); - } catch(Exception ex2) { - // no getter -- don't worry about it... - } - } - } - } - } - } - - private void storeName(Field annotation) { - if (annotation.value().equals(DEFAULT)) { - if (field != null) { - name = field.getName(); - } else { - String setterName = setter.getName(); - if (setterName.startsWith("set") && setterName.length() > 3) { - name = setterName.substring(3, 4).toLowerCase(Locale.ROOT) + setterName.substring(4); - } else { - name = setter.getName(); - } - } - } else if (annotation.value().indexOf('*') >= 0) { //dynamic fields are annotated as @Field("categories_*") - //if the field was annotated as a dynamic field, convert the name into a pattern - //the wildcard (*) is supposed to be either a prefix or a suffix, hence the use of replaceFirst - name = annotation.value().replaceFirst("\\*", "\\.*"); - dynamicFieldNamePatternMatcher = Pattern.compile("^"+name+"$"); - } else { - name = annotation.value(); - } - } - - private void storeType() { - if (field != null) { - type = field.getType(); - } else { - Class[] params = setter.getParameterTypes(); - if (params.length != 1) { - throw new BindingException("Invalid setter method. Must have one and only one parameter"); - } - type = params[0]; - } - - if (type == Collection.class || type == List.class || type == ArrayList.class) { - isList = true; - if (annotation.child()) { - populateChild(field.getGenericType()); - } else { - type = Object.class; - } - } else if (type == byte[].class) { - //no op - } else if (type.isArray()) { - isArray = true; - if (annotation.child()) { - populateChild(type.getComponentType()); - } else { - type = type.getComponentType(); - } - } else if (type == Map.class || type == HashMap.class) { //corresponding to the support for dynamicFields - if (annotation.child()) throw new BindingException("Map should is not a valid type for a child document"); - isContainedInMap = true; - //assigned a default type - type = Object.class; - if (field != null) { - if (field.getGenericType() instanceof ParameterizedType) { - //check what are the generic values - ParameterizedType parameterizedType = (ParameterizedType) field.getGenericType(); - Type[] types = parameterizedType.getActualTypeArguments(); - if (types != null && types.length == 2 && types[0] == String.class) { - //the key should always be String - //Raw and primitive types - if (types[1] instanceof Class) { - //the value could be multivalued then it is a List, Collection, ArrayList - if (types[1] == Collection.class || types[1] == List.class || types[1] == ArrayList.class) { - type = Object.class; - isList = true; - } else { - //else assume it is a primitive and put in the source type itself - type = (Class) types[1]; - } - } else if (types[1] instanceof ParameterizedType) { //Of all the Parameterized types, only List is supported - Type rawType = ((ParameterizedType) types[1]).getRawType(); - if (rawType == Collection.class || rawType == List.class || rawType == ArrayList.class) { - type = Object.class; - isList = true; - } - } else if (types[1] instanceof GenericArrayType) { //Array types - type = (Class) ((GenericArrayType) types[1]).getGenericComponentType(); - isArray = true; - } else { //Throw an Exception if types are not known - throw new BindingException("Allowed type for values of mapping a dynamicField are : " + - "Object, Object[] and List"); - } - } - } - } - } else { - if (annotation.child()) { - populateChild(type); - } - } - } - - private void populateChild(Type typ) { - if (typ == null) { - throw new RuntimeException("no type information available for" + (field == null ? setter : field)); - } - if (typ.getClass() == Class.class) {//of type class - type = (Class) typ; - } else if (typ instanceof ParameterizedType) { - type = (Class) ((ParameterizedType) typ).getActualTypeArguments()[0]; - } else { - throw new BindingException("Invalid type information available for" + (field == null ? setter : field)); - - } - child = getDocFields(type); - } - - /** - * Called by the {@link #inject} method to read the value(s) for a field - * This method supports reading of all "matching" fieldName's in the <code>SolrDocument</code> - * - * Returns <code>SolrDocument.getFieldValue</code> for regular fields, - * and <code>Map<String, List<Object>></code> for a dynamic field. The key is all matching fieldName's. - */ - @SuppressWarnings("unchecked") - private Object getFieldValue(SolrDocument solrDocument) { - if (child != null) { - List<SolrDocument> children = solrDocument.getChildDocuments(); - if (children == null || children.isEmpty()) return null; - if (isList) { - ArrayList list = new ArrayList(children.size()); - for (SolrDocument c : children) { - list.add(getBean(type, child, c)); - } - return list; - } else if (isArray) { - Object[] arr = (Object[]) Array.newInstance(type, children.size()); - for (int i = 0; i < children.size(); i++) { - arr[i] = getBean(type, child, children.get(i)); - } - return arr; - - } else { - return getBean(type, child, children.get(0)); - } - } - Object fieldValue = solrDocument.getFieldValue(name); - if (fieldValue != null) { - //this is not a dynamic field. so return the value - return fieldValue; - } - - if (dynamicFieldNamePatternMatcher == null) { - return null; - } - - //reading dynamic field values - Map<String, Object> allValuesMap = null; - List allValuesList = null; - if (isContainedInMap) { - allValuesMap = new HashMap<>(); - } else { - allValuesList = new ArrayList(); - } - - for (String field : solrDocument.getFieldNames()) { - if (dynamicFieldNamePatternMatcher.matcher(field).find()) { - Object val = solrDocument.getFieldValue(field); - if (val == null) { - continue; - } - - if (isContainedInMap) { - if (isList) { - if (!(val instanceof List)) { - List al = new ArrayList(); - al.add(val); - val = al; - } - } else if (isArray) { - if (!(val instanceof List)) { - Object[] arr = (Object[]) Array.newInstance(type, 1); - arr[0] = val; - val = arr; - } else { - val = Array.newInstance(type, ((List) val).size()); - } - } - allValuesMap.put(field, val); - } else { - if (val instanceof Collection) { - allValuesList.addAll((Collection) val); - } else { - allValuesList.add(val); - } - } - } - } - if (isContainedInMap) { - return allValuesMap.isEmpty() ? null : allValuesMap; - } else { - return allValuesList.isEmpty() ? null : allValuesList; - } - } - - <T> void inject(T obj, SolrDocument sdoc) { - Object val = getFieldValue(sdoc); - if(val == null) { - return; - } - - if (isArray && !isContainedInMap) { - List list; - if (val.getClass().isArray()) { - set(obj, val); - return; - } else if (val instanceof List) { - list = (List) val; - } else { - list = new ArrayList(); - list.add(val); - } - set(obj, list.toArray((Object[]) Array.newInstance(type, list.size()))); - } else if (isList && !isContainedInMap) { - if (!(val instanceof List)) { - List list = new ArrayList(); - list.add(val); - val = list; - } - set(obj, val); - } else if (isContainedInMap) { - if (val instanceof Map) { - set(obj, val); - } - } else { - set(obj, val); - } - - } - - private void set(Object obj, Object v) { - if (v != null && type == ByteBuffer.class && v.getClass() == byte[].class) { - v = ByteBuffer.wrap((byte[]) v); - } - try { - if (field != null) { - field.set(obj, v); - } else if (setter != null) { - setter.invoke(obj, v); - } - } - catch (Exception e) { - throw new BindingException("Exception while setting value : " + v + " on " + (field != null ? field : setter), e); - } - } - - public Object get(final Object obj) { - if (field != null) { - try { - return field.get(obj); - } catch (Exception e) { - throw new BindingException("Exception while getting value: " + field, e); - } - } else if (getter == null) { - throw new BindingException("Missing getter for field: " + name + " -- You can only call the 'get' for fields that have a field of 'get' method"); - } - - try { - return getter.invoke(obj, (Object[]) null); - } catch (Exception e) { - throw new BindingException("Exception while getting value: " + getter, e); - } - } - } - public static final String DEFAULT = "#default"; -}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java deleted file mode 100644 index 39f6752..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.beans; - -import static java.lang.annotation.ElementType.*; -import static java.lang.annotation.RetentionPolicy.RUNTIME; -import static org.apache.solr.client.solrj.beans.DocumentObjectBinder.DEFAULT; - -import java.lang.annotation.Target; -import java.lang.annotation.Retention; - - -/** - * This class can be used to annotate a field or a setter an any class - * and SlrJ would help you convert to SolrInputDocument and from SolrDocument - * - * @since solr 1.3 - */ -@Target({FIELD, METHOD}) -@Retention(RUNTIME) -public @interface Field { - boolean child() default false; - String value() default DEFAULT; -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java deleted file mode 100644 index 890005f..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Annotation based mapping of client objects to Solr documents. - */ -package org.apache.solr.client.solrj.beans; - - http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java deleted file mode 100644 index 629c537..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.impl; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.common.util.ContentStream; - -import java.io.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * A RequestWriter which writes requests in the javabin format - * - * - * @see org.apache.solr.client.solrj.request.RequestWriter - * @since solr 1.4 - */ -public class BinaryRequestWriter extends RequestWriter { - - @Override - public Collection<ContentStream> getContentStreams(SolrRequest req) throws IOException { - if (req instanceof UpdateRequest) { - UpdateRequest updateRequest = (UpdateRequest) req; - if (isNull(updateRequest.getDocuments()) && - isNull(updateRequest.getDeleteByIdMap()) && - isNull(updateRequest.getDeleteQuery()) - && (updateRequest.getDocIterator() == null) ) { - return null; - } - List<ContentStream> l = new ArrayList<>(); - l.add(new LazyContentStream(updateRequest)); - return l; - } else { - return super.getContentStreams(req); - } - - } - - - @Override - public String getUpdateContentType() { - return "application/javabin"; - } - - @Override - public ContentStream getContentStream(final UpdateRequest request) throws IOException { - final BAOS baos = new BAOS(); - new JavaBinUpdateRequestCodec().marshal(request, baos); - - return new ContentStream() { - @Override - public String getName() { - return null; - } - - @Override - public String getSourceInfo() { - return "javabin"; - } - - @Override - public String getContentType() { - return "application/javabin"; - } - - @Override - public Long getSize() // size if we know it, otherwise null - { - return Long.valueOf(baos.size()); - } - - @Override - public InputStream getStream() { - return new ByteArrayInputStream(baos.getbuf(), 0, baos.size()); - } - - @Override - public Reader getReader() { - throw new RuntimeException("No reader available . this is a binarystream"); - } - }; - } - - - @Override - public void write(SolrRequest request, OutputStream os) throws IOException { - if (request instanceof UpdateRequest) { - UpdateRequest updateRequest = (UpdateRequest) request; - new JavaBinUpdateRequestCodec().marshal(updateRequest, os); - } - } - - /* - * A hack to get access to the protected internal buffer and avoid an additional copy - */ - class BAOS extends ByteArrayOutputStream { - byte[] getbuf() { - return super.buf; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java deleted file mode 100644 index 4f3caf1..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.impl; - -import org.apache.solr.client.solrj.ResponseParser; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.JavaBinCodec; - -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; - -/** - * - * @since solr 1.3 - */ -public class BinaryResponseParser extends ResponseParser { - public static final String BINARY_CONTENT_TYPE = "application/octet-stream"; - - @Override - public String getWriterType() { - return "javabin"; - } - - @Override - public NamedList<Object> processResponse(InputStream body, String encoding) { - try { - return (NamedList<Object>) new JavaBinCodec().unmarshal(body); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e); - - } - } - - @Override - public String getContentType() { - return BINARY_CONTENT_TYPE; - } - - @Override - public String getVersion() { - return "2"; - } - - @Override - public NamedList<Object> processResponse(Reader reader) { - throw new RuntimeException("Cannot handle character stream"); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java deleted file mode 100644 index 8508268..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ /dev/null @@ -1,1232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.impl; - -import org.apache.http.NoHttpResponseException; -import org.apache.http.client.HttpClient; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.solr.client.solrj.ResponseParser; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.AbstractUpdateRequest; -import org.apache.solr.client.solrj.request.IsUpdateRequest; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.DocRouter; -import org.apache.solr.common.cloud.ImplicitDocRouter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkCoreNodeProps; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.ShardParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.util.Hash; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SolrjNamedThreadFactory; -import org.apache.solr.common.util.StrUtils; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.ConnectException; -import java.net.SocketException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -/** - * SolrJ client class to communicate with SolrCloud. - * Instances of this class communicate with Zookeeper to discover - * Solr endpoints for SolrCloud collections, and then use the - * {@link LBHttpSolrClient} to issue requests. - * - * This class assumes the id field for your documents is called - * 'id' - if this is not the case, you must set the right name - * with {@link #setIdField(String)}. - */ -@SuppressWarnings("serial") -public class CloudSolrClient extends SolrClient { - protected static final Logger log = LoggerFactory.getLogger(CloudSolrClient.class); - - private volatile ZkStateReader zkStateReader; - private String zkHost; // the zk server connect string - private int zkConnectTimeout = 10000; - private int zkClientTimeout = 10000; - private volatile String defaultCollection; - private final LBHttpSolrClient lbClient; - private final boolean shutdownLBHttpSolrServer; - private HttpClient myClient; - private final boolean clientIsInternal; - //no of times collection state to be reloaded if stale state error is received - private static final int MAX_STALE_RETRIES = 5; - Random rand = new Random(); - - private final boolean updatesToLeaders; - private boolean parallelUpdates = true; - private ExecutorService threadPool = Executors - .newCachedThreadPool(new SolrjNamedThreadFactory( - "CloudSolrServer ThreadPool")); - private String idField = "id"; - public static final String STATE_VERSION = "_stateVer_"; - private final Set<String> NON_ROUTABLE_PARAMS; - { - NON_ROUTABLE_PARAMS = new HashSet<>(); - NON_ROUTABLE_PARAMS.add(UpdateParams.EXPUNGE_DELETES); - NON_ROUTABLE_PARAMS.add(UpdateParams.MAX_OPTIMIZE_SEGMENTS); - NON_ROUTABLE_PARAMS.add(UpdateParams.COMMIT); - NON_ROUTABLE_PARAMS.add(UpdateParams.WAIT_SEARCHER); - NON_ROUTABLE_PARAMS.add(UpdateParams.OPEN_SEARCHER); - - NON_ROUTABLE_PARAMS.add(UpdateParams.SOFT_COMMIT); - NON_ROUTABLE_PARAMS.add(UpdateParams.PREPARE_COMMIT); - NON_ROUTABLE_PARAMS.add(UpdateParams.OPTIMIZE); - - // Not supported via SolrCloud - // NON_ROUTABLE_PARAMS.add(UpdateParams.ROLLBACK); - - } - private volatile long timeToLive = 60* 1000L; - private volatile List<Object> locks = objectList(3); - - - protected final Map<String, ExpiringCachedDocCollection> collectionStateCache = new ConcurrentHashMap<String, ExpiringCachedDocCollection>(){ - @Override - public ExpiringCachedDocCollection get(Object key) { - ExpiringCachedDocCollection val = super.get(key); - if(val == null) return null; - if(val.isExpired(timeToLive)) { - super.remove(key); - return null; - } - return val; - } - - }; - - class ExpiringCachedDocCollection { - final DocCollection cached; - long cachedAt; - - ExpiringCachedDocCollection(DocCollection cached) { - this.cached = cached; - this.cachedAt = System.currentTimeMillis(); - } - - boolean isExpired(long timeToLive) { - return (System.currentTimeMillis() - cachedAt) > timeToLive; - } - } - - /** - * Create a new client object that connects to Zookeeper and is always aware - * of the SolrCloud state. If there is a fully redundant Zookeeper quorum and - * SolrCloud has enough replicas for every shard in a collection, there is no - * single point of failure. Updates will be sent to shard leaders by default. - * - * @param zkHost - * The client endpoint of the zookeeper quorum containing the cloud - * state. The full specification for this string is one or more comma - * separated HOST:PORT values, followed by an optional chroot value - * that starts with a forward slash. Using a chroot allows multiple - * applications to coexist in one ensemble. For full details, see the - * Zookeeper documentation. Some examples: - * <p> - * "host1:2181" - * <p> - * "host1:2181,host2:2181,host3:2181/mysolrchroot" - * <p> - * "zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181" - */ - public CloudSolrClient(String zkHost) { - this.zkHost = zkHost; - this.clientIsInternal = true; - this.myClient = HttpClientUtil.createClient(null); - this.lbClient = new LBHttpSolrClient(myClient); - this.lbClient.setRequestWriter(new BinaryRequestWriter()); - this.lbClient.setParser(new BinaryResponseParser()); - this.updatesToLeaders = true; - shutdownLBHttpSolrServer = true; - lbClient.addQueryParams(STATE_VERSION); - } - - /** - * Create a new client object that connects to Zookeeper and is always aware - * of the SolrCloud state. If there is a fully redundant Zookeeper quorum and - * SolrCloud has enough replicas for every shard in a collection, there is no - * single point of failure. Updates will be sent to shard leaders by default. - * - * @param zkHost - * The client endpoint of the zookeeper quorum containing the cloud - * state. The full specification for this string is one or more comma - * separated HOST:PORT values, followed by an optional chroot value - * that starts with a forward slash. Using a chroot allows multiple - * applications to coexist in one ensemble. For full details, see the - * Zookeeper documentation. Some examples: - * <p> - * "host1:2181" - * <p> - * "host1:2181,host2:2181,host3:2181/mysolrchroot" - * <p> - * "zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181" - * @param httpClient - * the {@link HttpClient} instance to be used for all requests. The - * provided httpClient should use a multi-threaded connection manager. - */ - public CloudSolrClient(String zkHost, HttpClient httpClient) { - this.zkHost = zkHost; - this.clientIsInternal = httpClient == null; - this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient; - this.lbClient = new LBHttpSolrClient(myClient); - this.lbClient.setRequestWriter(new BinaryRequestWriter()); - this.lbClient.setParser(new BinaryResponseParser()); - this.updatesToLeaders = true; - shutdownLBHttpSolrServer = true; - lbClient.addQueryParams(STATE_VERSION); - } - - /** - * Create a new client object using multiple string values in a Collection - * instead of a standard zkHost connection string. Note that this method will - * not be used if there is only one String argument - that will use - * {@link #CloudSolrClient(String)} instead. - * - * @param zkHosts - * A Java Collection (List, Set, etc) of HOST:PORT strings, one for - * each host in the zookeeper ensemble. Note that with certain - * Collection types like HashSet, the order of hosts in the final - * connect string may not be in the same order you added them. - * @param chroot - * A chroot value for zookeeper, starting with a forward slash. If no - * chroot is required, use null. - * @throws IllegalArgumentException - * if the chroot value does not start with a forward slash. - * @see #CloudSolrClient(String) - */ - public CloudSolrClient(Collection<String> zkHosts, String chroot) { - this(zkHosts, chroot, null); - } - - /** - * Create a new client object using multiple string values in a Collection - * instead of a standard zkHost connection string. Note that this method will - * not be used if there is only one String argument - that will use - * {@link #CloudSolrClient(String)} instead. - * - * @param zkHosts - * A Java Collection (List, Set, etc) of HOST:PORT strings, one for - * each host in the zookeeper ensemble. Note that with certain - * Collection types like HashSet, the order of hosts in the final - * connect string may not be in the same order you added them. - * @param chroot - * A chroot value for zookeeper, starting with a forward slash. If no - * chroot is required, use null. - * @param httpClient - * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a - * multi-threaded connection manager. - * @throws IllegalArgumentException - * if the chroot value does not start with a forward slash. - * @see #CloudSolrClient(String) - */ - public CloudSolrClient(Collection<String> zkHosts, String chroot, HttpClient httpClient) { - StringBuilder zkBuilder = new StringBuilder(); - int lastIndexValue = zkHosts.size() - 1; - int i = 0; - for (String zkHost : zkHosts) { - zkBuilder.append(zkHost); - if (i < lastIndexValue) { - zkBuilder.append(","); - } - i++; - } - if (chroot != null) { - if (chroot.startsWith("/")) { - zkBuilder.append(chroot); - } else { - throw new IllegalArgumentException( - "The chroot must start with a forward slash."); - } - } - - /* Log the constructed connection string and then initialize. */ - log.info("Final constructed zkHost string: " + zkBuilder.toString()); - - this.zkHost = zkBuilder.toString(); - this.clientIsInternal = httpClient == null; - this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient; - this.lbClient = new LBHttpSolrClient(myClient); - this.lbClient.setRequestWriter(new BinaryRequestWriter()); - this.lbClient.setParser(new BinaryResponseParser()); - this.updatesToLeaders = true; - shutdownLBHttpSolrServer = true; - } - - /** - * @param zkHost - * A zookeeper client endpoint. - * @param updatesToLeaders - * If true, sends updates only to shard leaders. - * @see #CloudSolrClient(String) for full description and details on zkHost - */ - public CloudSolrClient(String zkHost, boolean updatesToLeaders) { - this(zkHost, updatesToLeaders, null); - } - - /** - * @param zkHost - * A zookeeper client endpoint. - * @param updatesToLeaders - * If true, sends updates only to shard leaders. - * @param httpClient - * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a - * multi-threaded connection manager. - * @see #CloudSolrClient(String) for full description and details on zkHost - */ - public CloudSolrClient(String zkHost, boolean updatesToLeaders, HttpClient httpClient) { - this.zkHost = zkHost; - this.clientIsInternal = httpClient == null; - this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient; - this.lbClient = new LBHttpSolrClient(myClient); - this.lbClient.setRequestWriter(new BinaryRequestWriter()); - this.lbClient.setParser(new BinaryResponseParser()); - this.updatesToLeaders = updatesToLeaders; - shutdownLBHttpSolrServer = true; - lbClient.addQueryParams(STATE_VERSION); - } - - /**Sets the cache ttl for DocCollection Objects cached . This is only applicable for collections which are persisted outside of clusterstate.json - * @param seconds ttl value in seconds - */ - public void setCollectionCacheTTl(int seconds){ - assert seconds > 0; - timeToLive = seconds*1000L; - } - - /** - * @param zkHost - * A zookeeper client endpoint. - * @param lbClient - * LBHttpSolrServer instance for requests. - * @see #CloudSolrClient(String) for full description and details on zkHost - */ - public CloudSolrClient(String zkHost, LBHttpSolrClient lbClient) { - this(zkHost, lbClient, true); - } - - /** - * @param zkHost - * A zookeeper client endpoint. - * @param lbClient - * LBHttpSolrServer instance for requests. - * @param updatesToLeaders - * If true, sends updates only to shard leaders. - * @see #CloudSolrClient(String) for full description and details on zkHost - */ - public CloudSolrClient(String zkHost, LBHttpSolrClient lbClient, boolean updatesToLeaders) { - this.zkHost = zkHost; - this.lbClient = lbClient; - this.updatesToLeaders = updatesToLeaders; - shutdownLBHttpSolrServer = false; - this.clientIsInternal = false; - lbClient.addQueryParams(STATE_VERSION); - } - - public ResponseParser getParser() { - return lbClient.getParser(); - } - - /** - * Note: This setter method is <b>not thread-safe</b>. - * - * @param processor - * Default Response Parser chosen to parse the response if the parser - * were not specified as part of the request. - * @see org.apache.solr.client.solrj.SolrRequest#getResponseParser() - */ - public void setParser(ResponseParser processor) { - lbClient.setParser(processor); - } - - public RequestWriter getRequestWriter() { - return lbClient.getRequestWriter(); - } - - public void setRequestWriter(RequestWriter requestWriter) { - lbClient.setRequestWriter(requestWriter); - } - - /** - * @return the zkHost value used to connect to zookeeper. - */ - public String getZkHost() { - return zkHost; - } - - public ZkStateReader getZkStateReader() { - return zkStateReader; - } - - /** - * @param idField the field to route documents on. - */ - public void setIdField(String idField) { - this.idField = idField; - } - - /** - * @return the field that updates are routed on. - */ - public String getIdField() { - return idField; - } - - /** Sets the default collection for request */ - public void setDefaultCollection(String collection) { - this.defaultCollection = collection; - } - - /** Gets the default collection for request */ - public String getDefaultCollection() { - return defaultCollection; - } - - /** Set the connect timeout to the zookeeper ensemble in ms */ - public void setZkConnectTimeout(int zkConnectTimeout) { - this.zkConnectTimeout = zkConnectTimeout; - } - - /** Set the timeout to the zookeeper ensemble in ms */ - public void setZkClientTimeout(int zkClientTimeout) { - this.zkClientTimeout = zkClientTimeout; - } - - /** - * Connect to the zookeeper ensemble. - * This is an optional method that may be used to force a connect before any other requests are sent. - * - */ - public void connect() { - if (zkStateReader == null) { - synchronized (this) { - if (zkStateReader == null) { - ZkStateReader zk = null; - try { - zk = new ZkStateReader(zkHost, zkClientTimeout, zkConnectTimeout); - zk.createClusterStateWatchersAndUpdate(); - zkStateReader = zk; - } catch (InterruptedException e) { - zk.close(); - Thread.currentThread().interrupt(); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (KeeperException e) { - zk.close(); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (Exception e) { - if (zk != null) zk.close(); - // do not wrap because clients may be relying on the underlying exception being thrown - throw e; - } - } - } - } - } - - public void setParallelUpdates(boolean parallelUpdates) { - this.parallelUpdates = parallelUpdates; - } - - /** - * Upload a set of config files to Zookeeper and give it a name - * - * NOTE: You should only allow trusted users to upload configs. If you - * are allowing client access to zookeeper, you should protect the - * /configs node against unauthorised write access. - * - * @param configPath {@link java.nio.file.Path} to the config files - * @param configName the name of the config - * @throws IOException if an IO error occurs - */ - public void uploadConfig(Path configPath, String configName) throws IOException { - zkStateReader.getConfigManager().uploadConfigDir(configPath, configName); - } - - /** - * Download a named config from Zookeeper to a location on the filesystem - * @param configName the name of the config - * @param downloadPath the path to write config files to - * @throws IOException if an I/O exception occurs - */ - public void downloadConfig(String configName, Path downloadPath) throws IOException { - zkStateReader.getConfigManager().downloadConfigDir(configName, downloadPath); - } - - private NamedList<Object> directUpdate(AbstractUpdateRequest request, ClusterState clusterState) throws SolrServerException { - UpdateRequest updateRequest = (UpdateRequest) request; - ModifiableSolrParams params = (ModifiableSolrParams) request.getParams(); - ModifiableSolrParams routableParams = new ModifiableSolrParams(); - ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams(); - - if(params != null) { - nonRoutableParams.add(params); - routableParams.add(params); - for(String param : NON_ROUTABLE_PARAMS) { - routableParams.remove(param); - } - } - - String collection = nonRoutableParams.get(UpdateParams.COLLECTION, defaultCollection); - if (collection == null) { - throw new SolrServerException("No collection param specified on request and no default collection has been set."); - } - - - //Check to see if the collection is an alias. - Aliases aliases = zkStateReader.getAliases(); - if(aliases != null) { - Map<String, String> collectionAliases = aliases.getCollectionAliasMap(); - if(collectionAliases != null && collectionAliases.containsKey(collection)) { - collection = collectionAliases.get(collection); - } - } - - DocCollection col = getDocCollection(clusterState, collection,null); - - DocRouter router = col.getRouter(); - - if (router instanceof ImplicitDocRouter) { - // short circuit as optimization - return null; - } - - //Create the URL map, which is keyed on slice name. - //The value is a list of URLs for each replica in the slice. - //The first value in the list is the leader for the slice. - Map<String,List<String>> urlMap = buildUrlMap(col); - if (urlMap == null) { - // we could not find a leader yet - use unoptimized general path - return null; - } - - NamedList<Throwable> exceptions = new NamedList<>(); - NamedList<NamedList> shardResponses = new NamedList<>(); - - Map<String, LBHttpSolrClient.Req> routes = updateRequest.getRoutes(router, col, urlMap, routableParams, this.idField); - if (routes == null) { - return null; - } - - long start = System.nanoTime(); - - if (parallelUpdates) { - final Map<String, Future<NamedList<?>>> responseFutures = new HashMap<>(routes.size()); - for (final Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) { - final String url = entry.getKey(); - final LBHttpSolrClient.Req lbRequest = entry.getValue(); - responseFutures.put(url, threadPool.submit(new Callable<NamedList<?>>() { - @Override - public NamedList<?> call() throws Exception { - return lbClient.request(lbRequest).getResponse(); - } - })); - } - - for (final Map.Entry<String, Future<NamedList<?>>> entry: responseFutures.entrySet()) { - final String url = entry.getKey(); - final Future<NamedList<?>> responseFuture = entry.getValue(); - try { - shardResponses.add(url, responseFuture.get()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } catch (ExecutionException e) { - exceptions.add(url, e.getCause()); - } - } - - if (exceptions.size() > 0) { - throw new RouteException(ErrorCode.SERVER_ERROR, exceptions, routes); - } - } else { - for (Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) { - String url = entry.getKey(); - LBHttpSolrClient.Req lbRequest = entry.getValue(); - try { - NamedList<Object> rsp = lbClient.request(lbRequest).getResponse(); - shardResponses.add(url, rsp); - } catch (Exception e) { - throw new SolrServerException(e); - } - } - } - - UpdateRequest nonRoutableRequest = null; - List<String> deleteQuery = updateRequest.getDeleteQuery(); - if (deleteQuery != null && deleteQuery.size() > 0) { - UpdateRequest deleteQueryRequest = new UpdateRequest(); - deleteQueryRequest.setDeleteQuery(deleteQuery); - nonRoutableRequest = deleteQueryRequest; - } - - Set<String> paramNames = nonRoutableParams.getParameterNames(); - - Set<String> intersection = new HashSet<>(paramNames); - intersection.retainAll(NON_ROUTABLE_PARAMS); - - if (nonRoutableRequest != null || intersection.size() > 0) { - if (nonRoutableRequest == null) { - nonRoutableRequest = new UpdateRequest(); - } - nonRoutableRequest.setParams(nonRoutableParams); - List<String> urlList = new ArrayList<>(); - urlList.addAll(routes.keySet()); - Collections.shuffle(urlList, rand); - LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(nonRoutableRequest, urlList); - try { - LBHttpSolrClient.Rsp rsp = lbClient.request(req); - shardResponses.add(urlList.get(0), rsp.getResponse()); - } catch (Exception e) { - throw new SolrException(ErrorCode.SERVER_ERROR, urlList.get(0), e); - } - } - - long end = System.nanoTime(); - - RouteResponse rr = condenseResponse(shardResponses, (long)((end - start)/1000000)); - rr.setRouteResponses(shardResponses); - rr.setRoutes(routes); - return rr; - } - - private Map<String,List<String>> buildUrlMap(DocCollection col) { - Map<String, List<String>> urlMap = new HashMap<>(); - Collection<Slice> slices = col.getActiveSlices(); - Iterator<Slice> sliceIterator = slices.iterator(); - while (sliceIterator.hasNext()) { - Slice slice = sliceIterator.next(); - String name = slice.getName(); - List<String> urls = new ArrayList<>(); - Replica leader = slice.getLeader(); - if (leader == null) { - // take unoptimized general path - we cannot find a leader yet - return null; - } - ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader); - String url = zkProps.getCoreUrl(); - urls.add(url); - Collection<Replica> replicas = slice.getReplicas(); - Iterator<Replica> replicaIterator = replicas.iterator(); - while (replicaIterator.hasNext()) { - Replica replica = replicaIterator.next(); - if (!replica.getNodeName().equals(leader.getNodeName()) && - !replica.getName().equals(leader.getName())) { - ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica); - String url1 = zkProps1.getCoreUrl(); - urls.add(url1); - } - } - urlMap.put(name, urls); - } - return urlMap; - } - - public RouteResponse condenseResponse(NamedList response, long timeMillis) { - RouteResponse condensed = new RouteResponse(); - int status = 0; - Integer rf = null; - Integer minRf = null; - for(int i=0; i<response.size(); i++) { - NamedList shardResponse = (NamedList)response.getVal(i); - NamedList header = (NamedList)shardResponse.get("responseHeader"); - Integer shardStatus = (Integer)header.get("status"); - int s = shardStatus.intValue(); - if(s > 0) { - status = s; - } - Object rfObj = header.get(UpdateRequest.REPFACT); - if (rfObj != null && rfObj instanceof Integer) { - Integer routeRf = (Integer)rfObj; - if (rf == null || routeRf < rf) - rf = routeRf; - } - minRf = (Integer)header.get(UpdateRequest.MIN_REPFACT); - } - - NamedList cheader = new NamedList(); - cheader.add("status", status); - cheader.add("QTime", timeMillis); - if (rf != null) - cheader.add(UpdateRequest.REPFACT, rf); - if (minRf != null) - cheader.add(UpdateRequest.MIN_REPFACT, minRf); - - condensed.add("responseHeader", cheader); - return condensed; - } - - public static class RouteResponse extends NamedList { - private NamedList routeResponses; - private Map<String, LBHttpSolrClient.Req> routes; - - public void setRouteResponses(NamedList routeResponses) { - this.routeResponses = routeResponses; - } - - public NamedList getRouteResponses() { - return routeResponses; - } - - public void setRoutes(Map<String, LBHttpSolrClient.Req> routes) { - this.routes = routes; - } - - public Map<String, LBHttpSolrClient.Req> getRoutes() { - return routes; - } - - } - - public static class RouteException extends SolrException { - - private NamedList<Throwable> throwables; - private Map<String, LBHttpSolrClient.Req> routes; - - public RouteException(ErrorCode errorCode, NamedList<Throwable> throwables, Map<String, LBHttpSolrClient.Req> routes){ - super(errorCode, throwables.getVal(0).getMessage(), throwables.getVal(0)); - this.throwables = throwables; - this.routes = routes; - } - - public NamedList<Throwable> getThrowables() { - return throwables; - } - - public Map<String, LBHttpSolrClient.Req> getRoutes() { - return this.routes; - } - } - - @Override - public NamedList<Object> request(SolrRequest request) throws SolrServerException, IOException { - SolrParams reqParams = request.getParams(); - String collection = (reqParams != null) ? reqParams.get("collection", getDefaultCollection()) : getDefaultCollection(); - return requestWithRetryOnStaleState(request, 0, collection); - } - - /** - * As this class doesn't watch external collections on the client side, - * there's a chance that the request will fail due to cached stale state, - * which means the state must be refreshed from ZK and retried. - */ - protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, String collection) - throws SolrServerException, IOException { - - connect(); // important to call this before you start working with the ZkStateReader - - // build up a _stateVer_ param to pass to the server containing all of the - // external collection state versions involved in this request, which allows - // the server to notify us that our cached state for one or more of the external - // collections is stale and needs to be refreshed ... this code has no impact on internal collections - String stateVerParam = null; - List<DocCollection> requestedCollections = null; - if (collection != null && !request.getPath().startsWith("/admin")) { // don't do _stateVer_ checking for admin requests - Set<String> requestedCollectionNames = getCollectionNames(getZkStateReader().getClusterState(), collection); - - StringBuilder stateVerParamBuilder = null; - for (String requestedCollection : requestedCollectionNames) { - // track the version of state we're using on the client side using the _stateVer_ param - DocCollection coll = getDocCollection(getZkStateReader().getClusterState(), requestedCollection,null); - int collVer = coll.getZNodeVersion(); - if (coll.getStateFormat()>1) { - if(requestedCollections == null) requestedCollections = new ArrayList<>(requestedCollectionNames.size()); - requestedCollections.add(coll); - - if (stateVerParamBuilder == null) { - stateVerParamBuilder = new StringBuilder(); - } else { - stateVerParamBuilder.append("|"); // hopefully pipe is not an allowed char in a collection name - } - - stateVerParamBuilder.append(coll.getName()).append(":").append(collVer); - } - } - - if (stateVerParamBuilder != null) { - stateVerParam = stateVerParamBuilder.toString(); - } - } - - if (request.getParams() instanceof ModifiableSolrParams) { - ModifiableSolrParams params = (ModifiableSolrParams) request.getParams(); - if (stateVerParam != null) { - params.set(STATE_VERSION, stateVerParam); - } else { - params.remove(STATE_VERSION); - } - } // else: ??? how to set this ??? - - NamedList<Object> resp = null; - try { - resp = sendRequest(request); - //to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there - Object o = resp.get(STATE_VERSION, resp.size()-1); - if(o != null && o instanceof Map) { - //remove this because no one else needs this and tests would fail if they are comparing responses - resp.remove(resp.size()-1); - Map invalidStates = (Map) o; - for (Object invalidEntries : invalidStates.entrySet()) { - Map.Entry e = (Map.Entry) invalidEntries; - getDocCollection(getZkStateReader().getClusterState(),(String)e.getKey(), (Integer)e.getValue()); - } - - } - } catch (Exception exc) { - - Throwable rootCause = SolrException.getRootCause(exc); - // don't do retry support for admin requests or if the request doesn't have a collection specified - if (collection == null || request.getPath().startsWith("/admin")) { - if (exc instanceof SolrServerException) { - throw (SolrServerException)exc; - } else if (exc instanceof IOException) { - throw (IOException)exc; - }else if (exc instanceof RuntimeException) { - throw (RuntimeException) exc; - } - else { - throw new SolrServerException(rootCause); - } - } - - int errorCode = (rootCause instanceof SolrException) ? - ((SolrException)rootCause).code() : SolrException.ErrorCode.UNKNOWN.code; - - log.error("Request to collection {} failed due to ("+errorCode+ - ") {}, retry? "+retryCount, collection, rootCause.toString()); - - boolean wasCommError = - (rootCause instanceof ConnectException || - rootCause instanceof ConnectTimeoutException || - rootCause instanceof NoHttpResponseException || - rootCause instanceof SocketException); - - boolean stateWasStale = false; - if (retryCount < MAX_STALE_RETRIES && - requestedCollections != null && - !requestedCollections.isEmpty() && - SolrException.ErrorCode.getErrorCode(errorCode) == SolrException.ErrorCode.INVALID_STATE) - { - // cached state for one or more external collections was stale - // re-issue request using updated state - stateWasStale = true; - - // just re-read state for all of them, which is a little heavy handed but hopefully a rare occurrence - for (DocCollection ext : requestedCollections) { - collectionStateCache.remove(ext.getName()); - } - } - - // if we experienced a communication error, it's worth checking the state - // with ZK just to make sure the node we're trying to hit is still part of the collection - if (retryCount < MAX_STALE_RETRIES && - !stateWasStale && - requestedCollections != null && - !requestedCollections.isEmpty() && - wasCommError) { - for (DocCollection ext : requestedCollections) { - DocCollection latestStateFromZk = getDocCollection(zkStateReader.getClusterState(), ext.getName(),null); - if (latestStateFromZk.getZNodeVersion() != ext.getZNodeVersion()) { - // looks like we couldn't reach the server because the state was stale == retry - stateWasStale = true; - // we just pulled state from ZK, so update the cache so that the retry uses it - collectionStateCache.put(ext.getName(), new ExpiringCachedDocCollection(latestStateFromZk)); - } - } - } - - if (requestedCollections != null) { - requestedCollections.clear(); // done with this - } - - // if the state was stale, then we retry the request once with new state pulled from Zk - if (stateWasStale) { - log.warn("Re-trying request to collection(s) "+collection+" after stale state error from server."); - resp = requestWithRetryOnStaleState(request, retryCount+1, collection); - } else { - if (exc instanceof SolrServerException) { - throw (SolrServerException)exc; - } else if (exc instanceof IOException) { - throw (IOException)exc; - } else { - throw new SolrServerException(rootCause); - } - } - } - - return resp; - } - - protected NamedList<Object> sendRequest(SolrRequest request) - throws SolrServerException, IOException { - connect(); - - ClusterState clusterState = zkStateReader.getClusterState(); - - boolean sendToLeaders = false; - List<String> replicas = null; - - if (request instanceof IsUpdateRequest) { - if (request instanceof UpdateRequest) { - NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, - clusterState); - if (response != null) { - return response; - } - } - sendToLeaders = true; - replicas = new ArrayList<>(); - } - - SolrParams reqParams = request.getParams(); - if (reqParams == null) { - reqParams = new ModifiableSolrParams(); - } - List<String> theUrlList = new ArrayList<>(); - if (request.getPath().equals("/admin/collections") - || request.getPath().equals("/admin/cores")) { - Set<String> liveNodes = clusterState.getLiveNodes(); - for (String liveNode : liveNodes) { - theUrlList.add(zkStateReader.getBaseUrlForNodeName(liveNode)); - } - } else { - String collection = reqParams.get(UpdateParams.COLLECTION, defaultCollection); - - if (collection == null) { - throw new SolrServerException( - "No collection param specified on request and no default collection has been set."); - } - - Set<String> collectionNames = getCollectionNames(clusterState, collection); - if (collectionNames.size() == 0) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Could not find collection: " + collection); - } - - String shardKeys = reqParams.get(ShardParams._ROUTE_); - - // TODO: not a big deal because of the caching, but we could avoid looking - // at every shard - // when getting leaders if we tweaked some things - - // Retrieve slices from the cloud state and, for each collection - // specified, - // add it to the Map of slices. - Map<String,Slice> slices = new HashMap<>(); - for (String collectionName : collectionNames) { - DocCollection col = getDocCollection(clusterState, collectionName, null); - Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams , col); - ClientUtils.addSlices(slices, collectionName, routeSlices, true); - } - Set<String> liveNodes = clusterState.getLiveNodes(); - - List<String> leaderUrlList = null; - List<String> urlList = null; - List<String> replicasList = null; - - // build a map of unique nodes - // TODO: allow filtering by group, role, etc - Map<String,ZkNodeProps> nodes = new HashMap<>(); - List<String> urlList2 = new ArrayList<>(); - for (Slice slice : slices.values()) { - for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) { - ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps); - String node = coreNodeProps.getNodeName(); - if (!liveNodes.contains(coreNodeProps.getNodeName()) - || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue; - if (nodes.put(node, nodeProps) == null) { - if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) { - String url; - if (reqParams.get(UpdateParams.COLLECTION) == null) { - url = ZkCoreNodeProps.getCoreUrl( - nodeProps.getStr(ZkStateReader.BASE_URL_PROP), - defaultCollection); - } else { - url = coreNodeProps.getCoreUrl(); - } - urlList2.add(url); - } else if (sendToLeaders) { - String url; - if (reqParams.get(UpdateParams.COLLECTION) == null) { - url = ZkCoreNodeProps.getCoreUrl( - nodeProps.getStr(ZkStateReader.BASE_URL_PROP), - defaultCollection); - } else { - url = coreNodeProps.getCoreUrl(); - } - replicas.add(url); - } - } - } - } - - if (sendToLeaders) { - leaderUrlList = urlList2; - replicasList = replicas; - } else { - urlList = urlList2; - } - - if (sendToLeaders) { - theUrlList = new ArrayList<>(leaderUrlList.size()); - theUrlList.addAll(leaderUrlList); - } else { - theUrlList = new ArrayList<>(urlList.size()); - theUrlList.addAll(urlList); - } - if(theUrlList.isEmpty()) { - for (String s : collectionNames) { - if(s!=null) collectionStateCache.remove(s); - } - throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Not enough nodes to handle the request"); - } - - Collections.shuffle(theUrlList, rand); - if (sendToLeaders) { - ArrayList<String> theReplicas = new ArrayList<>( - replicasList.size()); - theReplicas.addAll(replicasList); - Collections.shuffle(theReplicas, rand); - theUrlList.addAll(theReplicas); - } - - } - - LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(request, theUrlList); - LBHttpSolrClient.Rsp rsp = lbClient.request(req); - return rsp.getResponse(); - } - - private Set<String> getCollectionNames(ClusterState clusterState, - String collection) { - // Extract each comma separated collection name and store in a List. - List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true); - Set<String> collectionNames = new HashSet<>(); - // validate collections - for (String collectionName : rawCollectionsList) { - if (!clusterState.getCollections().contains(collectionName)) { - Aliases aliases = zkStateReader.getAliases(); - String alias = aliases.getCollectionAlias(collectionName); - if (alias != null) { - List<String> aliasList = StrUtils.splitSmart(alias, ",", true); - collectionNames.addAll(aliasList); - continue; - } - - throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName); - } - - collectionNames.add(collectionName); - } - return collectionNames; - } - - @Override - public void close() throws IOException { - shutdown(); - } - - @Override - @Deprecated - public void shutdown() { - if (zkStateReader != null) { - synchronized(this) { - if (zkStateReader!= null) - zkStateReader.close(); - zkStateReader = null; - } - } - - if (shutdownLBHttpSolrServer) { - lbClient.shutdown(); - } - - if (clientIsInternal && myClient!=null) { - HttpClientUtil.close(myClient); - } - - if(this.threadPool != null && !this.threadPool.isShutdown()) { - this.threadPool.shutdown(); - } - } - - public LBHttpSolrClient getLbClient() { - return lbClient; - } - - public boolean isUpdatesToLeaders() { - return updatesToLeaders; - } - - /**If caches are expired they are refreshed after acquiring a lock. - * use this to set the number of locks - */ - public void setParallelCacheRefreshes(int n){ locks = objectList(n); } - - private static ArrayList<Object> objectList(int n) { - ArrayList<Object> l = new ArrayList<>(n); - for(int i=0;i<n;i++) l.add(new Object()); - return l; - } - - - protected DocCollection getDocCollection(ClusterState clusterState, String collection, Integer expectedVersion) throws SolrException { - if (collection == null) return null; - DocCollection col = getFromCache(collection); - if (col != null) { - if (expectedVersion == null) return col; - if (expectedVersion.intValue() == col.getZNodeVersion()) return col; - } - - ClusterState.CollectionRef ref = clusterState.getCollectionRef(collection); - if (ref == null) { - //no such collection exists - return null; - } - if (!ref.isLazilyLoaded()) { - //it is readily available just return it - return ref.get(); - } - List locks = this.locks; - final Object lock = locks.get(Math.abs(Hash.murmurhash3_x86_32(collection, 0, collection.length(), 0) % locks.size())); - synchronized (lock) { - //we have waited for sometime just check once again - col = getFromCache(collection); - if (col != null) { - if (expectedVersion == null) return col; - if (expectedVersion.intValue() == col.getZNodeVersion()) { - return col; - } else { - collectionStateCache.remove(collection); - } - } - col = ref.get();//this is a call to ZK - } - if (col == null) return null; - if (col.getStateFormat() > 1) collectionStateCache.put(collection, new ExpiringCachedDocCollection(col)); - return col; - } - - private DocCollection getFromCache(String c){ - ExpiringCachedDocCollection cachedState = collectionStateCache.get(c); - return cachedState != null ? cachedState.cached : null; - } - - - /** - * Useful for determining the minimum achieved replication factor across - * all shards involved in processing an update request, typically useful - * for gauging the replication factor of a batch. - */ - @SuppressWarnings("rawtypes") - public int getMinAchievedReplicationFactor(String collection, NamedList resp) { - // it's probably already on the top-level header set by condense - NamedList header = (NamedList)resp.get("responseHeader"); - Integer achRf = (Integer)header.get(UpdateRequest.REPFACT); - if (achRf != null) - return achRf.intValue(); - - // not on the top-level header, walk the shard route tree - Map<String,Integer> shardRf = getShardReplicationFactor(collection, resp); - for (Integer rf : shardRf.values()) { - if (achRf == null || rf < achRf) { - achRf = rf; - } - } - return (achRf != null) ? achRf.intValue() : -1; - } - - /** - * Walks the NamedList response after performing an update request looking for - * the replication factor that was achieved in each shard involved in the request. - * For single doc updates, there will be only one shard in the return value. - */ - @SuppressWarnings("rawtypes") - public Map<String,Integer> getShardReplicationFactor(String collection, NamedList resp) { - connect(); - - Map<String,Integer> results = new HashMap<String,Integer>(); - if (resp instanceof CloudSolrClient.RouteResponse) { - NamedList routes = ((CloudSolrClient.RouteResponse)resp).getRouteResponses(); - ClusterState clusterState = zkStateReader.getClusterState(); - Map<String,String> leaders = new HashMap<String,String>(); - for (Slice slice : clusterState.getActiveSlices(collection)) { - Replica leader = slice.getLeader(); - if (leader != null) { - ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader); - String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName(); - leaders.put(leaderUrl, slice.getName()); - String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection; - leaders.put(altLeaderUrl, slice.getName()); - } - } - - Iterator<Map.Entry<String,Object>> routeIter = routes.iterator(); - while (routeIter.hasNext()) { - Map.Entry<String,Object> next = routeIter.next(); - String host = next.getKey(); - NamedList hostResp = (NamedList)next.getValue(); - Integer rf = (Integer)((NamedList)hostResp.get("responseHeader")).get(UpdateRequest.REPFACT); - if (rf != null) { - String shard = leaders.get(host); - if (shard == null) { - if (host.endsWith("/")) - shard = leaders.get(host.substring(0,host.length()-1)); - if (shard == null) { - shard = host; - } - } - results.put(shard, rf); - } - } - } - return results; - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java deleted file mode 100644 index 4e2a2e7..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.impl; - -import org.apache.http.client.HttpClient; - -import java.util.Collection; - -/** - * @deprecated Use {@link org.apache.solr.client.solrj.impl.CloudSolrClient} - */ -@Deprecated -public class CloudSolrServer extends CloudSolrClient { - - public CloudSolrServer(String zkHost) { - super(zkHost); - } - - public CloudSolrServer(String zkHost, HttpClient httpClient) { - super(zkHost, httpClient); - } - - public CloudSolrServer(Collection<String> zkHosts, String chroot) { - super(zkHosts, chroot); - } - - public CloudSolrServer(Collection<String> zkHosts, String chroot, HttpClient httpClient) { - super(zkHosts, chroot, httpClient); - } - - public CloudSolrServer(String zkHost, boolean updatesToLeaders) { - super(zkHost, updatesToLeaders); - } - - public CloudSolrServer(String zkHost, boolean updatesToLeaders, HttpClient httpClient) { - super(zkHost, updatesToLeaders, httpClient); - } - - public CloudSolrServer(String zkHost, LBHttpSolrClient lbClient) { - super(zkHost, lbClient); - } - - public CloudSolrServer(String zkHost, LBHttpSolrClient lbClient, boolean updatesToLeaders) { - super(zkHost, lbClient, updatesToLeaders); - } -}
