Repository: incubator-blur
Updated Branches:
  refs/heads/master e0a6a106e -> 5f16374ef


Fixing issues with cache when the layout of the cluster would change due to a 
shard server failure.  The cluster test was flawed because it disabled the use 
of cached results.  Also improved the error message when the controller can not 
complete a query due to shard server errors.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5f16374e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5f16374e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5f16374e

Branch: refs/heads/master
Commit: 5f16374efdbdd1cb3c62a167a10e4c859a04af1b
Parents: e0a6a10
Author: Aaron McCurry <[email protected]>
Authored: Sun Mar 1 13:56:10 2015 -0500
Committer: Aaron McCurry <[email protected]>
Committed: Sun Mar 1 13:56:10 2015 -0500

----------------------------------------------------------------------
 .../org/apache/blur/server/cache/ClassObj.java  | 51 ++++++++++++
 .../apache/blur/server/cache/ThriftCache.java   | 38 ++++++---
 .../blur/server/cache/ThriftCacheKey.java       | 54 ++++++++----
 .../blur/server/cache/ThriftCacheServer.java    | 48 ++++++++---
 .../blur/server/cache/ThriftCacheValue.java     | 35 +++++++-
 .../blur/thrift/BlurControllerServer.java       |  5 +-
 .../thrift/QueryCoundNotBeCompletedReport.java  | 81 ++++++++++++++++++
 .../blur/thrift/ThriftBlurShardServer.java      |  2 +-
 .../blur/server/cache/ThriftCacheKeyTest.java   | 86 +++++++++++++++-----
 .../server/cache/ThriftCacheServerTest.java     | 54 +++++++++++-
 .../blur/server/cache/ThriftCacheTest.java      |  6 +-
 .../apache/blur/thrift/BlurClusterTestBase.java |  1 -
 12 files changed, 393 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java 
b/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java
new file mode 100644
index 0000000..eabcc7b
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.server.cache;
+
+public class ClassObj<T> {
+
+  private final Class<T> _clazz;
+
+  public ClassObj(Class<T> clazz) {
+    if (clazz == null) {
+      throw new RuntimeException("Class cannot be null.");
+    }
+    _clazz = clazz;
+  }
+
+  public Class<T> getClazz() {
+    return _clazz;
+  }
+
+  @Override
+  public int hashCode() {
+    return _clazz.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ClassObj<?> other = (ClassObj<?>) obj;
+    return _clazz == other._clazz;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java 
b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
index e4b428c..7093a6a 100644
--- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
@@ -25,6 +25,8 @@ import static 
org.apache.blur.metrics.MetricsConstants.THRIFT_CACHE;
 import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -36,8 +38,8 @@ import org.apache.blur.user.User;
 import org.apache.blur.user.UserContext;
 
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import com.googlecode.concurrentlinkedhashmap.EntryWeigher;
 import com.googlecode.concurrentlinkedhashmap.EvictionListener;
-import com.googlecode.concurrentlinkedhashmap.Weigher;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Meter;
 import com.yammer.metrics.core.MetricName;
@@ -47,6 +49,7 @@ public class ThriftCache {
   private static final Log LOG = LogFactory.getLog(ThriftCache.class);
 
   private final ConcurrentLinkedHashMap<ThriftCacheKey<?>, 
ThriftCacheValue<?>> _cacheMap;
+  private final ConcurrentMap<String, Long> _lastModTimestamps = new 
ConcurrentHashMap<String, Long>();
   private final Meter _hits;
   private final Meter _misses;
   private final Meter _evictions;
@@ -59,10 +62,10 @@ public class ThriftCache {
     _misses = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, THRIFT_CACHE, 
MISS), MISS, TimeUnit.SECONDS);
     _evictions = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, 
THRIFT_CACHE, EVICTION), EVICTION, TimeUnit.SECONDS);
     _cacheMap = new ConcurrentLinkedHashMap.Builder<ThriftCacheKey<?>, 
ThriftCacheValue<?>>()
-        .weigher(new Weigher<ThriftCacheValue<?>>() {
+        .weigher(new EntryWeigher<ThriftCacheKey<?>, ThriftCacheValue<?>>() {
           @Override
-          public int weightOf(ThriftCacheValue<?> value) {
-            return value.size();
+          public int weightOf(ThriftCacheKey<?> key, ThriftCacheValue<?> 
value) {
+            return key.size() + value.size();
           }
         }).listener(new EvictionListener<ThriftCacheKey<?>, 
ThriftCacheValue<?>>() {
           @Override
@@ -77,6 +80,16 @@ public class ThriftCache {
   }
 
   public <K extends TBase<?, ?>, V extends TBase<?, ?>> V 
put(ThriftCacheKey<K> key, V t) throws BlurException {
+    synchronized (_lastModTimestamps) {
+      Long lastModTimestamp = _lastModTimestamps.get(key.getTable());
+      if (lastModTimestamp != null && key.getTimestamp() < lastModTimestamp) {
+        // This means that the key was created before the index was modified. 
So
+        // do not cache the value because it's already out of date with the
+        // index.
+        return t;
+      }
+    }
+    LOG.debug("Inserting into cache [{0}] with key [{1}]", t, key);
     _cacheMap.put(key, new ThriftCacheValue<V>(t));
     return t;
   }
@@ -97,17 +110,15 @@ public class ThriftCache {
     return value.getValue(clazz);
   }
 
-  public <K extends TBase<?, ?>> ThriftCacheKey<K> getKey(String table, K 
tkey, Class<K> clazz) {
+  public <K extends TBase<?, ?>> ThriftCacheKey<K> getKey(String table, int[] 
shards, K tkey, Class<K> clazz) throws BlurException {
     User user = UserContext.getUser();
-    return new ThriftCacheKey<K>(user, table, tkey, clazz);
-  }
-
-  public void clear() {
-    LOG.info("Clearing all cache.");
-    _cacheMap.clear();
+    return new ThriftCacheKey<K>(user, table, shards, tkey, clazz);
   }
 
   public void clearTable(String table) {
+    synchronized (_lastModTimestamps) {
+      _lastModTimestamps.put(table, System.nanoTime());
+    }
     LOG.info("Clearing cache for table [{0}]", table);
     Set<Entry<ThriftCacheKey<?>, ThriftCacheValue<?>>> entrySet = 
_cacheMap.entrySet();
     Iterator<Entry<ThriftCacheKey<?>, ThriftCacheValue<?>>> iterator = 
entrySet.iterator();
@@ -119,6 +130,11 @@ public class ThriftCache {
     }
   }
 
+  public void clear() {
+    LOG.info("Clearing all cache.");
+    _cacheMap.clear();
+  }
+
   public long size() {
     return _cacheMap.weightedSize();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java 
b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
index 790533b..5866966 100644
--- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
@@ -16,10 +16,12 @@
  */
 package org.apache.blur.server.cache;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.TreeMap;
 
 import org.apache.blur.thirdparty.thrift_0_9_0.TBase;
+import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.user.User;
 
 public class ThriftCacheKey<T extends TBase<?, ?>> {
@@ -27,11 +29,16 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
   private final String _username;
   private final Map<String, String> _attributes;
   private final String _table;
-  private final T _t;
-  private final String _clazz;
+  private final int[] _shards;
+  private final ClassObj<T> _clazz;
+  private final ThriftCacheValue<T> _key;
 
-  public ThriftCacheKey(User user, String table, T t, Class<T> clazz) {
-    _clazz = clazz.getName();
+  // not apart of the key for equally
+  private final transient long _timestamp;
+
+  public ThriftCacheKey(User user, String table, int[] shards, T t, Class<T> 
clazz) throws BlurException {
+    _timestamp = System.nanoTime();
+    _clazz = new ClassObj<T>(clazz);
     if (user != null) {
       _username = user.getUsername();
       Map<String, String> attributes = user.getAttributes();
@@ -45,7 +52,12 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
       _attributes = null;
     }
     _table = table;
-    _t = t;
+    _shards = shards;
+    _key = new ThriftCacheValue<T>(t);
+  }
+
+  public long getTimestamp() {
+    return _timestamp;
   }
 
   public String getTable() {
@@ -53,12 +65,28 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
   }
 
   @Override
+  public String toString() {
+    try {
+      return "ThriftCacheKey [_username=" + _username + ", _attributes=" + 
_attributes + ", _table=" + _table
+          + ", _shards=" + Arrays.toString(_shards) + ", _clazz=" + _clazz + 
", _key="
+          + _key.getValue(_clazz.getClazz()) + ", _timestamp=" + _timestamp + 
"]";
+    } catch (BlurException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public int size() {
+    return _key.size();
+  }
+
+  @Override
   public int hashCode() {
     final int prime = 31;
     int result = 1;
     result = prime * result + ((_attributes == null) ? 0 : 
_attributes.hashCode());
     result = prime * result + ((_clazz == null) ? 0 : _clazz.hashCode());
-    result = prime * result + ((_t == null) ? 0 : _t.hashCode());
+    result = prime * result + ((_key == null) ? 0 : _key.hashCode());
+    result = prime * result + Arrays.hashCode(_shards);
     result = prime * result + ((_table == null) ? 0 : _table.hashCode());
     result = prime * result + ((_username == null) ? 0 : _username.hashCode());
     return result;
@@ -83,10 +111,12 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
         return false;
     } else if (!_clazz.equals(other._clazz))
       return false;
-    if (_t == null) {
-      if (other._t != null)
+    if (_key == null) {
+      if (other._key != null)
         return false;
-    } else if (!_t.equals(other._t))
+    } else if (!_key.equals(other._key))
+      return false;
+    if (!Arrays.equals(_shards, other._shards))
       return false;
     if (_table == null) {
       if (other._table != null)
@@ -101,10 +131,4 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
     return true;
   }
 
-  @Override
-  public String toString() {
-    return "ThriftCacheKey [_username=" + _username + ", _attributes=" + 
_attributes + ", _table=" + _table + ", _t="
-        + _t + ", _clazz=" + _clazz + "]";
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java 
b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
index dbd395f..7264801 100644
--- 
a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
+++ 
b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
@@ -16,15 +16,20 @@
  */
 package org.apache.blur.server.cache;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 
 import org.apache.blur.BlurConfiguration;
+import org.apache.blur.manager.IndexServer;
 import org.apache.blur.server.FilteredBlurServer;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BException;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
@@ -32,19 +37,23 @@ import org.apache.blur.thrift.generated.BlurResults;
 import org.apache.blur.thrift.generated.FetchResult;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.TableStats;
+import org.apache.blur.utils.ShardUtil;
 
 public class ThriftCacheServer extends FilteredBlurServer {
 
   private final ThriftCache _thriftCache;
+  private final IndexServer _indexServer;
 
-  public ThriftCacheServer(BlurConfiguration configuration, Iface iface, 
ThriftCache thriftCache) {
+  public ThriftCacheServer(BlurConfiguration configuration, Iface iface, 
IndexServer indexServer,
+      ThriftCache thriftCache) {
     super(configuration, iface, true);
     _thriftCache = thriftCache;
+    _indexServer = indexServer;
   }
 
   @Override
   public TableStats tableStats(String table) throws BlurException, TException {
-    ThriftCacheKey<TableStats> key = _thriftCache.getKey(table, null, 
TableStats.class);
+    ThriftCacheKey<TableStats> key = _thriftCache.getKey(table, 
getShards(table), null, TableStats.class);
     TableStats results = _thriftCache.get(key, TableStats.class);
     if (results != null) {
       return results;
@@ -52,6 +61,22 @@ public class ThriftCacheServer extends FilteredBlurServer {
     return _thriftCache.put(key, super.tableStats(table));
   }
 
+  private int[] getShards(String table) throws BlurException {
+    try {
+      Set<String> keySet = _indexServer.getIndexes(table).keySet();
+      int[] shards = new int[keySet.size()];
+      int i = 0;
+      for (String s : keySet) {
+        int shardIndex = ShardUtil.getShardIndex(s);
+        shards[i++] = shardIndex;
+      }
+      Arrays.sort(shards);
+      return shards;
+    } catch (IOException e) {
+      throw new BException("Unknown error while trying to get current shards 
for table [{0}]", e, table);
+    }
+  }
+
   @Override
   public BlurResults query(String table, BlurQuery blurQuery) throws 
BlurException, TException {
     boolean useCacheIfPresent = blurQuery.isUseCacheIfPresent();
@@ -66,14 +91,16 @@ public class ThriftCacheServer extends FilteredBlurServer {
     copy.userContext = null;
     copy.cacheResult = false;
     copy.startTime = 0;
-    ThriftCacheKey<BlurQuery> key = _thriftCache.getKey(table, copy, 
BlurQuery.class);
-    BlurResults results = _thriftCache.get(key, BlurResults.class);
-    if (results != null && useCacheIfPresent) {
-      return results;
+    ThriftCacheKey<BlurQuery> key = _thriftCache.getKey(table, 
getShards(table), copy, BlurQuery.class);
+    if (useCacheIfPresent) {
+      BlurResults results = _thriftCache.get(key, BlurResults.class);
+      if (results != null) {
+        return results;
+      }
     }
     BlurResults blurResults = super.query(table, blurQuery);
     if (cacheResult) {
-      return _thriftCache.put(key, blurResults);  
+      return _thriftCache.put(key, blurResults);
     }
     return blurResults;
   }
@@ -81,7 +108,7 @@ public class ThriftCacheServer extends FilteredBlurServer {
   @Override
   public FetchResult fetchRow(String table, Selector selector) throws 
BlurException, TException {
     Selector copy = new Selector(selector);
-    ThriftCacheKey<Selector> key = _thriftCache.getKey(table, copy, 
Selector.class);
+    ThriftCacheKey<Selector> key = _thriftCache.getKey(table, 
getShards(table), copy, Selector.class);
     FetchResult results = _thriftCache.get(key, FetchResult.class);
     if (results != null) {
       return results;
@@ -101,7 +128,7 @@ public class ThriftCacheServer extends FilteredBlurServer {
     for (int i = 0; i < selectors.size(); i++) {
       Selector selector = selectors.get(i);
       Selector copy = new Selector(selector);
-      ThriftCacheKey<Selector> key = _thriftCache.getKey(table, copy, 
Selector.class);
+      ThriftCacheKey<Selector> key = _thriftCache.getKey(table, 
getShards(table), copy, Selector.class);
       FetchResult fetchResult = _thriftCache.get(key, FetchResult.class);
       if (fetchResult != null) {
         resultMap.put(i, fetchResult);
@@ -117,7 +144,8 @@ public class ThriftCacheServer extends FilteredBlurServer {
       for (int i = 0; i < missingResults.size(); i++) {
         Selector selector = selectorRequest.get(i);
         FetchResult fetchResult = missingResults.get(i);
-        ThriftCacheKey<Selector> key = _thriftCache.getKey(table, new 
Selector(selector), Selector.class);
+        ThriftCacheKey<Selector> key = _thriftCache.getKey(table, 
getShards(table), new Selector(selector),
+            Selector.class);
         _thriftCache.put(key, fetchResult);
         int originalIndex = requestMapping.get(i);
         resultMap.put(originalIndex, fetchResult);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java 
b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
index 7d259f4..84d5606 100644
--- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
@@ -16,6 +16,8 @@
  */
 package org.apache.blur.server.cache;
 
+import java.util.Arrays;
+
 import org.apache.blur.thirdparty.thrift_0_9_0.TBase;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TCompactProtocol;
@@ -29,8 +31,12 @@ public class ThriftCacheValue<T extends TBase<?, ?>> {
   private final byte[] _data;
 
   public ThriftCacheValue(T t) throws BlurException {
+    _data = toBytes(t);
+  }
+
+  public static <T extends TBase<?, ?>> byte[] toBytes(T t) throws BException {
     if (t == null) {
-      _data = null;
+      return null;
     } else {
       TMemoryBuffer transport = new TMemoryBuffer(1024);
       try {
@@ -38,11 +44,11 @@ public class ThriftCacheValue<T extends TBase<?, ?>> {
       } catch (TException e) {
         throw new BException("Unknown error while trying to read from cache.", 
e);
       }
-      _data = trim(transport.getArray(), transport.length());
+      return trim(transport.getArray(), transport.length());
     }
   }
 
-  private byte[] trim(byte[] bs, int len) {
+  public static byte[] trim(byte[] bs, int len) {
     if (bs.length == len) {
       return bs;
     }
@@ -75,4 +81,27 @@ public class ThriftCacheValue<T extends TBase<?, ?>> {
       throw new BException("Unknown error while trying to read from cache.", 
e);
     }
   }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + Arrays.hashCode(_data);
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ThriftCacheValue<?> other = (ThriftCacheValue<?>) obj;
+    if (!Arrays.equals(_data, other._data))
+      return false;
+    return true;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java 
b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index a29867b..4c45a90 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -475,6 +475,7 @@ public class BlurControllerServer extends TableAdmin 
implements Iface {
 
       BlurUtil.setStartTime(blurQuery);
 
+      QueryCoundNotBeCompletedReport report = null;
       OUTER: for (int retries = 0; retries < _maxDefaultRetries; retries++) {
         Tracer selectorTrace = Trace.trace("selector - setup", 
Trace.param("retries", retries));
         final AtomicLongArray facetCounts = 
BlurUtil.getAtomicLongArraySameLengthAsList(blurQuery.facets);
@@ -534,6 +535,8 @@ public class BlurControllerServer extends TableAdmin 
implements Iface {
           if (!validResults(results, shardCount, blurQuery)) {
             BlurClientManager.sleep(_defaultDelay, _maxDefaultDelay, retries, 
_maxDefaultRetries);
             Map<String, String> map = getTableLayout(table);
+            Map<String, Long> shardInfo = results.getShardInfo();
+            report = new QueryCoundNotBeCompletedReport(map, shardCount, 
shardInfo, blurQuery);
             LOG.info("Current layout for table [{0}] is [{1}]", table, map);
             continue OUTER;
           }
@@ -544,7 +547,7 @@ public class BlurControllerServer extends TableAdmin 
implements Iface {
           }
         }
       }
-      throw new BException("Query could not be completed.");
+      throw new BException("Query could not be completed, report [{0}].", 
report);
     } catch (Exception e) {
       LOG.error("Unknown error during search of [table={0},blurQuery={1}]", e, 
table, blurQuery);
       if (e instanceof BlurException) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
 
b/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
new file mode 100644
index 0000000..0c62724
--- /dev/null
+++ 
b/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.thrift;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class QueryCoundNotBeCompletedReport {
+
+  private final String _message;
+
+  public QueryCoundNotBeCompletedReport(Map<String, String> tableLayout, int 
shardCount, Map<String, Long> shardInfo,
+      BlurQuery blurQuery) throws JSONException {
+    String missingShardsFromResult = "\n";
+    if (shardCount != shardInfo.size()) {
+
+      Set<String> missingShardResults = new TreeSet<String>();
+      Set<String> servers = new TreeSet<String>();
+      for (Entry<String, String> e : tableLayout.entrySet()) {
+        String shard = e.getKey();
+        String server = e.getValue();
+        if (!shardInfo.containsKey(shard)) {
+          missingShardResults.add(shard);
+          servers.add(server);
+        }
+      }
+      missingShardsFromResult = "\nThere appears to be some missing shards 
from the results.\n";
+      missingShardsFromResult += "Missing Shards:\n\t" + missingShardResults + 
"\n";
+      missingShardsFromResult += "From Servers:\n\t" + servers + "\n";
+    }
+
+    JSONObject jsonObject = new JSONObject();
+    jsonObject.put("tableLayout", getTableLayout(tableLayout));
+    jsonObject.put("tableShardCount", shardCount);
+    jsonObject.put("queryShardCount", shardInfo.size());
+    jsonObject.put("queryResultCount", getShardInfo(shardInfo));
+    _message = missingShardsFromResult + jsonObject.toString(1);
+  }
+
+  private JSONObject getShardInfo(Map<String, Long> shardInfo) throws 
JSONException {
+    JSONObject j = new JSONObject();
+    for (Entry<String, Long> e : shardInfo.entrySet()) {
+      j.put(e.getKey(), e.getValue());
+    }
+    return j;
+  }
+
+  private JSONObject getTableLayout(Map<String, String> tableLayout) throws 
JSONException {
+    JSONObject j = new JSONObject();
+    for (Entry<String, String> e : tableLayout.entrySet()) {
+      j.put(e.getKey(), e.getValue());
+    }
+    return j;
+  }
+
+  @Override
+  public String toString() {
+    return _message;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java 
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 545bf48..d8f15b0 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -289,7 +289,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     List<ServerSecurityFilter> serverSecurity = 
getServerSecurityList(configuration,
         ServerSecurityFilterFactory.ServerType.SHARD);
 
-    Iface iface = new ThriftCacheServer(configuration, shardServer, 
thriftCache);
+    Iface iface = new ThriftCacheServer(configuration, shardServer, 
indexServer, thriftCache);
     iface = BlurUtil.wrapFilteredBlurServer(configuration, iface, true);
     iface = ServerSecurityUtil.applySecurity(iface, serverSecurity, true);
     iface = BlurUtil.recordMethodCallsAndAverageTimes(iface, Iface.class, 
false);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java 
b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
index 0ca81a6..8c5da28 100644
--- 
a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
+++ 
b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
@@ -16,11 +16,15 @@
  */
 package org.apache.blur.server.cache;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
+import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.user.User;
 import org.junit.Test;
@@ -28,92 +32,122 @@ import org.junit.Test;
 public class ThriftCacheKeyTest {
 
   @Test
-  public void test1() {
+  public void test1() throws BlurException {
     User user = null;
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test2() {
+  public void test2() throws BlurException {
     User user = null;
     String table = "t";
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, null, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, null, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, null,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, null,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test3() {
+  public void test3() throws BlurException {
     User user = new User("test", null);
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test4() {
+  public void test4() throws BlurException {
     User user = new User("test", map("a", "b"));
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, 
table, new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test5() {
+  public void test5a() throws BlurException {
     User user1 = new User("test1", null);
     User user2 = new User("test2", null);
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, 
table, bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, 
table, bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, 
table, new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, 
table, new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertFalse(key1.equals(key2));
     assertFalse(key1.hashCode() == key2.hashCode());
   }
 
   @Test
-  public void test6() {
+  public void test5b() throws BlurException {
+    User user1 = new User("test1", null);
+    User user2 = new User("test1", null);
+    String table = "t";
+    BlurQuery bq1 = new BlurQuery();
+    BlurQuery bq2 = new BlurQuery();
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, 
table, new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, 
table, new int[] { 0, 2 }, bq2,
+        BlurQuery.class);
+
+    assertFalse(key1.equals(key2));
+    assertFalse(key1.hashCode() == key2.hashCode());
+  }
+
+  @Test
+  public void test6() throws BlurException {
     User user1 = new User("test1", map("a", "b"));
     User user2 = new User("test1", map("a", "c"));
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, 
table, bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, 
table, bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, 
table, new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, 
table, new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertFalse(key1.equals(key2));
     assertFalse(key1.hashCode() == key2.hashCode());
   }
 
   @Test
-  public void test7() {
+  public void test7() throws BlurException {
     User user1 = new User("test1", map("a", "b"));
     User user2 = new User("test2", map("a", "b"));
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, 
table, bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, 
table, bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, 
table, new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, 
table, new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertFalse(key1.equals(key2));
     assertFalse(key1.hashCode() == key2.hashCode());
@@ -132,4 +166,12 @@ public class ThriftCacheKeyTest {
     }
     return map;
   }
+
+  public static SortedSet<String> getShards(String... shards) {
+    SortedSet<String> set = new TreeSet<String>();
+    for (String s : shards) {
+      set.add(s);
+    }
+    return set;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
 
b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
index 33accc3..587f1fb 100644
--- 
a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
+++ 
b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
@@ -22,11 +22,15 @@ import static org.junit.Assert.assertFalse;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
 
 import org.apache.blur.BlurConfiguration;
+import org.apache.blur.manager.IndexServer;
+import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.Arguments;
 import org.apache.blur.thrift.generated.Blur.Iface;
@@ -65,7 +69,7 @@ public class ThriftCacheServerTest {
   public void setup() throws IOException {
     _configuration = new BlurConfiguration();
     _thriftCache = new ThriftCache(10000);
-    _thriftCacheServer = new ThriftCacheServer(_configuration, getMock(), 
_thriftCache);
+    _thriftCacheServer = new ThriftCacheServer(_configuration, getMock(), 
getMockIndexServer(), _thriftCache);
     _table = "t";
   }
 
@@ -507,4 +511,52 @@ public class ThriftCacheServerTest {
       }
     };
   }
+
+  private IndexServer getMockIndexServer() {
+    return new IndexServer() {
+
+      @Override
+      public long getTableSize(String table) throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public Map<String, ShardState> getShardState(String table) {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public SortedSet<String> getShardListCurrentServerOnly(String table) 
throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public long getRowCount(String table) throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public long getRecordCount(String table) throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public String getNodeName() {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public Map<String, BlurIndex> getIndexes(String table) throws 
IOException {
+        Map<String, BlurIndex> map = new HashMap<String, BlurIndex>();
+        map.put("shard-000000", null);
+        map.put("shard-000001", null);
+        return map;
+      }
+
+      @Override
+      public void close() throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+    };
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java 
b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
index da96ba5..6fe8e8c 100644
--- a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
+++ b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
@@ -33,7 +33,7 @@ public class ThriftCacheTest {
   public void test1() throws BlurException {
     ThriftCache thriftCache = new ThriftCache(10000);
     BlurQuery blurQuery = new BlurQuery();
-    ThriftCacheKey<?> key = thriftCache.getKey("t", blurQuery, 
BlurQuery.class);
+    ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, 
blurQuery, BlurQuery.class);
     assertNull(thriftCache.get(key, BlurResults.class));
     BlurResults value = new BlurResults();
     // assert same instance
@@ -46,7 +46,7 @@ public class ThriftCacheTest {
   @Test
   public void test2() throws BlurException {
     ThriftCache thriftCache = new ThriftCache(10000);
-    ThriftCacheKey<?> key = thriftCache.getKey("t", null, TableStats.class);
+    ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, null, 
TableStats.class);
     assertNull(thriftCache.get(key, TableStats.class));
     TableStats value = new TableStats();
     // assert same instance
@@ -64,7 +64,7 @@ public class ThriftCacheTest {
       BlurQuery blurQuery = new BlurQuery();
       // just make keys different
       blurQuery.fetch = i;
-      ThriftCacheKey<?> key = thriftCache.getKey("t", blurQuery, 
BlurQuery.class);
+      ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, 
blurQuery, BlurQuery.class);
       assertNull(thriftCache.get(key, BlurResults.class));
       BlurResults value = new BlurResults();
       // assert same instance

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
----------------------------------------------------------------------
diff --git 
a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java 
b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
index dd95513..335ae21 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
@@ -792,7 +792,6 @@ public abstract class BlurClusterTestBase {
     loadTable(tableName);
     Iface client = getClient();
     BlurQuery blurQuery = new BlurQuery();
-    blurQuery.setUseCacheIfPresent(false);
     Query query = new Query();
     query.setQuery("test.test:value");
     blurQuery.setQuery(query);

Reply via email to