Repository: incubator-blur Updated Branches: refs/heads/master e0a6a106e -> 5f16374ef
Fixing issues with cache when the layout of the cluster would change due to a shard server failure. The cluster test was flawed because it disabled the use of cached results. Also improved the error message when the controller can not complete a query due to shard server errors. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5f16374e Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5f16374e Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5f16374e Branch: refs/heads/master Commit: 5f16374efdbdd1cb3c62a167a10e4c859a04af1b Parents: e0a6a10 Author: Aaron McCurry <[email protected]> Authored: Sun Mar 1 13:56:10 2015 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sun Mar 1 13:56:10 2015 -0500 ---------------------------------------------------------------------- .../org/apache/blur/server/cache/ClassObj.java | 51 ++++++++++++ .../apache/blur/server/cache/ThriftCache.java | 38 ++++++--- .../blur/server/cache/ThriftCacheKey.java | 54 ++++++++---- .../blur/server/cache/ThriftCacheServer.java | 48 ++++++++--- .../blur/server/cache/ThriftCacheValue.java | 35 +++++++- .../blur/thrift/BlurControllerServer.java | 5 +- .../thrift/QueryCoundNotBeCompletedReport.java | 81 ++++++++++++++++++ .../blur/thrift/ThriftBlurShardServer.java | 2 +- .../blur/server/cache/ThriftCacheKeyTest.java | 86 +++++++++++++++----- .../server/cache/ThriftCacheServerTest.java | 54 +++++++++++- .../blur/server/cache/ThriftCacheTest.java | 6 +- .../apache/blur/thrift/BlurClusterTestBase.java | 1 - 12 files changed, 393 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java b/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java new file mode 100644 index 0000000..eabcc7b --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.server.cache; + +public class ClassObj<T> { + + private final Class<T> _clazz; + + public ClassObj(Class<T> clazz) { + if (clazz == null) { + throw new RuntimeException("Class cannot be null."); + } + _clazz = clazz; + } + + public Class<T> getClazz() { + return _clazz; + } + + @Override + public int hashCode() { + return _clazz.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ClassObj<?> other = (ClassObj<?>) obj; + return _clazz == other._clazz; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java index e4b428c..7093a6a 100644 --- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java +++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java @@ -25,6 +25,8 @@ import static org.apache.blur.metrics.MetricsConstants.THRIFT_CACHE; import java.util.Iterator; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -36,8 +38,8 @@ import org.apache.blur.user.User; import org.apache.blur.user.UserContext; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import com.googlecode.concurrentlinkedhashmap.EntryWeigher; import com.googlecode.concurrentlinkedhashmap.EvictionListener; -import com.googlecode.concurrentlinkedhashmap.Weigher; import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Meter; import com.yammer.metrics.core.MetricName; @@ -47,6 +49,7 @@ public class ThriftCache { private static final Log LOG = LogFactory.getLog(ThriftCache.class); private final ConcurrentLinkedHashMap<ThriftCacheKey<?>, ThriftCacheValue<?>> _cacheMap; + private final ConcurrentMap<String, Long> _lastModTimestamps = new ConcurrentHashMap<String, Long>(); private final Meter _hits; private final Meter _misses; private final Meter _evictions; @@ -59,10 +62,10 @@ public class ThriftCache { _misses = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, THRIFT_CACHE, MISS), MISS, TimeUnit.SECONDS); _evictions = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, THRIFT_CACHE, EVICTION), EVICTION, TimeUnit.SECONDS); _cacheMap = new ConcurrentLinkedHashMap.Builder<ThriftCacheKey<?>, ThriftCacheValue<?>>() - .weigher(new Weigher<ThriftCacheValue<?>>() { + .weigher(new EntryWeigher<ThriftCacheKey<?>, ThriftCacheValue<?>>() { @Override - public int weightOf(ThriftCacheValue<?> value) { - return value.size(); + public int weightOf(ThriftCacheKey<?> key, ThriftCacheValue<?> value) { + return key.size() + value.size(); } }).listener(new EvictionListener<ThriftCacheKey<?>, ThriftCacheValue<?>>() { @Override @@ -77,6 +80,16 @@ public class ThriftCache { } public <K extends TBase<?, ?>, V extends TBase<?, ?>> V put(ThriftCacheKey<K> key, V t) throws BlurException { + synchronized (_lastModTimestamps) { + Long lastModTimestamp = _lastModTimestamps.get(key.getTable()); + if (lastModTimestamp != null && key.getTimestamp() < lastModTimestamp) { + // This means that the key was created before the index was modified. So + // do not cache the value because it's already out of date with the + // index. + return t; + } + } + LOG.debug("Inserting into cache [{0}] with key [{1}]", t, key); _cacheMap.put(key, new ThriftCacheValue<V>(t)); return t; } @@ -97,17 +110,15 @@ public class ThriftCache { return value.getValue(clazz); } - public <K extends TBase<?, ?>> ThriftCacheKey<K> getKey(String table, K tkey, Class<K> clazz) { + public <K extends TBase<?, ?>> ThriftCacheKey<K> getKey(String table, int[] shards, K tkey, Class<K> clazz) throws BlurException { User user = UserContext.getUser(); - return new ThriftCacheKey<K>(user, table, tkey, clazz); - } - - public void clear() { - LOG.info("Clearing all cache."); - _cacheMap.clear(); + return new ThriftCacheKey<K>(user, table, shards, tkey, clazz); } public void clearTable(String table) { + synchronized (_lastModTimestamps) { + _lastModTimestamps.put(table, System.nanoTime()); + } LOG.info("Clearing cache for table [{0}]", table); Set<Entry<ThriftCacheKey<?>, ThriftCacheValue<?>>> entrySet = _cacheMap.entrySet(); Iterator<Entry<ThriftCacheKey<?>, ThriftCacheValue<?>>> iterator = entrySet.iterator(); @@ -119,6 +130,11 @@ public class ThriftCache { } } + public void clear() { + LOG.info("Clearing all cache."); + _cacheMap.clear(); + } + public long size() { return _cacheMap.weightedSize(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java index 790533b..5866966 100644 --- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java +++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java @@ -16,10 +16,12 @@ */ package org.apache.blur.server.cache; +import java.util.Arrays; import java.util.Map; import java.util.TreeMap; import org.apache.blur.thirdparty.thrift_0_9_0.TBase; +import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.user.User; public class ThriftCacheKey<T extends TBase<?, ?>> { @@ -27,11 +29,16 @@ public class ThriftCacheKey<T extends TBase<?, ?>> { private final String _username; private final Map<String, String> _attributes; private final String _table; - private final T _t; - private final String _clazz; + private final int[] _shards; + private final ClassObj<T> _clazz; + private final ThriftCacheValue<T> _key; - public ThriftCacheKey(User user, String table, T t, Class<T> clazz) { - _clazz = clazz.getName(); + // not apart of the key for equally + private final transient long _timestamp; + + public ThriftCacheKey(User user, String table, int[] shards, T t, Class<T> clazz) throws BlurException { + _timestamp = System.nanoTime(); + _clazz = new ClassObj<T>(clazz); if (user != null) { _username = user.getUsername(); Map<String, String> attributes = user.getAttributes(); @@ -45,7 +52,12 @@ public class ThriftCacheKey<T extends TBase<?, ?>> { _attributes = null; } _table = table; - _t = t; + _shards = shards; + _key = new ThriftCacheValue<T>(t); + } + + public long getTimestamp() { + return _timestamp; } public String getTable() { @@ -53,12 +65,28 @@ public class ThriftCacheKey<T extends TBase<?, ?>> { } @Override + public String toString() { + try { + return "ThriftCacheKey [_username=" + _username + ", _attributes=" + _attributes + ", _table=" + _table + + ", _shards=" + Arrays.toString(_shards) + ", _clazz=" + _clazz + ", _key=" + + _key.getValue(_clazz.getClazz()) + ", _timestamp=" + _timestamp + "]"; + } catch (BlurException e) { + throw new RuntimeException(e); + } + } + + public int size() { + return _key.size(); + } + + @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((_attributes == null) ? 0 : _attributes.hashCode()); result = prime * result + ((_clazz == null) ? 0 : _clazz.hashCode()); - result = prime * result + ((_t == null) ? 0 : _t.hashCode()); + result = prime * result + ((_key == null) ? 0 : _key.hashCode()); + result = prime * result + Arrays.hashCode(_shards); result = prime * result + ((_table == null) ? 0 : _table.hashCode()); result = prime * result + ((_username == null) ? 0 : _username.hashCode()); return result; @@ -83,10 +111,12 @@ public class ThriftCacheKey<T extends TBase<?, ?>> { return false; } else if (!_clazz.equals(other._clazz)) return false; - if (_t == null) { - if (other._t != null) + if (_key == null) { + if (other._key != null) return false; - } else if (!_t.equals(other._t)) + } else if (!_key.equals(other._key)) + return false; + if (!Arrays.equals(_shards, other._shards)) return false; if (_table == null) { if (other._table != null) @@ -101,10 +131,4 @@ public class ThriftCacheKey<T extends TBase<?, ?>> { return true; } - @Override - public String toString() { - return "ThriftCacheKey [_username=" + _username + ", _attributes=" + _attributes + ", _table=" + _table + ", _t=" - + _t + ", _clazz=" + _clazz + "]"; - } - } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java index dbd395f..7264801 100644 --- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java +++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java @@ -16,15 +16,20 @@ */ package org.apache.blur.server.cache; +import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import org.apache.blur.BlurConfiguration; +import org.apache.blur.manager.IndexServer; import org.apache.blur.server.FilteredBlurServer; import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.BException; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; @@ -32,19 +37,23 @@ import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.TableStats; +import org.apache.blur.utils.ShardUtil; public class ThriftCacheServer extends FilteredBlurServer { private final ThriftCache _thriftCache; + private final IndexServer _indexServer; - public ThriftCacheServer(BlurConfiguration configuration, Iface iface, ThriftCache thriftCache) { + public ThriftCacheServer(BlurConfiguration configuration, Iface iface, IndexServer indexServer, + ThriftCache thriftCache) { super(configuration, iface, true); _thriftCache = thriftCache; + _indexServer = indexServer; } @Override public TableStats tableStats(String table) throws BlurException, TException { - ThriftCacheKey<TableStats> key = _thriftCache.getKey(table, null, TableStats.class); + ThriftCacheKey<TableStats> key = _thriftCache.getKey(table, getShards(table), null, TableStats.class); TableStats results = _thriftCache.get(key, TableStats.class); if (results != null) { return results; @@ -52,6 +61,22 @@ public class ThriftCacheServer extends FilteredBlurServer { return _thriftCache.put(key, super.tableStats(table)); } + private int[] getShards(String table) throws BlurException { + try { + Set<String> keySet = _indexServer.getIndexes(table).keySet(); + int[] shards = new int[keySet.size()]; + int i = 0; + for (String s : keySet) { + int shardIndex = ShardUtil.getShardIndex(s); + shards[i++] = shardIndex; + } + Arrays.sort(shards); + return shards; + } catch (IOException e) { + throw new BException("Unknown error while trying to get current shards for table [{0}]", e, table); + } + } + @Override public BlurResults query(String table, BlurQuery blurQuery) throws BlurException, TException { boolean useCacheIfPresent = blurQuery.isUseCacheIfPresent(); @@ -66,14 +91,16 @@ public class ThriftCacheServer extends FilteredBlurServer { copy.userContext = null; copy.cacheResult = false; copy.startTime = 0; - ThriftCacheKey<BlurQuery> key = _thriftCache.getKey(table, copy, BlurQuery.class); - BlurResults results = _thriftCache.get(key, BlurResults.class); - if (results != null && useCacheIfPresent) { - return results; + ThriftCacheKey<BlurQuery> key = _thriftCache.getKey(table, getShards(table), copy, BlurQuery.class); + if (useCacheIfPresent) { + BlurResults results = _thriftCache.get(key, BlurResults.class); + if (results != null) { + return results; + } } BlurResults blurResults = super.query(table, blurQuery); if (cacheResult) { - return _thriftCache.put(key, blurResults); + return _thriftCache.put(key, blurResults); } return blurResults; } @@ -81,7 +108,7 @@ public class ThriftCacheServer extends FilteredBlurServer { @Override public FetchResult fetchRow(String table, Selector selector) throws BlurException, TException { Selector copy = new Selector(selector); - ThriftCacheKey<Selector> key = _thriftCache.getKey(table, copy, Selector.class); + ThriftCacheKey<Selector> key = _thriftCache.getKey(table, getShards(table), copy, Selector.class); FetchResult results = _thriftCache.get(key, FetchResult.class); if (results != null) { return results; @@ -101,7 +128,7 @@ public class ThriftCacheServer extends FilteredBlurServer { for (int i = 0; i < selectors.size(); i++) { Selector selector = selectors.get(i); Selector copy = new Selector(selector); - ThriftCacheKey<Selector> key = _thriftCache.getKey(table, copy, Selector.class); + ThriftCacheKey<Selector> key = _thriftCache.getKey(table, getShards(table), copy, Selector.class); FetchResult fetchResult = _thriftCache.get(key, FetchResult.class); if (fetchResult != null) { resultMap.put(i, fetchResult); @@ -117,7 +144,8 @@ public class ThriftCacheServer extends FilteredBlurServer { for (int i = 0; i < missingResults.size(); i++) { Selector selector = selectorRequest.get(i); FetchResult fetchResult = missingResults.get(i); - ThriftCacheKey<Selector> key = _thriftCache.getKey(table, new Selector(selector), Selector.class); + ThriftCacheKey<Selector> key = _thriftCache.getKey(table, getShards(table), new Selector(selector), + Selector.class); _thriftCache.put(key, fetchResult); int originalIndex = requestMapping.get(i); resultMap.put(originalIndex, fetchResult); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java index 7d259f4..84d5606 100644 --- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java +++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java @@ -16,6 +16,8 @@ */ package org.apache.blur.server.cache; +import java.util.Arrays; + import org.apache.blur.thirdparty.thrift_0_9_0.TBase; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TCompactProtocol; @@ -29,8 +31,12 @@ public class ThriftCacheValue<T extends TBase<?, ?>> { private final byte[] _data; public ThriftCacheValue(T t) throws BlurException { + _data = toBytes(t); + } + + public static <T extends TBase<?, ?>> byte[] toBytes(T t) throws BException { if (t == null) { - _data = null; + return null; } else { TMemoryBuffer transport = new TMemoryBuffer(1024); try { @@ -38,11 +44,11 @@ public class ThriftCacheValue<T extends TBase<?, ?>> { } catch (TException e) { throw new BException("Unknown error while trying to read from cache.", e); } - _data = trim(transport.getArray(), transport.length()); + return trim(transport.getArray(), transport.length()); } } - private byte[] trim(byte[] bs, int len) { + public static byte[] trim(byte[] bs, int len) { if (bs.length == len) { return bs; } @@ -75,4 +81,27 @@ public class ThriftCacheValue<T extends TBase<?, ?>> { throw new BException("Unknown error while trying to read from cache.", e); } } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(_data); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ThriftCacheValue<?> other = (ThriftCacheValue<?>) obj; + if (!Arrays.equals(_data, other._data)) + return false; + return true; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index a29867b..4c45a90 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -475,6 +475,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { BlurUtil.setStartTime(blurQuery); + QueryCoundNotBeCompletedReport report = null; OUTER: for (int retries = 0; retries < _maxDefaultRetries; retries++) { Tracer selectorTrace = Trace.trace("selector - setup", Trace.param("retries", retries)); final AtomicLongArray facetCounts = BlurUtil.getAtomicLongArraySameLengthAsList(blurQuery.facets); @@ -534,6 +535,8 @@ public class BlurControllerServer extends TableAdmin implements Iface { if (!validResults(results, shardCount, blurQuery)) { BlurClientManager.sleep(_defaultDelay, _maxDefaultDelay, retries, _maxDefaultRetries); Map<String, String> map = getTableLayout(table); + Map<String, Long> shardInfo = results.getShardInfo(); + report = new QueryCoundNotBeCompletedReport(map, shardCount, shardInfo, blurQuery); LOG.info("Current layout for table [{0}] is [{1}]", table, map); continue OUTER; } @@ -544,7 +547,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { } } } - throw new BException("Query could not be completed."); + throw new BException("Query could not be completed, report [{0}].", report); } catch (Exception e) { LOG.error("Unknown error during search of [table={0},blurQuery={1}]", e, table, blurQuery); if (e instanceof BlurException) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java b/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java new file mode 100644 index 0000000..0c62724 --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.thrift; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.blur.thrift.generated.BlurQuery; +import org.json.JSONException; +import org.json.JSONObject; + +public class QueryCoundNotBeCompletedReport { + + private final String _message; + + public QueryCoundNotBeCompletedReport(Map<String, String> tableLayout, int shardCount, Map<String, Long> shardInfo, + BlurQuery blurQuery) throws JSONException { + String missingShardsFromResult = "\n"; + if (shardCount != shardInfo.size()) { + + Set<String> missingShardResults = new TreeSet<String>(); + Set<String> servers = new TreeSet<String>(); + for (Entry<String, String> e : tableLayout.entrySet()) { + String shard = e.getKey(); + String server = e.getValue(); + if (!shardInfo.containsKey(shard)) { + missingShardResults.add(shard); + servers.add(server); + } + } + missingShardsFromResult = "\nThere appears to be some missing shards from the results.\n"; + missingShardsFromResult += "Missing Shards:\n\t" + missingShardResults + "\n"; + missingShardsFromResult += "From Servers:\n\t" + servers + "\n"; + } + + JSONObject jsonObject = new JSONObject(); + jsonObject.put("tableLayout", getTableLayout(tableLayout)); + jsonObject.put("tableShardCount", shardCount); + jsonObject.put("queryShardCount", shardInfo.size()); + jsonObject.put("queryResultCount", getShardInfo(shardInfo)); + _message = missingShardsFromResult + jsonObject.toString(1); + } + + private JSONObject getShardInfo(Map<String, Long> shardInfo) throws JSONException { + JSONObject j = new JSONObject(); + for (Entry<String, Long> e : shardInfo.entrySet()) { + j.put(e.getKey(), e.getValue()); + } + return j; + } + + private JSONObject getTableLayout(Map<String, String> tableLayout) throws JSONException { + JSONObject j = new JSONObject(); + for (Entry<String, String> e : tableLayout.entrySet()) { + j.put(e.getKey(), e.getValue()); + } + return j; + } + + @Override + public String toString() { + return _message; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 545bf48..d8f15b0 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -289,7 +289,7 @@ public class ThriftBlurShardServer extends ThriftServer { List<ServerSecurityFilter> serverSecurity = getServerSecurityList(configuration, ServerSecurityFilterFactory.ServerType.SHARD); - Iface iface = new ThriftCacheServer(configuration, shardServer, thriftCache); + Iface iface = new ThriftCacheServer(configuration, shardServer, indexServer, thriftCache); iface = BlurUtil.wrapFilteredBlurServer(configuration, iface, true); iface = ServerSecurityUtil.applySecurity(iface, serverSecurity, true); iface = BlurUtil.recordMethodCallsAndAverageTimes(iface, Iface.class, false); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java index 0ca81a6..8c5da28 100644 --- a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java +++ b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java @@ -16,11 +16,15 @@ */ package org.apache.blur.server.cache; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import java.util.HashMap; import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.user.User; import org.junit.Test; @@ -28,92 +32,122 @@ import org.junit.Test; public class ThriftCacheKeyTest { @Test - public void test1() { + public void test1() throws BlurException { User user = null; String table = "t"; BlurQuery bq1 = new BlurQuery(); BlurQuery bq2 = new BlurQuery(); - ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, bq1, BlurQuery.class); - ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, bq2, BlurQuery.class); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, bq1, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, bq2, + BlurQuery.class); assertEquals(key1, key2); assertEquals(key1.hashCode(), key2.hashCode()); } @Test - public void test2() { + public void test2() throws BlurException { User user = null; String table = "t"; - ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, null, BlurQuery.class); - ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, null, BlurQuery.class); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, null, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, null, + BlurQuery.class); assertEquals(key1, key2); assertEquals(key1.hashCode(), key2.hashCode()); } @Test - public void test3() { + public void test3() throws BlurException { User user = new User("test", null); String table = "t"; BlurQuery bq1 = new BlurQuery(); BlurQuery bq2 = new BlurQuery(); - ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, bq1, BlurQuery.class); - ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, bq2, BlurQuery.class); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, bq1, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, bq2, + BlurQuery.class); assertEquals(key1, key2); assertEquals(key1.hashCode(), key2.hashCode()); } @Test - public void test4() { + public void test4() throws BlurException { User user = new User("test", map("a", "b")); String table = "t"; BlurQuery bq1 = new BlurQuery(); BlurQuery bq2 = new BlurQuery(); - ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, bq1, BlurQuery.class); - ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, bq2, BlurQuery.class); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, bq1, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table, new int[] { 0, 1 }, bq2, + BlurQuery.class); assertEquals(key1, key2); assertEquals(key1.hashCode(), key2.hashCode()); } @Test - public void test5() { + public void test5a() throws BlurException { User user1 = new User("test1", null); User user2 = new User("test2", null); String table = "t"; BlurQuery bq1 = new BlurQuery(); BlurQuery bq2 = new BlurQuery(); - ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table, bq1, BlurQuery.class); - ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table, bq2, BlurQuery.class); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table, new int[] { 0, 1 }, bq1, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table, new int[] { 0, 1 }, bq2, + BlurQuery.class); assertFalse(key1.equals(key2)); assertFalse(key1.hashCode() == key2.hashCode()); } @Test - public void test6() { + public void test5b() throws BlurException { + User user1 = new User("test1", null); + User user2 = new User("test1", null); + String table = "t"; + BlurQuery bq1 = new BlurQuery(); + BlurQuery bq2 = new BlurQuery(); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table, new int[] { 0, 1 }, bq1, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table, new int[] { 0, 2 }, bq2, + BlurQuery.class); + + assertFalse(key1.equals(key2)); + assertFalse(key1.hashCode() == key2.hashCode()); + } + + @Test + public void test6() throws BlurException { User user1 = new User("test1", map("a", "b")); User user2 = new User("test1", map("a", "c")); String table = "t"; BlurQuery bq1 = new BlurQuery(); BlurQuery bq2 = new BlurQuery(); - ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table, bq1, BlurQuery.class); - ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table, bq2, BlurQuery.class); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table, new int[] { 0, 1 }, bq1, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table, new int[] { 0, 1 }, bq2, + BlurQuery.class); assertFalse(key1.equals(key2)); assertFalse(key1.hashCode() == key2.hashCode()); } @Test - public void test7() { + public void test7() throws BlurException { User user1 = new User("test1", map("a", "b")); User user2 = new User("test2", map("a", "b")); String table = "t"; BlurQuery bq1 = new BlurQuery(); BlurQuery bq2 = new BlurQuery(); - ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table, bq1, BlurQuery.class); - ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table, bq2, BlurQuery.class); + ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table, new int[] { 0, 1 }, bq1, + BlurQuery.class); + ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table, new int[] { 0, 1 }, bq2, + BlurQuery.class); assertFalse(key1.equals(key2)); assertFalse(key1.hashCode() == key2.hashCode()); @@ -132,4 +166,12 @@ public class ThriftCacheKeyTest { } return map; } + + public static SortedSet<String> getShards(String... shards) { + SortedSet<String> set = new TreeSet<String>(); + for (String s : shards) { + set.add(s); + } + return set; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java index 33accc3..587f1fb 100644 --- a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java +++ b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java @@ -22,11 +22,15 @@ import static org.junit.Assert.assertFalse; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; import org.apache.blur.BlurConfiguration; +import org.apache.blur.manager.IndexServer; +import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Arguments; import org.apache.blur.thrift.generated.Blur.Iface; @@ -65,7 +69,7 @@ public class ThriftCacheServerTest { public void setup() throws IOException { _configuration = new BlurConfiguration(); _thriftCache = new ThriftCache(10000); - _thriftCacheServer = new ThriftCacheServer(_configuration, getMock(), _thriftCache); + _thriftCacheServer = new ThriftCacheServer(_configuration, getMock(), getMockIndexServer(), _thriftCache); _table = "t"; } @@ -507,4 +511,52 @@ public class ThriftCacheServerTest { } }; } + + private IndexServer getMockIndexServer() { + return new IndexServer() { + + @Override + public long getTableSize(String table) throws IOException { + throw new RuntimeException("Not implemented."); + } + + @Override + public Map<String, ShardState> getShardState(String table) { + throw new RuntimeException("Not implemented."); + } + + @Override + public SortedSet<String> getShardListCurrentServerOnly(String table) throws IOException { + throw new RuntimeException("Not implemented."); + } + + @Override + public long getRowCount(String table) throws IOException { + throw new RuntimeException("Not implemented."); + } + + @Override + public long getRecordCount(String table) throws IOException { + throw new RuntimeException("Not implemented."); + } + + @Override + public String getNodeName() { + throw new RuntimeException("Not implemented."); + } + + @Override + public Map<String, BlurIndex> getIndexes(String table) throws IOException { + Map<String, BlurIndex> map = new HashMap<String, BlurIndex>(); + map.put("shard-000000", null); + map.put("shard-000001", null); + return map; + } + + @Override + public void close() throws IOException { + throw new RuntimeException("Not implemented."); + } + }; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java index da96ba5..6fe8e8c 100644 --- a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java +++ b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java @@ -33,7 +33,7 @@ public class ThriftCacheTest { public void test1() throws BlurException { ThriftCache thriftCache = new ThriftCache(10000); BlurQuery blurQuery = new BlurQuery(); - ThriftCacheKey<?> key = thriftCache.getKey("t", blurQuery, BlurQuery.class); + ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, blurQuery, BlurQuery.class); assertNull(thriftCache.get(key, BlurResults.class)); BlurResults value = new BlurResults(); // assert same instance @@ -46,7 +46,7 @@ public class ThriftCacheTest { @Test public void test2() throws BlurException { ThriftCache thriftCache = new ThriftCache(10000); - ThriftCacheKey<?> key = thriftCache.getKey("t", null, TableStats.class); + ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, null, TableStats.class); assertNull(thriftCache.get(key, TableStats.class)); TableStats value = new TableStats(); // assert same instance @@ -64,7 +64,7 @@ public class ThriftCacheTest { BlurQuery blurQuery = new BlurQuery(); // just make keys different blurQuery.fetch = i; - ThriftCacheKey<?> key = thriftCache.getKey("t", blurQuery, BlurQuery.class); + ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, blurQuery, BlurQuery.class); assertNull(thriftCache.get(key, BlurResults.class)); BlurResults value = new BlurResults(); // assert same instance http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java index dd95513..335ae21 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java @@ -792,7 +792,6 @@ public abstract class BlurClusterTestBase { loadTable(tableName); Iface client = getClient(); BlurQuery blurQuery = new BlurQuery(); - blurQuery.setUseCacheIfPresent(false); Query query = new Query(); query.setQuery("test.test:value"); blurQuery.setQuery(query);
