Updated Branches: refs/heads/master 387861869 -> 32e4ada09
Cleaning up old code, removing duplicate functionality Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/32e4ada0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/32e4ada0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/32e4ada0 Branch: refs/heads/master Commit: 32e4ada09fc4acd4028835ae9a56528ea74954b9 Parents: 3878618 Author: Aaron McCurry <[email protected]> Authored: Tue Oct 8 07:53:53 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Oct 8 07:54:46 2013 -0400 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 13 ++-- .../org/apache/blur/manager/IndexServer.java | 44 -------------- .../indexserver/AbstractIndexServer.java | 2 + .../indexserver/DistributedIndexServer.java | 63 +++----------------- .../manager/indexserver/LocalIndexServer.java | 24 +------- 5 files changed, 18 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/32e4ada0/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 2aff9fd..728fdc8 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -110,6 +110,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -402,7 +403,7 @@ public class IndexManager { Query facetedQuery = getFacetedQuery(blurQuery, userQuery, facetedCounts, fieldManager, context, postFilter, preFilter); call = new SimpleQueryParallelCall(running, table, status, _indexServer, facetedQuery, blurQuery.selector, - _queriesInternalMeter, shardServerContext, runSlow, _fetchCount, _maxHeapPerRowFetch); + _queriesInternalMeter, shardServerContext, runSlow, _fetchCount, _maxHeapPerRowFetch, context.getSimilarity()); MergerBlurResultIterable merger = new MergerBlurResultIterable(blurQuery); return ForkJoin.execute(_executor, blurIndexes.entrySet(), call, new Cancel() { @Override @@ -1048,7 +1049,7 @@ public class IndexManager { } private int getNumberOfShards(String table) { - return _indexServer.getShardCount(table); + return getTableContext(table).getDescriptor().getShardCount(); } static class SimpleQueryParallelCall implements ParallelCall<Entry<String, BlurIndex>, BlurResultIterable> { @@ -1064,10 +1065,11 @@ public class IndexManager { private final boolean _runSlow; private final int _fetchCount; private final int _maxHeapPerRowFetch; + private final Similarity _similarity; public SimpleQueryParallelCall(AtomicBoolean running, String table, QueryStatus status, IndexServer indexServer, Query query, Selector selector, Meter queriesInternalMeter, ShardServerContext shardServerContext, - boolean runSlow, int fetchCount, int maxHeapPerRowFetch) { + boolean runSlow, int fetchCount, int maxHeapPerRowFetch, Similarity similarity) { _running = running; _table = table; _status = status; @@ -1079,6 +1081,7 @@ public class IndexManager { _runSlow = runSlow; _fetchCount = fetchCount; _maxHeapPerRowFetch = maxHeapPerRowFetch; + _similarity = similarity; } @Override @@ -1098,7 +1101,7 @@ public class IndexManager { if (_shardServerContext != null) { _shardServerContext.setIndexSearcherClosable(_table, shard, searcher); } - searcher.setSimilarity(_indexServer.getSimilarity(_table)); + searcher.setSimilarity(_similarity); Query rewrite; try { rewrite = searcher.rewrite((Query) _query.clone()); @@ -1137,7 +1140,7 @@ public class IndexManager { public void setMutateThreadCount(int mutateThreadCount) { _mutateThreadCount = mutateThreadCount; } - + public void setFilterCache(BlurFilterCache filterCache) { _filterCache = filterCache; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/32e4ada0/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java index 7147c53..4d04a89 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java @@ -23,7 +23,6 @@ import java.util.SortedSet; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.thrift.generated.ShardState; -import org.apache.lucene.search.similarities.Similarity; /** * The {@link IndexServer} interface provides the internal API to interact with @@ -31,13 +30,6 @@ import org.apache.lucene.search.similarities.Similarity; */ public interface IndexServer { - /** - * Enum that describes whether a table is enabled or not. - */ - public enum TABLE_STATUS { - ENABLED, DISABLED - } - // Server state /** @@ -73,48 +65,12 @@ public interface IndexServer { List<String> getShardList(String table); /** - * Gets the similarity object used by lucene for this table. - * - * @param table - * the table name. - * @return the similarity object. - */ - Similarity getSimilarity(String table); - - /** - * Gets the status of the table. - * - * @param table - * the table name. - * @return the status. - */ - TABLE_STATUS getTableStatus(String table); - - /** * Gets the current nodes name. * * @return */ String getNodeName(); - /** - * Gets the table uri. (hdfs://cluster1:9000/blur/tables/tablename1234) - * - * @param table - * the table name - * @return the uri to the table directory that contains all the shards.. - */ - String getTableUri(String table); - - /** - * Gets the shard count for the given table. - * - * @param table - * the name of the table. - * @return - */ - int getShardCount(String table); - // Metrics /** http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/32e4ada0/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java index e183a0b..50f7df8 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java @@ -24,6 +24,7 @@ import org.apache.blur.manager.writer.BlurIndex; public abstract class AbstractIndexServer implements IndexServer { + @Override public long getRecordCount(String table) throws IOException { long recordCount = 0; Map<String, BlurIndex> indexes = getIndexes(table); @@ -33,6 +34,7 @@ public abstract class AbstractIndexServer implements IndexServer { return recordCount; } + @Override public long getRowCount(String table) throws IOException { long rowCount = 0; Map<String, BlurIndex> indexes = getIndexes(table); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/32e4ada0/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index ff3ce79..7c75234 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -47,7 +47,6 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.blur.concurrent.Executors; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.lucene.search.FairSimilarity; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.store.refcounter.IndexInputCloser; import org.apache.blur.manager.BlurFilterCache; @@ -71,7 +70,6 @@ import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurUtil; import org.apache.blur.zookeeper.WatchChildren; import org.apache.blur.zookeeper.WatchChildren.OnChange; -import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; @@ -545,8 +543,7 @@ public class DistributedIndexServer extends AbstractIndexServer { } private Set<String> getShardsToServe(String table) { - TABLE_STATUS tableStatus = getTableStatus(table); - if (tableStatus == TABLE_STATUS.DISABLED) { + if (!isEnabled(table)) { return new HashSet<String>(); } DistributedLayout layoutManager = _layoutManagers.get(table); @@ -593,13 +590,6 @@ public class DistributedIndexServer extends AbstractIndexServer { } @Override - public int getShardCount(String table) { - checkTable(table); - TableDescriptor descriptor = getTableDescriptor(table); - return descriptor.shardCount; - } - - @Override public List<String> getShardList(String table) { checkTable(table); List<String> result = new ArrayList<String>(); @@ -626,40 +616,23 @@ public class DistributedIndexServer extends AbstractIndexServer { } } - @Override - public Similarity getSimilarity(String table) { - checkTable(table); - Similarity similarity = _tableSimilarity.get(table); - if (similarity == null) { - TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, _cluster, table); - String similarityClass = tableDescriptor.similarityClass; - if (similarityClass == null) { - similarity = new FairSimilarity(); - } else { - similarity = getInstance(similarityClass, Similarity.class); - } - _tableSimilarity.put(table, similarity); - } - return similarity; + private TableContext getTableContext(final String table) { + return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true, table), table)); } @Override public long getTableSize(String table) throws IOException { checkTable(table); - Path tablePath = new Path(getTableUri(table)); + String tableUri = getTableContext(table).getTablePath().toUri().toString(); + Path tablePath = new Path(tableUri); FileSystem fileSystem = FileSystem.get(tablePath.toUri(), _configuration); ContentSummary contentSummary = fileSystem.getContentSummary(tablePath); return contentSummary.getLength(); } - @Override - public TABLE_STATUS getTableStatus(String table) { + public boolean isEnabled(String table) { checkTable(table); - boolean enabled = _clusterStatus.isEnabled(true, _cluster, table); - if (enabled) { - return TABLE_STATUS.ENABLED; - } - return TABLE_STATUS.DISABLED; + return _clusterStatus.isEnabled(true, _cluster, table); } private void checkTable(String table) { @@ -669,13 +642,6 @@ public class DistributedIndexServer extends AbstractIndexServer { throw new RuntimeException("Table [" + table + "] does not exist."); } - @Override - public String getTableUri(String table) { - checkTable(table); - TableDescriptor descriptor = getTableDescriptor(table); - return descriptor.tableUri; - } - private TableDescriptor getTableDescriptor(String table) { TableDescriptor tableDescriptor = _tableDescriptors.get(table); if (tableDescriptor == null) { @@ -685,21 +651,6 @@ public class DistributedIndexServer extends AbstractIndexServer { return tableDescriptor; } - @SuppressWarnings("unchecked") - private <T> T getInstance(String className, Class<T> c) { - try { - Class<? extends T> clazz = (Class<? extends T>) Class.forName(className); - Object object = clazz.newInstance(); - if (object instanceof Configurable) { - Configurable configurable = (Configurable) object; - configurable.setConf(_configuration); - } - return (T) object; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - public void setClusterStatus(ClusterStatus clusterStatus) { _clusterStatus = clusterStatus; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/32e4ada0/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java index 2ffa20a..7075818 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java @@ -34,7 +34,6 @@ import java.util.concurrent.Executors; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.lucene.search.FairSimilarity; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.store.refcounter.IndexInputCloser; import org.apache.blur.manager.writer.BlurIndex; @@ -52,7 +51,6 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MMapDirectory; @@ -121,11 +119,6 @@ public class LocalIndexServer extends AbstractIndexServer { return tableMap; } - @Override - public Similarity getSimilarity(String table) { - return new FairSimilarity(); - } - private void close(Map<String, BlurIndex> map) { for (BlurIndex index : map.values()) { try { @@ -165,11 +158,6 @@ public class LocalIndexServer extends AbstractIndexServer { } @Override - public TABLE_STATUS getTableStatus(String table) { - return TABLE_STATUS.ENABLED; - } - - @Override public List<String> getShardList(String table) { try { List<String> result = new ArrayList<String>(); @@ -194,19 +182,9 @@ public class LocalIndexServer extends AbstractIndexServer { } @Override - public String getTableUri(String table) { - return _tableContext.getTablePath().toUri().toString(); - } - - @Override - public int getShardCount(String table) { - return _tableContext.getDescriptor().getShardCount(); - } - - @Override public long getTableSize(String table) throws IOException { try { - File file = new File(new URI(getTableUri(table))); + File file = new File(new URI(_tableContext.getTablePath().toUri().toString())); return getFolderSize(file); } catch (URISyntaxException e) { throw new IOException("bad URI", e);
