Updated Branches: refs/heads/apache-blur-0.2 16f98d06b -> 4e35bdd75
Adding a way to use the FilterCache object. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4e35bdd7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4e35bdd7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4e35bdd7 Branch: refs/heads/apache-blur-0.2 Commit: 4e35bdd751dc1b46c6f246211bc5fd442660b1e0 Parents: 16f98d0 Author: Aaron McCurry <[email protected]> Authored: Tue Oct 29 06:42:34 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Oct 29 06:42:34 2013 -0400 ---------------------------------------------------------------------- .../apache/blur/manager/BlurFilterCache.java | 12 +- .../blur/manager/DefaultBlurFilterCache.java | 163 ++++++++++++++++++- .../apache/blur/manager/QueryParserUtil.java | 29 +++- .../blur/thrift/ThriftBlurShardServer.java | 8 +- .../manager/DefaultBlurFilterCacheTest.java | 123 ++++++++++++++ .../org/apache/blur/filter/FilterCache.java | 4 + .../org/apache/blur/utils/BlurConstants.java | 2 + 7 files changed, 325 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4e35bdd7/blur-core/src/main/java/org/apache/blur/manager/BlurFilterCache.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/BlurFilterCache.java b/blur-core/src/main/java/org/apache/blur/manager/BlurFilterCache.java index 9c666b7..c23c6b3 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/BlurFilterCache.java +++ b/blur-core/src/main/java/org/apache/blur/manager/BlurFilterCache.java @@ -21,7 +21,9 @@ import org.apache.blur.lucene.search.SuperParser; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.Row; +import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; /** * The {@link BlurFilterCache} class provides the ability to cache pre and post @@ -30,6 +32,10 @@ import org.apache.lucene.search.Filter; */ public abstract class BlurFilterCache { + public interface FilterParser { + Query parse(String query) throws ParseException; + } + protected final BlurConfiguration _configuration; public BlurFilterCache(BlurConfiguration configuration) { @@ -70,8 +76,9 @@ public abstract class BlurFilterCache { * @param filterStr * the filter query string, should be used as a key. * @return the {@link Filter} that was parsed by the {@link SuperParser}. + * @throws ParseException */ - public abstract Filter storePreFilter(String table, String filterStr, Filter filter); + public abstract Filter storePreFilter(String table, String filterStr, Filter filter, FilterParser filterParser) throws ParseException; /** * The storePreFilter method stores the parsed post {@link Filter} (or @@ -83,8 +90,9 @@ public abstract class BlurFilterCache { * @param filterStr * the filter query string, should be used as a key. * @return the {@link Filter} that was parsed by the {@link SuperParser}. + * @throws ParseException */ - public abstract Filter storePostFilter(String table, String filterStr, Filter filter); + public abstract Filter storePostFilter(String table, String filterStr, Filter filter, FilterParser filterParser) throws ParseException; /** * Notifies the cache that the index is closing on this shard server. http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4e35bdd7/blur-core/src/main/java/org/apache/blur/manager/DefaultBlurFilterCache.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/DefaultBlurFilterCache.java b/blur-core/src/main/java/org/apache/blur/manager/DefaultBlurFilterCache.java index 3a9c0b9..a84c17e 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/DefaultBlurFilterCache.java +++ b/blur-core/src/main/java/org/apache/blur/manager/DefaultBlurFilterCache.java @@ -16,47 +16,198 @@ package org.apache.blur.manager; * See the License for the specific language governing permissions and * limitations under the License. */ +import static org.apache.blur.utils.BlurConstants.BLUR_FILTER_ALIAS; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + import org.apache.blur.BlurConfiguration; +import org.apache.blur.filter.FilterCache; +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; import org.apache.blur.manager.writer.BlurIndex; +import org.apache.lucene.index.Term; +import org.apache.lucene.queries.BooleanFilter; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.TermQuery; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; /** - * This implementation on {@link BlurFilterCache} does nothing and it is the default {@link BlurFilterCache}. + * This implementation on {@link BlurFilterCache} does nothing and it is the + * default {@link BlurFilterCache}. */ public class DefaultBlurFilterCache extends BlurFilterCache { + private static final Log LOG = LogFactory.getLog(DefaultBlurFilterCache.class); + + static class FilterKey { + final String _table; + final String _filterStr; + + FilterKey(String _table, String _filterStr) { + this._table = _table; + this._filterStr = _filterStr; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((_filterStr == null) ? 0 : _filterStr.hashCode()); + result = prime * result + ((_table == null) ? 0 : _table.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + FilterKey other = (FilterKey) obj; + if (_filterStr == null) { + if (other._filterStr != null) + return false; + } else if (!_filterStr.equals(other._filterStr)) + return false; + if (_table == null) { + if (other._table != null) + return false; + } else if (!_table.equals(other._table)) + return false; + return true; + } + + } + + private final Map<FilterKey, Filter> _preFilterCacheMap; + private final Map<FilterKey, Filter> _postFilterCacheMap; + private final ConcurrentMap<String, ConcurrentMap<String, String>> _tableAliasFilterMap = new ConcurrentHashMap<String, ConcurrentMap<String, String>>(); + public DefaultBlurFilterCache(BlurConfiguration configuration) { super(configuration); + long _cacheEntries = 100; + _preFilterCacheMap = new ConcurrentLinkedHashMap.Builder<FilterKey, Filter>() + .maximumWeightedCapacity(_cacheEntries).build(); + _postFilterCacheMap = new ConcurrentLinkedHashMap.Builder<FilterKey, Filter>().maximumWeightedCapacity( + _cacheEntries).build(); + Map<String, String> properties = configuration.getProperties(); + for (Entry<String, String> entry : properties.entrySet()) { + if (isFilterAlias(entry.getKey())) { + String name = getFilterAlias(entry.getKey()); + int index = name.indexOf('.'); + String table = name.substring(0, index); + String alias = name.substring(index + 1); + ConcurrentMap<String, String> aliasFilterMap = _tableAliasFilterMap.get(table); + if (aliasFilterMap == null) { + aliasFilterMap = new ConcurrentHashMap<String, String>(); + _tableAliasFilterMap.put(table, aliasFilterMap); + } + aliasFilterMap.put(alias, entry.getValue()); + } + } } @Override - public Filter storePreFilter(String table, String filterStr, Filter filter) { + public Filter storePreFilter(String table, String filterStr, Filter filter, FilterParser filterParser) + throws ParseException { + if (filter instanceof QueryWrapperFilter) { + QueryWrapperFilter queryWrapperFilter = (QueryWrapperFilter) filter; + Query query = queryWrapperFilter.getQuery(); + Filter newFilter = buildNewFilter(query, _tableAliasFilterMap.get(table), filterParser); + FilterKey key = new FilterKey(table, filterStr); + _preFilterCacheMap.put(key, newFilter); + return newFilter; + } return filter; } @Override - public Filter storePostFilter(String table, String filterStr, Filter filter) { + public Filter storePostFilter(String table, String filterStr, Filter filter, FilterParser filterParser) + throws ParseException { return filter; } @Override public Filter fetchPreFilter(String table, String filterStr) { - return null; + FilterKey filterKey = new FilterKey(table, filterStr); + return _preFilterCacheMap.get(filterKey); } @Override public Filter fetchPostFilter(String table, String filterStr) { - return null; + FilterKey filterKey = new FilterKey(table, filterStr); + return _postFilterCacheMap.get(filterKey); } @Override public void closing(String table, String shard, BlurIndex index) { - + _tableAliasFilterMap.remove(table); } @Override public void opening(String table, String shard, BlurIndex index) { + Map<String, String> properties = _configuration.getProperties(); + for (Entry<String, String> entry : properties.entrySet()) { + if (isFilterAlias(entry.getKey())) { + String filterAlias = getFilterAlias(entry.getKey()); + String filterQuery = entry.getValue(); + Map<String, String> map = getThisTablesMap(table); + LOG.info("Loading filter alias [{0}] with query [{1}] for table [{2}]", filterAlias, filterQuery, table); + map.put(filterAlias, filterQuery); + } + } + } + + private Map<String, String> getThisTablesMap(String table) { + _tableAliasFilterMap.putIfAbsent(table, new ConcurrentHashMap<String, String>()); + return _tableAliasFilterMap.get(table); + } + + private String getFilterAlias(String key) { + return key.substring(BLUR_FILTER_ALIAS.length()); + } + private boolean isFilterAlias(String key) { + return key.startsWith(BLUR_FILTER_ALIAS); } + + private Filter buildNewFilter(Query query, ConcurrentMap<String, String> filterAlias, FilterParser filterParser) + throws ParseException { + if (query instanceof BooleanQuery) { + BooleanQuery booleanQuery = (BooleanQuery) query; + BooleanFilter booleanFilter = new BooleanFilter(); + for (BooleanClause clause : booleanQuery.clauses()) { + booleanFilter.add(buildNewFilter(clause.getQuery(), filterAlias, filterParser), clause.getOccur()); + } + return booleanFilter; + } else if (query instanceof TermQuery) { + TermQuery termQuery = (TermQuery) query; + Term term = termQuery.getTerm(); + String key = term.toString(); + String queryStr = filterAlias.get(key); + if (queryStr == null) { + return new QueryWrapperFilter(termQuery); + } + String id = getId(key); + return new FilterCache(id, new QueryWrapperFilter(filterParser.parse(queryStr))); + } else { + return new QueryWrapperFilter(query); + } + } + + private String getId(String key) { + return key.replace(':', '-'); + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4e35bdd7/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java b/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java index 57a43a7..de7c1af 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java +++ b/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java @@ -31,15 +31,19 @@ import org.apache.lucene.search.QueryWrapperFilter; public class QueryParserUtil { - public static Query parseQuery(String query, boolean superQueryOn, FieldManager fieldManager, Filter postFilter, Filter preFilter, ScoreType scoreType, TableContext tableContext) throws ParseException { - Query result = new SuperParser(LUCENE_VERSION, fieldManager,superQueryOn, preFilter, scoreType, tableContext.getDefaultPrimeDocTerm()).parse(query); + public static Query parseQuery(String query, boolean superQueryOn, FieldManager fieldManager, Filter postFilter, + Filter preFilter, ScoreType scoreType, TableContext tableContext) throws ParseException { + Query result = new SuperParser(LUCENE_VERSION, fieldManager, superQueryOn, preFilter, scoreType, + tableContext.getDefaultPrimeDocTerm()).parse(query); if (postFilter == null) { return result; } return new FilteredQuery(result, postFilter); } - public static Filter parseFilter(String table, String filterStr, boolean superQueryOn, FieldManager fieldManager, BlurFilterCache filterCache, TableContext tableContext) throws ParseException, BlurException { + public static Filter parseFilter(String table, String filterStr, boolean superQueryOn, + final FieldManager fieldManager, BlurFilterCache filterCache, final TableContext tableContext) + throws ParseException, BlurException { if (filterStr == null) { return null; } @@ -53,11 +57,24 @@ public class QueryParserUtil { if (filter != null) { return filter; } - filter = new QueryWrapperFilter(new SuperParser(LUCENE_VERSION, fieldManager, superQueryOn, null, ScoreType.CONSTANT, tableContext.getDefaultPrimeDocTerm()).parse(filterStr)); + filter = new QueryWrapperFilter(new SuperParser(LUCENE_VERSION, fieldManager, superQueryOn, null, + ScoreType.CONSTANT, tableContext.getDefaultPrimeDocTerm()).parse(filterStr)); if (superQueryOn) { - filter = filterCache.storePostFilter(table, filterStr, filter); + filter = filterCache.storePostFilter(table, filterStr, filter, new BlurFilterCache.FilterParser() { + @Override + public Query parse(String query) throws ParseException { + return new SuperParser(LUCENE_VERSION, fieldManager, true, null, ScoreType.CONSTANT, tableContext + .getDefaultPrimeDocTerm()).parse(query); + } + }); } else { - filter = filterCache.storePreFilter(table, filterStr, filter); + filter = filterCache.storePreFilter(table, filterStr, filter, new BlurFilterCache.FilterParser() { + @Override + public Query parse(String query) throws ParseException { + return new SuperParser(LUCENE_VERSION, fieldManager, false, null, ScoreType.CONSTANT, tableContext + .getDefaultPrimeDocTerm()).parse(query); + } + }); } return filter; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4e35bdd7/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 5837fc4..3d672c2 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -47,6 +47,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT; import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT; import static org.apache.blur.utils.BlurUtil.quietClose; +import java.lang.reflect.Constructor; import java.util.concurrent.TimeUnit; import org.apache.blur.BlurConfiguration; @@ -258,12 +259,15 @@ public class ThriftBlurShardServer extends ThriftServer { return server; } + @SuppressWarnings("unchecked") private static BlurFilterCache getFilterCache(BlurConfiguration configuration) { String blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS); if (blurFilterCacheClass != null) { try { - Class<?> clazz = Class.forName(blurFilterCacheClass); - return (BlurFilterCache) clazz.newInstance(); + Class<? extends BlurFilterCache> clazz = (Class<? extends BlurFilterCache>) Class.forName(blurFilterCacheClass); + Class<?>[] types = new Class<?>[] {BlurConfiguration.class}; + Constructor<? extends BlurFilterCache> constructor = clazz.getConstructor(types); + return constructor.newInstance(new Object[]{configuration}); } catch (Exception e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4e35bdd7/blur-core/src/test/java/org/apache/blur/manager/DefaultBlurFilterCacheTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/DefaultBlurFilterCacheTest.java b/blur-core/src/test/java/org/apache/blur/manager/DefaultBlurFilterCacheTest.java new file mode 100644 index 0000000..267356f --- /dev/null +++ b/blur-core/src/test/java/org/apache/blur/manager/DefaultBlurFilterCacheTest.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.manager; + +import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.blur.BlurConfiguration; +import org.apache.blur.analysis.BaseFieldManager; +import org.apache.blur.analysis.FieldTypeDefinition; +import org.apache.blur.lucene.search.SuperParser; +import org.apache.blur.manager.BlurFilterCache.FilterParser; +import org.apache.blur.server.TableContext; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.ScoreType; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.utils.BlurConstants; +import org.apache.hadoop.conf.Configuration; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.junit.Test; + +public class DefaultBlurFilterCacheTest { + + private static final String TABLE = "test"; + private static final String TEST_FILTER = "(abc1 abc2 abc3)"; + + @Test + public void testFetchPreFilterEmpty() throws IOException { + BlurConfiguration configuration = new BlurConfiguration(); + DefaultBlurFilterCache defaultBlurFilterCache = new DefaultBlurFilterCache(configuration); + Filter fetchPreFilter = defaultBlurFilterCache.fetchPreFilter(TABLE, TEST_FILTER); + assertNull(fetchPreFilter); + } + + @Test + public void testFetchPreFilterNotEmpty() throws IOException, BlurException, ParseException { + BlurConfiguration configuration = new BlurConfiguration(); + configuration.set("blur.filter.alias.test.super:abc1", "(fam1.f1:abc1 fam2.f1:abc1)"); + configuration.set("blur.filter.alias.test.super:abc2", "(fam1.f1:abc2 fam2.f1:abc2)"); + configuration.set("blur.filter.alias.test.super:abc3", "(fam1.f1:abc3 fam2.f1:abc3)"); + DefaultBlurFilterCache defaultBlurFilterCache = new DefaultBlurFilterCache(configuration); + + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName(TABLE); + tableDescriptor.setTableUri("file:///"); + final TableContext tableContext = TableContext.create(tableDescriptor); + + final BaseFieldManager fieldManager = getFieldManager(new WhitespaceAnalyzer(LUCENE_VERSION)); + + Filter filter = QueryParserUtil.parseFilter(TABLE, TEST_FILTER, false, fieldManager, defaultBlurFilterCache, + tableContext); + Filter filterToRun = defaultBlurFilterCache.storePreFilter(TABLE, TEST_FILTER, filter, new FilterParser() { + @Override + public Query parse(String query) throws ParseException { + return new SuperParser(LUCENE_VERSION, fieldManager, false, null, ScoreType.CONSTANT, tableContext + .getDefaultPrimeDocTerm()).parse(query); + } + }); + assertNotNull(filterToRun); + assertEquals("BooleanFilter(" + "FilterCache(super-abc1,QueryWrapperFilter(fam1.f1:abc1 fam2.f1:abc1)) " + + "FilterCache(super-abc2,QueryWrapperFilter(fam1.f1:abc2 fam2.f1:abc2)) " + + "FilterCache(super-abc3,QueryWrapperFilter(fam1.f1:abc3 fam2.f1:abc3))" + ")", filterToRun.toString()); + + Filter fetchPreFilter = defaultBlurFilterCache.fetchPreFilter(TABLE, TEST_FILTER); + assertNotNull(fetchPreFilter); + + assertTrue(filterToRun == fetchPreFilter); + + } + + private BaseFieldManager getFieldManager(Analyzer a) throws IOException { + BaseFieldManager fieldManager = new BaseFieldManager(BlurConstants.SUPER, a, new Configuration()) { + @Override + protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) { + return true; + } + + @Override + protected void tryToLoad(String fieldName) { + + } + + @Override + protected List<String> getFieldNamesToLoad() throws IOException { + return new ArrayList<String>(); + } + }; + + fieldManager.addColumnDefinition(null, "bin", null, false, "string", null); + fieldManager.addColumnDefinitionInt("a", "id_i"); + fieldManager.addColumnDefinitionDouble("a", "id_d"); + fieldManager.addColumnDefinitionFloat("a", "id_f"); + fieldManager.addColumnDefinitionLong("a", "id_l"); + fieldManager.addColumnDefinitionDate("a", "id_date", "yyyy-MM-dd"); + fieldManager.addColumnDefinitionGisRecursivePrefixTree("a", "id_gis"); + return fieldManager; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4e35bdd7/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java b/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java index ab5ac68..667af7f 100644 --- a/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java +++ b/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java @@ -133,4 +133,8 @@ public class FilterCache extends Filter { return _misses.get(); } + @Override + public String toString() { + return "FilterCache(" + _id + "," + _filter + ")"; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4e35bdd7/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index ccf3ea9..c0667d7 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -130,6 +130,8 @@ public class BlurConstants { public static final String DELETE_MARKER = "_deletemarker_"; public static final String SHARED_MERGE_SCHEDULER = "sharedMergeScheduler"; + + public static final String BLUR_FILTER_ALIAS = "blur.filter.alias."; static { try {
