KYLIN-1526 Move query cache back to query controller level temp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a13952e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a13952e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a13952e Branch: refs/heads/master Commit: 1a13952eadaee154bfb1920653bf741d12c1001a Parents: 8703220 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Mar 23 11:15:41 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Mar 24 14:45:02 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/BasicTest.java | 1 - .../kylin/metadata/tuple/TeeTupleIterator.java | 71 ------- .../metadata/tuple/TeeTupleItrListener.java | 27 --- .../kylin/storage/ICachableStorageQuery.java | 51 ----- .../cache/AbstractCacheFledgedQuery.java | 100 ---------- .../storage/cache/CacheFledgedDynamicQuery.java | 180 ------------------ .../storage/cache/CacheFledgedStaticQuery.java | 77 -------- .../kylin/storage/cache/StreamSQLResult.java | 103 ---------- .../kylin/storage/cache/DynamicCacheTest.java | 186 ------------------- .../kylin/storage/cache/StaticCacheTest.java | 141 -------------- .../apache/kylin/query/ITKylinQueryTest.java | 11 +- .../kylin/storage/hbase/ITStorageTest.java | 23 +-- .../kylin/query/enumerator/OLAPEnumerator.java | 1 - .../kylin/rest/controller/QueryController.java | 37 +++- .../apache/kylin/rest/response/SQLResponse.java | 3 + .../apache/kylin/rest/service/CacheService.java | 8 +- .../apache/kylin/rest/service/QueryService.java | 11 +- .../org/apache/kylin/rest/StorageCacheTest.java | 116 ------------ .../kylin/storage/hbase/HBaseStorage.java | 34 +--- .../storage/hbase/cube/v1/CubeStorageQuery.java | 24 +-- .../storage/hbase/cube/v2/CubeStorageQuery.java | 22 +-- .../hbase/ii/InvertedIndexStorageQuery.java | 20 +- .../hbase/cube/CubeStorageQueryTest.java | 8 +- 23 files changed, 66 insertions(+), 1189 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index 8fcb66a..f1f5aa4 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -33,7 +33,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.kylin.common.KylinVersion; import org.junit.Ignore; import org.junit.Test; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java deleted file mode 100644 index e67443f..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.metadata.tuple; - -import java.util.List; - -import com.google.common.collect.Lists; - -/** - * Like "tee" command in linux, it effectively duplicates the underlying - * ITupleIterator's results - */ -public class TeeTupleIterator implements ITupleIterator { - - private ITupleIterator underlying; - private List<ITuple> duplicatedData; - private List<TeeTupleItrListener> listeners = Lists.newArrayList(); - private long createTime; - - public TeeTupleIterator(ITupleIterator underlying) { - this.underlying = underlying; - this.duplicatedData = Lists.newArrayList(); - this.createTime = System.currentTimeMillis(); - } - - @Override - public void close() { - this.underlying.close(); - - for (TeeTupleItrListener listener : this.listeners) { - listener.notify(this.duplicatedData, this.createTime); - } - } - - @Override - public boolean hasNext() { - return this.underlying.hasNext(); - } - - @Override - public ITuple next() { - ITuple ret = this.underlying.next(); - duplicatedData.add(ret.makeCopy()); - return ret; - } - - @Override - public void remove() { - this.underlying.remove(); - } - - public void addCloseListener(TeeTupleItrListener listener) { - this.listeners.add(listener); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java deleted file mode 100644 index b5fca16..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.metadata.tuple; - -import java.util.List; - -/** - */ -public interface TeeTupleItrListener { - void notify(List<ITuple> duplicated, long createTime); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java deleted file mode 100644 index f401f54..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage; - -import com.google.common.collect.Range; - -/** - */ -public interface ICachableStorageQuery extends IStorageQuery { - /** - * - * being dynamic => getVolatilePeriod() return not null - * being dynamic => partition column of its realization not null - * - * @return true for static storage like cubes - * false for dynamic storage like II - */ - boolean isDynamic(); - - /** - * volatile period is the period of time in which the returned data is not stable - * e.g. inverted index's last several minutes' data is dynamic as time goes by. - * data in this period cannot be cached - * - * This method should not be called before ITupleIterator.close() is called - * - * @return null if the underlying storage guarantees the data is static - */ - Range<Long> getVolatilePeriod(); - - /** - * get the uuid for the realization that serves this query - */ - String getStorageUUID(); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java deleted file mode 100644 index 4b94cb6..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.cache; - -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; -import net.sf.ehcache.Status; -import net.sf.ehcache.config.CacheConfiguration; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.metadata.realization.StreamSQLDigest; -import org.apache.kylin.metadata.tuple.TeeTupleItrListener; -import org.apache.kylin.storage.ICachableStorageQuery; -import org.apache.kylin.storage.IStorageQuery; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - */ -public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTupleItrListener { - private static final Logger logger = LoggerFactory.getLogger(AbstractCacheFledgedQuery.class); - - private static final String storageCacheTemplate = "StorageCache"; - - protected static CacheManager CACHE_MANAGER; - - protected ICachableStorageQuery underlyingStorage; - protected StreamSQLDigest streamSQLDigest; - - public AbstractCacheFledgedQuery(ICachableStorageQuery underlyingStorage) { - this.underlyingStorage = underlyingStorage; - this.makeCacheIfNecessary(underlyingStorage.getStorageUUID()); - } - - public static void setCacheManager(CacheManager cacheManager) { - CACHE_MANAGER = cacheManager; - } - - protected StreamSQLResult getStreamSQLResult(StreamSQLDigest streamSQLDigest) { - - Cache cache = CACHE_MANAGER.getCache(this.underlyingStorage.getStorageUUID()); - Element element = cache.get(streamSQLDigest.hashCode());//TODO: hash code cannot guarantee uniqueness - if (element != null) { - return (StreamSQLResult) element.getObjectValue(); - } - return null; - } - - protected boolean needSaveCache(long createTime) { - long storageQueryTime = System.currentTimeMillis() - createTime; - long durationThreshold = KylinConfig.getInstanceFromEnv().getQueryDurationCacheThreshold(); - //TODO: check scan count necessary? - - if (storageQueryTime < durationThreshold) { - logger.info("Skip saving storage caching for storage cache because storage query time {} less than {}", storageQueryTime, durationThreshold); - return false; - } - - return true; - } - - private void makeCacheIfNecessary(String storageUUID) { - if (CACHE_MANAGER == null || (!(CACHE_MANAGER.getStatus().equals(Status.STATUS_ALIVE)))) { - throw new RuntimeException("CACHE_MANAGER is not provided or not alive"); - } - - if (CACHE_MANAGER.getCache(storageUUID) == null) { - logger.info("Cache for {} initializing...", storageUUID); - - //Create a Cache specifying its configuration. - CacheConfiguration templateConf = CACHE_MANAGER.getCache(storageCacheTemplate).getCacheConfiguration(); - - Cache storageCache = new Cache(new CacheConfiguration(storageUUID, 0).// - memoryStoreEvictionPolicy(templateConf.getMemoryStoreEvictionPolicy()).// - eternal(templateConf.isEternal()).// - timeToIdleSeconds(templateConf.getTimeToIdleSeconds()).// - //maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).//using pooled size - persistence(templateConf.getPersistenceConfiguration())); - - CACHE_MANAGER.addCacheIfAbsent(storageCache); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java deleted file mode 100644 index a93e746..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.cache; - -import java.util.List; - -import net.sf.ehcache.Element; - -import org.apache.kylin.common.util.RangeUtil; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.realization.SQLDigestUtil; -import org.apache.kylin.metadata.realization.StreamSQLDigest; -import org.apache.kylin.metadata.tuple.CompoundTupleIterator; -import org.apache.kylin.metadata.tuple.ITuple; -import org.apache.kylin.metadata.tuple.ITupleIterator; -import org.apache.kylin.metadata.tuple.SimpleTupleIterator; -import org.apache.kylin.metadata.tuple.TeeTupleIterator; -import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.storage.ICachableStorageQuery; -import org.apache.kylin.storage.StorageContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Range; - -/** - */ -public class CacheFledgedDynamicQuery extends AbstractCacheFledgedQuery { - private static final Logger logger = LoggerFactory.getLogger(CacheFledgedDynamicQuery.class); - - private final TblColRef partitionColRef; - - private boolean noCacheUsed = true; - - private Range<Long> ts; - - public CacheFledgedDynamicQuery(ICachableStorageQuery underlyingStorage, TblColRef partitionColRef) { - super(underlyingStorage); - this.partitionColRef = partitionColRef; - - Preconditions.checkArgument(this.partitionColRef != null, "For dynamic columns like " + // - this.underlyingStorage.getStorageUUID() + ", partition column must be provided"); - } - - @Override - public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) { - //check if ts condition in sqlDigest valid - ts = TsConditionExtractor.extractTsCondition(partitionColRef, sqlDigest.filter); - if (ts == null || ts.isEmpty()) { - logger.info("ts range in the query conflicts,return empty directly"); - return ITupleIterator.EMPTY_TUPLE_ITERATOR; - } - - ITupleIterator ret = null; - - //enable dynamic cache iff group by columns contains partition col - //because cache extraction requires partition col value as selection key - boolean enableDynamicCache = sqlDigest.groupbyColumns.contains(partitionColRef); - - if (enableDynamicCache) { - streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef); - StreamSQLResult cachedResult = getStreamSQLResult(streamSQLDigest); - if (cachedResult != null) { - ret = tryReuseCache(context, sqlDigest, returnTupleInfo, cachedResult); - } else { - logger.info("no cache entry for this query"); - } - } - - if (ret == null) { - ret = underlyingStorage.search(context, sqlDigest, returnTupleInfo); - logger.info("No Cache being used"); - } else { - logger.info("Cache being used"); - } - - if (enableDynamicCache) { - //use another nested ITupleIterator to deal with cache - final TeeTupleIterator tee = new TeeTupleIterator(ret); - tee.addCloseListener(this); - return tee; - } else { - return ret; - } - } - - /** - * if cache is not enough it will try to combine existing cache as well as fresh records - */ - private ITupleIterator tryReuseCache(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo, StreamSQLResult cachedResult) { - Range<Long> reusePeriod = cachedResult.getReusableResults(ts); - - logger.info("existing cache: " + cachedResult); - logger.info("ts Range in query: " + RangeUtil.formatTsRange(ts)); - logger.info("potential reusable range: " + RangeUtil.formatTsRange(reusePeriod)); - - if (reusePeriod != null) { - List<Range<Long>> remainings = RangeUtil.remove(ts, reusePeriod); - if (remainings.size() == 1) {//if using cache causes two underlyingStorage searches, we'd rather not use the cache - - SimpleTupleIterator reusedTuples = new SimpleTupleIterator(cachedResult.reuse(reusePeriod)); - List<ITupleIterator> iTupleIteratorList = Lists.newArrayList(); - iTupleIteratorList.add(reusedTuples); - - Range<Long> remaining = remainings.get(0); - logger.info("Appending ts " + RangeUtil.formatTsRange(remaining) + " as additional filter"); - - ITupleIterator freshTuples = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, remaining, new Function<Void, ITupleIterator>() { - @Override - public ITupleIterator apply(Void input) { - return underlyingStorage.search(context, sqlDigest, returnTupleInfo); - } - }); - iTupleIteratorList.add(freshTuples); - - context.setReusedPeriod(reusePeriod); - return new CompoundTupleIterator(iTupleIteratorList); - } else if (remainings.size() == 0) { - logger.info("The ts range in new query was fully cached"); - context.setReusedPeriod(reusePeriod); - return new SimpleTupleIterator(cachedResult.reuse(reusePeriod)); - } else { - //if using cache causes more than one underlyingStorage searches - //the incurred overhead might be more expensive than the cache benefit - logger.info("Give up using cache to avoid complexity"); - return null; - } - } else { - logger.info("cached results not reusable by current query"); - return null; - } - } - - @Override - public void notify(List<ITuple> duplicated, long createTime) { - - //for streaming sql only check if needSaveCache at first entry of cache - if (noCacheUsed && !needSaveCache(createTime)) { - return; - } - - Range<Long> cacheExclude = this.underlyingStorage.getVolatilePeriod(); - if (cacheExclude != null) { - List<Range<Long>> cachablePeriods = RangeUtil.remove(ts, cacheExclude); - if (cachablePeriods.size() == 1) { - if (!ts.equals(cachablePeriods.get(0))) { - logger.info("tsRange shrinks from " + RangeUtil.formatTsRange(ts) + " to " + RangeUtil.formatTsRange(cachablePeriods.get(0))); - } - ts = cachablePeriods.get(0); - } else { - //give up updating the cache, in avoid to make cache complicated - logger.info("Skip updating cache to avoid complexity"); - } - } - - StreamSQLResult newCacheEntry = new StreamSQLResult(duplicated, ts, partitionColRef); - CACHE_MANAGER.getCache(this.underlyingStorage.getStorageUUID()).put(new Element(streamSQLDigest.hashCode(), newCacheEntry)); - logger.info("cache after the query: " + newCacheEntry); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java deleted file mode 100644 index 45c13fe..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.cache; - -import java.util.List; - -import net.sf.ehcache.Element; - -import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.realization.StreamSQLDigest; -import org.apache.kylin.metadata.tuple.ITuple; -import org.apache.kylin.metadata.tuple.ITupleIterator; -import org.apache.kylin.metadata.tuple.SimpleTupleIterator; -import org.apache.kylin.metadata.tuple.TeeTupleIterator; -import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.storage.ICachableStorageQuery; -import org.apache.kylin.storage.StorageContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Ranges; - -public class CacheFledgedStaticQuery extends AbstractCacheFledgedQuery { - private static final Logger logger = LoggerFactory.getLogger(CacheFledgedStaticQuery.class); - - public CacheFledgedStaticQuery(ICachableStorageQuery underlyingStorage) { - super(underlyingStorage); - } - - @Override - public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) { - - streamSQLDigest = new StreamSQLDigest(sqlDigest, null); - StreamSQLResult cachedResult = getStreamSQLResult(streamSQLDigest); - ITupleIterator ret; - - if (cachedResult != null) { - logger.info("using existing cache"); - context.setReusedPeriod(Ranges.<Long> all()); - return new SimpleTupleIterator(cachedResult.reuse(Ranges.<Long> all())); - } else { - logger.info("no existing cache to use"); - ret = underlyingStorage.search(context, sqlDigest, returnTupleInfo); - - //use another nested ITupleIterator to deal with cache - final TeeTupleIterator tee = new TeeTupleIterator(ret); - tee.addCloseListener(this); - return tee; - } - } - - @Override - public void notify(List<ITuple> duplicated, long createTime) { - if (needSaveCache(createTime)) { - StreamSQLResult newCacheEntry = new StreamSQLResult(duplicated, Ranges.<Long> all(), null); - CACHE_MANAGER.getCache(this.underlyingStorage.getStorageUUID()).put(new Element(streamSQLDigest.hashCode(), newCacheEntry)); - logger.info("cache after the query: " + newCacheEntry); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java deleted file mode 100644 index f6809df..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.cache; - -import java.util.Iterator; -import java.util.List; -import java.util.NavigableMap; - -import javax.annotation.Nullable; - -import org.apache.kylin.common.util.RangeUtil; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.tuple.ITuple; -import org.apache.kylin.metadata.tuple.Tuple; - -import com.google.common.base.Function; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Range; - -/** - */ -public class StreamSQLResult { - private Range<Long> timeCovered; - private NavigableMap<Long, List<ITuple>> sortedRows; - - public StreamSQLResult(List<ITuple> rows, Range<Long> timeCovered, TblColRef partitionCol) { - - sortedRows = Maps.newTreeMap(); - for (ITuple row : rows) { - - if (partitionCol != null) { - long t = Tuple.getTs(row, partitionCol); - - //will only cache rows that are within the time range - if (timeCovered.contains(t)) { - if (!this.sortedRows.containsKey(t)) { - this.sortedRows.put(t, Lists.newArrayList(row)); - } else { - this.sortedRows.get(t).add(row); - } - } - } else { - if (!this.sortedRows.containsKey(0L)) { - this.sortedRows.put(0L, Lists.<ITuple> newArrayList()); - } - this.sortedRows.get(0L).add(row); - } - } - this.timeCovered = timeCovered; - } - - public Range<Long> getReusableResults(Range<Long> tsRange) { - - if (tsRange.equals(timeCovered)) - return timeCovered; - - if (!timeCovered.isConnected(tsRange)) { - //share nothing in common - return null; - } - - Range<Long> ret = timeCovered.intersection(tsRange); - return ret.isEmpty() ? null : ret; - } - - public Iterator<ITuple> reuse(Range<Long> reusablePeriod) { - NavigableMap<Long, List<ITuple>> submap = RangeUtil.filter(sortedRows, reusablePeriod); - return Iterators.concat(Iterators.transform(submap.values().iterator(), new Function<List<ITuple>, Iterator<ITuple>>() { - @Nullable - @Override - public Iterator<ITuple> apply(List<ITuple> input) { - return input.iterator(); - } - })); - } - - @Override - public String toString() { - return sortedRows.size() + " tuples cached for period " + RangeUtil.formatTsRange(timeCovered); - } - - public long getEndTime() { - return this.timeCovered.upperEndpoint(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java deleted file mode 100644 index 0adaff3..0000000 --- a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.cache; - -import java.util.ArrayList; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import net.sf.ehcache.CacheManager; -import org.apache.commons.lang.NotImplementedException; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.common.util.IdentityUtils; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.tuple.ITuple; -import org.apache.kylin.metadata.tuple.ITupleIterator; -import org.apache.kylin.metadata.tuple.SimpleTupleIterator; -import org.apache.kylin.metadata.tuple.Tuple; -import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.storage.ICachableStorageQuery; -import org.apache.kylin.storage.StorageContext; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; -import com.google.common.collect.Range; -import com.google.common.collect.Ranges; - -/** - */ -public class DynamicCacheTest { - - private static CacheManager cacheManager; - - @BeforeClass - public static void setupResource() throws Exception { - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cache.threshold.duration", "0"); - - cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); - AbstractCacheFledgedQuery.setCacheManager(cacheManager); - } - - @AfterClass - public static void tearDownResource() { - cacheManager.shutdown(); - AbstractCacheFledgedQuery.setCacheManager(null); - } - - - class TsOnlyTuple implements ITuple { - private TblColRef partitionCol; - private String tsStr; - - public TsOnlyTuple(TblColRef partitionCol, String tsStr) { - this.partitionCol = partitionCol; - this.tsStr = tsStr; - } - - @Override - public List<String> getAllFields() { - throw new NotImplementedException(); - } - - @Override - public List<TblColRef> getAllColumns() { - throw new NotImplementedException(); - } - - @Override - public Object[] getAllValues() { - throw new NotImplementedException(); - } - - @Override - public ITuple makeCopy() { - return new TsOnlyTuple(this.partitionCol, this.tsStr); - } - - @Override - public Object getValue(TblColRef col) { - if (col.equals(partitionCol)) { - return Tuple.dateToEpicDays(this.tsStr); - } else { - throw new NotImplementedException(); - } - } - } - - @Test - public void basicTest() { - - final StorageContext context = new StorageContext(); - final List<TblColRef> groups = StorageMockUtils.buildGroups(); - final TblColRef partitionCol = groups.get(0); - final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations(); - final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations); - - SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef> newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef> newArrayList(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); - - ITuple aTuple = new TsOnlyTuple(partitionCol, "2011-02-01"); - ITuple bTuple = new TsOnlyTuple(partitionCol, "2012-02-01"); - final List<ITuple> allTuples = Lists.newArrayList(aTuple, bTuple); - - //counts for verifying - final AtomicInteger underlyingSEHitCount = new AtomicInteger(0); - final List<Integer> returnedRowPerSearch = Lists.newArrayList(); - - CacheFledgedDynamicQuery dynamicCache = new CacheFledgedDynamicQuery(new ICachableStorageQuery() { - @Override - public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { - Range<Long> tsRagneInQuery = TsConditionExtractor.extractTsCondition(partitionCol, sqlDigest.filter); - List<ITuple> ret = Lists.newArrayList(); - for (ITuple tuple : allTuples) { - if (tsRagneInQuery.contains(Tuple.getTs(tuple, partitionCol))) { - ret.add(tuple); - } - } - - underlyingSEHitCount.incrementAndGet(); - returnedRowPerSearch.add(ret.size()); - - return new SimpleTupleIterator(ret.iterator()); - } - - @Override - public boolean isDynamic() { - return true; - } - - @Override - public Range<Long> getVolatilePeriod() { - return Ranges.greaterThan(DateFormat.stringToMillis("2011-02-01")); - } - - @Override - public String getStorageUUID() { - return "111ca32a-a33e-4b69-12aa-0bb8b1f8c191"; - } - }, partitionCol); - - sqlDigest.filter = StorageMockUtils.buildTs2010Filter(groups.get(0)); - ITupleIterator firstIterator = dynamicCache.search(context, sqlDigest, tupleInfo); - IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>(); - while (firstIterator.hasNext()) { - firstResults.put(firstIterator.next(), null); - } - firstIterator.close(); - - sqlDigest.filter = StorageMockUtils.buildTs2011Filter(groups.get(0)); - ITupleIterator secondIterator = dynamicCache.search(context, sqlDigest, tupleInfo); - IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>(); - while (secondIterator.hasNext()) { - secondResults.put(secondIterator.next(), null); - } - secondIterator.close(); - - Assert.assertEquals(2, firstResults.size()); - IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet()); - Assert.assertEquals(2, underlyingSEHitCount.get()); - Assert.assertEquals(new Integer(2), returnedRowPerSearch.get(0)); - Assert.assertEquals(new Integer(1), returnedRowPerSearch.get(1)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java deleted file mode 100644 index 0a4795f..0000000 --- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.cache; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import net.sf.ehcache.CacheManager; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.IdentityUtils; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.tuple.ITuple; -import org.apache.kylin.metadata.tuple.ITupleIterator; -import org.apache.kylin.metadata.tuple.SimpleTupleIterator; -import org.apache.kylin.metadata.tuple.Tuple; -import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.storage.ICachableStorageQuery; -import org.apache.kylin.storage.StorageContext; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; -import com.google.common.collect.Range; - -/** - */ -public class StaticCacheTest { - - private static CacheManager cacheManager; - - @BeforeClass - public static void setupResource() throws Exception { - - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - - KylinConfig config = KylinConfig.getInstanceFromEnv(); - config.setProperty("kylin.query.cache.threshold.duration", "0"); - - cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); - AbstractCacheFledgedQuery.setCacheManager(cacheManager); - } - - @AfterClass - public static void tearDownResource() { - cacheManager.shutdown(); - AbstractCacheFledgedQuery.setCacheManager(null); - } - - @Test - public void basicTest() { - - final StorageContext context = new StorageContext(); - final List<TblColRef> groups = StorageMockUtils.buildGroups(); - final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations(); - final TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0)); - final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); - final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations); - - final List<ITuple> ret = Lists.newArrayList(); - ret.add(new Tuple(tupleInfo)); - ret.add(new Tuple(tupleInfo)); - ret.add(new Tuple(tupleInfo)); - - final AtomicInteger underlyingSEHitCount = new AtomicInteger(0); - - CacheFledgedStaticQuery cacheFledgedStaticStorageEngine = new CacheFledgedStaticQuery(new ICachableStorageQuery() { - @Override - public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { - underlyingSEHitCount.incrementAndGet(); - return new SimpleTupleIterator(ret.iterator()); - } - - @Override - public boolean isDynamic() { - return false; - } - - @Override - public Range<Long> getVolatilePeriod() { - return null; - } - - @Override - public String getStorageUUID() { - return "111ca32a-a33e-4b69-12aa-0bb8b1f8c092"; - } - }); - - ITupleIterator firstIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo); - IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>(); - while (firstIterator.hasNext()) { - firstResults.put(firstIterator.next(), null); - } - firstIterator.close(); - - ITupleIterator secondIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo); - IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>(); - while (secondIterator.hasNext()) { - secondResults.put(secondIterator.next(), null); - } - secondIterator.close(); - - ITupleIterator thirdIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo); - IdentityHashMap<ITuple, Void> thirdResults = new IdentityHashMap<>(); - while (thirdIterator.hasNext()) { - thirdResults.put(thirdIterator.next(), null); - } - thirdIterator.close(); - - Assert.assertEquals(3, firstResults.size()); - IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet()); - IdentityUtils.collectionReferenceEquals(thirdResults.keySet(), secondResults.keySet()); - - Assert.assertEquals(1, underlyingSEHitCount.get()); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 3fb93d7..4168df2 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -25,16 +25,14 @@ import java.sql.DriverManager; import java.util.List; import java.util.Properties; -import net.sf.ehcache.CacheManager; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.query.enumerator.OLAPQuery; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.schema.OLAPSchemaFactory; -import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; -import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.dbunit.database.DatabaseConnection; import org.dbunit.database.IDatabaseConnection; import org.junit.AfterClass; @@ -44,7 +42,6 @@ import org.junit.Test; @Ignore("KylinQueryTest is contained by ITCombinationTest") public class ITKylinQueryTest extends KylinTestBase { - private static CacheManager cacheManager; @BeforeClass public static void setUp() throws Exception { @@ -77,8 +74,6 @@ public class ITKylinQueryTest extends KylinTestBase { H2Database h2DB = new H2Database(h2Connection, config); h2DB.loadAllTables(); - cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); - AbstractCacheFledgedQuery.setCacheManager(cacheManager); } protected static void clean() { @@ -90,10 +85,6 @@ public class ITKylinQueryTest extends KylinTestBase { ObserverEnabler.forceCoprocessorUnset(); HBaseMetadataTestCase.staticCleanupTestMetadata(); - if (cacheManager != null) { - cacheManager.shutdown(); - } - AbstractCacheFledgedQuery.setCacheManager(null); } @Ignore("this is only for debug") http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index c253770..15e435e 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -24,8 +24,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import net.sf.ehcache.CacheManager; - import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.apache.kylin.cube.CubeInstance; @@ -40,7 +38,6 @@ import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.storage.IStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.StorageFactory; -import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; import org.apache.kylin.storage.cache.StorageMockUtils; import org.apache.kylin.storage.exception.ScanOutOfLimitException; import org.junit.After; @@ -57,18 +54,12 @@ public class ITStorageTest extends HBaseMetadataTestCase { private CubeInstance cube; private StorageContext context; - private static CacheManager cacheManager; - @BeforeClass public static void setupResource() throws Exception { - cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); - AbstractCacheFledgedQuery.setCacheManager(cacheManager); } @AfterClass public static void tearDownResource() { - cacheManager.shutdown(); - AbstractCacheFledgedQuery.setCacheManager(null); } @Before @@ -115,36 +106,36 @@ public class ITStorageTest extends HBaseMetadataTestCase { List<TblColRef> groups = buildGroups(); List<FunctionDesc> aggregations = buildAggregations(); TupleFilter filter = buildFilter2(groups.get(1)); - + int count = search(groups, aggregations, filter, context); assertTrue(count > 0); } - + @Test public void test03() { List<TblColRef> groups = buildGroups(); List<FunctionDesc> aggregations = buildAggregations(); TupleFilter filter = buildAndFilter(groups); - + int count = search(groups, aggregations, filter, context); assertTrue(count > 0); } - + @Test public void test04() { List<TblColRef> groups = buildGroups(); List<FunctionDesc> aggregations = buildAggregations(); TupleFilter filter = buildOrFilter(groups); - + int count = search(groups, aggregations, filter, context); assertTrue(count > 0); } - + @Test public void test05() { List<TblColRef> groups = buildGroups(); List<FunctionDesc> aggregations = buildAggregations(); - + int count = search(groups, aggregations, null, context); assertTrue(count > 0); } http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java index aa8b2a7..07f72b1 100644 --- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java +++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java @@ -125,7 +125,6 @@ public class OLAPEnumerator implements Enumerator<Object[]> { ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest, olapContext.returnTupleInfo); if (logger.isDebugEnabled()) { logger.debug("return TupleIterator..."); - logger.debug("Storage cache used for this storage query:" + olapContext.storageContext.getReusedPeriod()); } return iterator; http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java index f60894e..88cf912 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java @@ -46,7 +46,6 @@ import org.apache.kylin.rest.request.SaveSqlRequest; import org.apache.kylin.rest.response.SQLResponse; import org.apache.kylin.rest.service.QueryService; import org.apache.kylin.rest.util.QueryUtil; -import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; import org.apache.kylin.storage.exception.ScanOutOfLimitException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,6 +74,7 @@ public class QueryController extends BasicController { private static final Logger logger = LoggerFactory.getLogger(QueryController.class); + public static final String SUCCESS_QUERY_CACHE = "StorageCache"; public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache"; @Autowired @@ -86,7 +86,6 @@ public class QueryController extends BasicController { @PostConstruct public void init() throws IOException { Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet"); - AbstractCacheFledgedQuery.setCacheManager(cacheManager); } @RequestMapping(value = "/query", method = RequestMethod.POST) @@ -185,10 +184,23 @@ public class QueryController extends BasicController { throw new InternalErrorException("Not Supported SQL."); } + long startTime = System.currentTimeMillis(); + SQLResponse sqlResponse = searchQueryInCache(sqlRequest); try { if (null == sqlResponse) { sqlResponse = queryService.query(sqlRequest); + + long durationThreshold = KylinConfig.getInstanceFromEnv().getQueryDurationCacheThreshold(); + long scancountThreshold = KylinConfig.getInstanceFromEnv().getQueryScanCountCacheThreshold(); + sqlResponse.setDuration(System.currentTimeMillis() - startTime); + logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", // + new String[] { String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()) }); + if (!sqlResponse.getIsException() && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) { + cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse)); + } + } else { + sqlResponse.setDuration(System.currentTimeMillis() - startTime); } checkQueryAuth(sqlResponse); @@ -221,13 +233,20 @@ public class QueryController extends BasicController { private SQLResponse searchQueryInCache(SQLRequest sqlRequest) { SQLResponse response = null; Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE); - - if (KylinConfig.getInstanceFromEnv().isQueryCacheEnabled() && // - !BackdoorToggles.getDisableCache() && // - exceptionCache.get(sqlRequest) != null) { - Element element = exceptionCache.get(sqlRequest); - response = (SQLResponse) element.getObjectValue(); - response.setHitExceptionCache(true); + Cache successCache = cacheManager.getCache(SUCCESS_QUERY_CACHE); + + if (KylinConfig.getInstanceFromEnv().isQueryCacheEnabled() && !BackdoorToggles.getDisableCache()) { + if (exceptionCache.get(sqlRequest) != null) { + logger.info("The sqlResponse is found in EXCEPTION_QUERY_CACHE"); + Element element = exceptionCache.get(sqlRequest); + response = (SQLResponse) element.getObjectValue(); + response.setHitExceptionCache(true); + } else if (successCache.get(sqlRequest) != null) { + logger.info("The sqlResponse is found in SUCCESS_QUERY_CACHE"); + Element element = successCache.get(sqlRequest); + response = (SQLResponse) element.getObjectValue(); + response.setStorageCacheUsed(true); + } } return response; http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java index f5ef8c5..8fb8788 100644 --- a/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java +++ b/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java @@ -35,6 +35,9 @@ public class SQLResponse implements Serializable { // the results rows, each row contains several columns private List<List<String>> results; + /** + * for historical reasons it is named "cube", however it might also refer to any realizations like hybrid, II or etc. + */ private String cube; // if not select query, only return affected row count http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/server/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java index 83b9ac7..9a6036d 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -49,6 +49,7 @@ import org.apache.kylin.metadata.realization.RealizationRegistry; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.query.enumerator.OLAPQuery; import org.apache.kylin.query.schema.OLAPSchemaFactory; +import org.apache.kylin.rest.controller.QueryController; import org.apache.kylin.source.kafka.KafkaConfigManager; import org.apache.kylin.storage.hbase.HBaseConnection; import org.apache.kylin.storage.hybrid.HybridManager; @@ -101,9 +102,10 @@ public class CacheService extends BasicService { } protected void cleanDataCache(String storageUUID) { - if (cacheManager != null && cacheManager.getCache(storageUUID) != null) { - logger.info("cleaning cache for " + storageUUID); - cacheManager.getCache(storageUUID).removeAll(); + if (cacheManager != null) { + logger.info("cleaning cache for " + storageUUID + " (currently remove all entries)"); + cacheManager.getCache(QueryController.SUCCESS_QUERY_CACHE).removeAll(); + cacheManager.getCache(QueryController.EXCEPTION_QUERY_CACHE).removeAll(); } else { logger.warn("skip cleaning cache for " + storageUUID); } http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/server/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java index bf371be..059a095 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -203,7 +203,7 @@ public class QueryService extends BasicService { final Set<String> realizationNames = new HashSet<String>(); final Set<Long> cuboidIds = new HashSet<Long>(); float duration = response.getDuration() / (float) 1000; - boolean storageCacheUsed = false; + boolean storageCacheUsed = response.isStorageCacheUsed(); if (!response.isHitExceptionCache() && null != OLAPContext.getThreadLocalContexts()) { for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) { @@ -218,10 +218,6 @@ public class QueryService extends BasicService { realizationNames.add(realizationName); } - if (ctx.storageContext.getReusedPeriod() != null) { - response.setStorageCacheUsed(true); - storageCacheUsed = true; - } } } @@ -340,7 +336,6 @@ public class QueryService extends BasicService { Connection conn = null; Statement stat = null; ResultSet resultSet = null; - long startTime = System.currentTimeMillis(); List<List<String>> results = new LinkedList<List<String>>(); List<SelectedColumnMeta> columnMetas = new LinkedList<SelectedColumnMeta>(); @@ -386,6 +381,7 @@ public class QueryService extends BasicService { boolean isPartialResult = false; String cube = ""; + StringBuilder sb = new StringBuilder("Scan count for each storageContext: "); long totalScanCount = 0; if (OLAPContext.getThreadLocalContexts() != null) { // contexts can be null in case of 'explain plan for' for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) { @@ -393,13 +389,14 @@ public class QueryService extends BasicService { isPartialResult |= ctx.storageContext.isPartialResultReturned(); cube = ctx.realization.getName(); totalScanCount += ctx.storageContext.getTotalScanCount(); + sb.append(ctx.storageContext.getTotalScanCount() + ","); } } } + logger.info(sb.toString()); SQLResponse response = new SQLResponse(columnMetas, results, cube, 0, false, null, isPartialResult); response.setTotalScanCount(totalScanCount); - response.setDuration(System.currentTimeMillis() - startTime); return response; } http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java deleted file mode 100644 index b6cc6d0..0000000 --- a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.rest; - -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; - -import org.apache.commons.lang.NotImplementedException; -import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.tuple.ITupleIterator; -import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.rest.service.ServiceTestBase; -import org.apache.kylin.storage.ICachableStorageQuery; -import org.apache.kylin.storage.StorageContext; -import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; -import org.apache.kylin.storage.cache.CacheFledgedStaticQuery; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import com.google.common.collect.Range; - -public class StorageCacheTest extends ServiceTestBase { - - public class MockedCachableStorageQuery implements ICachableStorageQuery { - private String uuid; - - public MockedCachableStorageQuery(String uuid) { - this.uuid = uuid; - } - - @Override - public boolean isDynamic() { - return false; - } - - @Override - public Range<Long> getVolatilePeriod() { - throw new NotImplementedException(); - } - - @Override - public String getStorageUUID() { - return this.uuid; - } - - @Override - public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { - throw new NotImplementedException(); - } - } - - @Autowired - private CacheManager cacheManager; - - @Before - public void setup() throws Exception { - AbstractCacheFledgedQuery.setCacheManager(cacheManager); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("1")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("2")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("3")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("4")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("5")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("6")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("7")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("8")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("9")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("10")); - new CacheFledgedStaticQuery(new MockedCachableStorageQuery("11")); - } - - @Test - public void test1() { - int oneM = 1 << 20; - cacheManager.getCache("1").put(new Element("xx", new byte[oneM])); - Element xx = cacheManager.getCache("1").get("xx"); - Assert.assertEquals(oneM, ((byte[]) xx.getObjectValue()).length); - - cacheManager.getCache("2").put(new Element("yy", new byte[3 * oneM])); - Element yy = cacheManager.getCache("2").get("yy"); - Assert.assertEquals(3 * oneM, ((byte[]) yy.getObjectValue()).length); - - cacheManager.getCache("3").put(new Element("zz", new byte[10 * oneM])); - Element zz = cacheManager.getCache("3").get("zz"); - Assert.assertEquals(null, zz); - - cacheManager.getCache("4").put(new Element("aa", new byte[oneM])); - Element aa = cacheManager.getCache("4").get("aa"); - Assert.assertEquals(oneM, ((byte[]) aa.getObjectValue()).length); - - cacheManager.getCache("2").put(new Element("bb", new byte[3 * oneM])); - Element bb = cacheManager.getCache("2").get("bb"); - Assert.assertEquals(3 * oneM, ((byte[]) bb.getObjectValue()).length); - } - - @Test - public void test2() { - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java index e71434f..20bc229 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java @@ -31,11 +31,8 @@ import org.apache.kylin.metadata.model.PartitionDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.RealizationType; -import org.apache.kylin.storage.ICachableStorageQuery; import org.apache.kylin.storage.IStorage; import org.apache.kylin.storage.IStorageQuery; -import org.apache.kylin.storage.cache.CacheFledgedDynamicQuery; -import org.apache.kylin.storage.cache.CacheFledgedStaticQuery; import org.apache.kylin.storage.hbase.steps.HBaseMROutput; import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition; @@ -54,22 +51,15 @@ public class HBaseStorage implements IStorage { @Override public IStorageQuery createQuery(IRealization realization) { - boolean queryCacheGloballyEnabled = KylinConfig.getInstanceFromEnv().isQueryCacheEnabled(); - boolean queryCacheQueryLevelEnabled = !BackdoorToggles.getDisableCache(); - if (realization.getType() == RealizationType.INVERTED_INDEX) { - ICachableStorageQuery ret; + IStorageQuery ret; try { - ret = (ICachableStorageQuery) Class.forName(defaultIIStorageQuery).getConstructor(IIInstance.class).newInstance((IIInstance) realization); + ret = (IStorageQuery) Class.forName(defaultIIStorageQuery).getConstructor(IIInstance.class).newInstance((IIInstance) realization); } catch (Exception e) { throw new RuntimeException("Failed to initialize storage query for " + defaultIIStorageQuery, e); } + return ret; - if (queryCacheGloballyEnabled && queryCacheQueryLevelEnabled) { - return wrapWithCache(ret, realization); - } else { - return ret; - } } else if (realization.getType() == RealizationType.CUBE) { CubeInstance cubeInstance = (CubeInstance) realization; @@ -84,31 +74,19 @@ public class HBaseStorage implements IStorage { cubeStorageQuery = v2CubeStorageQuery;//by default use v2 } - ICachableStorageQuery ret; + IStorageQuery ret; try { - ret = (ICachableStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization); + ret = (IStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization); } catch (Exception e) { throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e); } - if (queryCacheGloballyEnabled && queryCacheQueryLevelEnabled) { - return wrapWithCache(ret, realization); - } else { - return ret; - } + return ret; } else { throw new IllegalArgumentException("Unknown realization type " + realization.getType()); } } - private static IStorageQuery wrapWithCache(ICachableStorageQuery underlyingStorageEngine, IRealization realization) { - if (underlyingStorageEngine.isDynamic()) { - return new CacheFledgedDynamicQuery(underlyingStorageEngine, getPartitionCol(realization)); - } else { - return new CacheFledgedStaticQuery(underlyingStorageEngine); - } - } - private static TblColRef getPartitionCol(IRealization realization) { String modelName = realization.getModelName(); DataModelDesc dataModelDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName); http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 075328f..47e94de 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -62,7 +62,7 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.SQLDigest; import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.storage.ICachableStorageQuery; +import org.apache.kylin.storage.IStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.hbase.HBaseConnection; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; @@ -75,11 +75,10 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Range; import com.google.common.collect.Sets; @SuppressWarnings("unused") -public class CubeStorageQuery implements ICachableStorageQuery { +public class CubeStorageQuery implements IStorageQuery { private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class); @@ -154,21 +153,6 @@ public class CubeStorageQuery implements ICachableStorageQuery { return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo); } - @Override - public Range<Long> getVolatilePeriod() { - return null; - } - - @Override - public String getStorageUUID() { - return this.uuid; - } - - @Override - public boolean isDynamic() { - return false; - } - private void buildDimensionsAndMetrics(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, SQLDigest sqlDigest) { for (FunctionDesc func : sqlDigest.aggregations) { @@ -493,7 +477,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { List<Collection<ColumnValueRange>> result = Lists.newArrayList(); if (flatFilter == null) { - result.add(Collections.<ColumnValueRange>emptyList()); + result.add(Collections.<ColumnValueRange> emptyList()); return result; } @@ -535,7 +519,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { } if (globalAlwaysTrue) { orAndRanges.clear(); - orAndRanges.add(Collections.<ColumnValueRange>emptyList()); + orAndRanges.add(Collections.<ColumnValueRange> emptyList()); } return orAndRanges; } http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java index 85082ca..863dd67 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java @@ -45,17 +45,16 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.SQLDigest; import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.storage.ICachableStorageQuery; +import org.apache.kylin.storage.IStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.translate.DerivedFilterTranslator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; -import com.google.common.collect.Range; import com.google.common.collect.Sets; -public class CubeStorageQuery implements ICachableStorageQuery { +public class CubeStorageQuery implements IStorageQuery { private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class); @@ -379,21 +378,4 @@ public class CubeStorageQuery implements ICachableStorageQuery { } } - // ============================================================================ - - @Override - public Range<Long> getVolatilePeriod() { - return null; - } - - @Override - public String getStorageUUID() { - return cubeInstance.getUuid(); - } - - @Override - public boolean isDynamic() { - return false; - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java index 5afb62d..fef9662 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java @@ -26,19 +26,17 @@ import org.apache.kylin.invertedindex.IISegment; import org.apache.kylin.metadata.realization.SQLDigest; import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.metadata.tuple.TupleInfo; -import org.apache.kylin.storage.ICachableStorageQuery; +import org.apache.kylin.storage.IStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.hbase.HBaseConnection; import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.EndpointTupleIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Range; - /** * @author yangli9 */ -public class InvertedIndexStorageQuery implements ICachableStorageQuery { +public class InvertedIndexStorageQuery implements IStorageQuery { private static Logger logger = LoggerFactory.getLogger(InvertedIndexStorageQuery.class); @@ -66,18 +64,4 @@ public class InvertedIndexStorageQuery implements ICachableStorageQuery { } } - @Override - public Range<Long> getVolatilePeriod() { - return dataIterator.getCacheExcludedPeriod(); - } - - @Override - public String getStorageUUID() { - return this.uuid; - } - - @Override - public boolean isDynamic() { - return true; - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/1a13952e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java index 8c90abd..71f751b 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java @@ -41,7 +41,7 @@ import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.storage.ICachableStorageQuery; +import org.apache.kylin.storage.IStorageQuery; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -77,7 +77,7 @@ public class CubeStorageQueryTest extends LocalFileMetadataTestCase { this.cleanupTestMetadata(); } - private void validateIdentifyCuboidOnStorageQnery(CubeDesc cubeDesc, ICachableStorageQuery query) { + private void validateIdentifyCuboidOnStorageQnery(CubeDesc cubeDesc, IStorageQuery query) { long baseCuboidId = cubeDesc.getRowkey().getFullMask(); try { @@ -113,14 +113,14 @@ public class CubeStorageQueryTest extends LocalFileMetadataTestCase { @Test public void testIdentifyCuboidV1() { CubeDesc cubeDesc = cube.getDescriptor(); - ICachableStorageQuery query = new org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery(cube); + IStorageQuery query = new org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery(cube); validateIdentifyCuboidOnStorageQnery(cubeDesc, query); } @Test public void testIdentifyCuboidV2() { CubeDesc cubeDesc = cube.getDescriptor(); - ICachableStorageQuery query = new org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery(cube); + IStorageQuery query = new org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery(cube); validateIdentifyCuboidOnStorageQnery(cubeDesc, query); }
