Repository: hive Updated Branches: refs/heads/master 2820fc4c6 -> 2d097dcbf
HIVE-20242: Query results cache: Improve ability of queries to use pending query results (Jason Dere, reviewed by GopalV) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d097dcb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d097dcb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d097dcb Branch: refs/heads/master Commit: 2d097dcbf707a7f8491e8fed9a44f5764d73d580 Parents: 2820fc4 Author: Jason Dere <[email protected]> Authored: Thu Jul 26 11:57:49 2018 -0700 Committer: Jason Dere <[email protected]> Committed: Thu Jul 26 11:57:49 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 31 ++++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2d097dcb/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 86b53cc..b5adf1b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12185,7 +12185,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { QueryResultsCache.LookupInfo lookupInfo = null; if (isCacheEnabled && !needsTransform && queryTypeCanUseCache()) { lookupInfo = createLookupInfoForQuery(ast); - if (checkResultsCache(lookupInfo)) { + if (checkResultsCache(lookupInfo, false)) { return; } } @@ -12230,7 +12230,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // TODO: Enable caching for queries with masking/filtering if (isCacheEnabled && needsTransform && !usesMasking && queryTypeCanUseCache()) { lookupInfo = createLookupInfoForQuery(ast); - if (checkResultsCache(lookupInfo)) { + if (checkResultsCache(lookupInfo, false)) { return; } } @@ -12390,11 +12390,19 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (isCacheEnabled && lookupInfo != null) { if (queryCanBeCached()) { - QueryResultsCache.QueryInfo queryInfo = createCacheQueryInfoForQuery(lookupInfo); + // Last chance - check if the query is available in the cache. + // Since we have already generated a query plan, using a cached query result at this point + // requires SemanticAnalyzer state to be reset. + if (checkResultsCache(lookupInfo, true)) { + LOG.info("Cached result found on second lookup"); + return; + } else { + QueryResultsCache.QueryInfo queryInfo = createCacheQueryInfoForQuery(lookupInfo); - // Specify that the results of this query can be cached. - setCacheUsage(new CacheUsage( - CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS, queryInfo)); + // Specify that the results of this query can be cached. + setCacheUsage(new CacheUsage( + CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS, queryInfo)); + } } } } @@ -14741,7 +14749,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * Set the query plan to use cache entry passed in to return the query results. * @param cacheEntry The results cache entry that will be used to resolve the query. */ - private void useCachedResult(QueryResultsCache.CacheEntry cacheEntry) { + private void useCachedResult(QueryResultsCache.CacheEntry cacheEntry, boolean needsReset) { + if (needsReset) { + reset(true); + inputs.clear(); + } + // Change query FetchTask to use new location specified in results cache. FetchTask fetchTask = (FetchTask) TaskFactory.get(cacheEntry.getFetchWork()); setFetchTask(fetchTask); @@ -14866,7 +14879,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * answered using the results cache. If the cache contains a suitable entry, the semantic analyzer * will be configured to use the found cache entry to anwer the query. */ - private boolean checkResultsCache(QueryResultsCache.LookupInfo lookupInfo) { + private boolean checkResultsCache(QueryResultsCache.LookupInfo lookupInfo, boolean needsReset) { if (lookupInfo == null) { return false; } @@ -14910,7 +14923,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } // Use the cache rather than full query execution. // At this point the caller should return from semantic analysis. - useCachedResult(cacheEntry); + useCachedResult(cacheEntry, needsReset); return true; } }
