This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-0.15 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 171ee71caa731f522e3651fa766152030d631df7 Author: Zeno Yang <[email protected]> AuthorDate: Sat Oct 16 21:56:24 2021 +0800 [SQL Cache] Add all view stmt as the suffix of cache sqlkey (#6832) Use all view stmt as the cache sqlkey suffix, so that when the view is modified, the cache can recognize. --- .../org/apache/doris/analysis/InlineViewRef.java | 8 ++++ .../main/java/org/apache/doris/qe/cache/Cache.java | 1 + .../org/apache/doris/qe/cache/CacheAnalyzer.java | 43 +++++++++++++++++++++- .../org/apache/doris/qe/cache/PartitionCache.java | 7 +++- .../java/org/apache/doris/qe/cache/SqlCache.java | 6 ++- 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java index 807949e..874f020 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java @@ -417,6 +417,14 @@ public class InlineViewRef extends TableRef { return baseTblSmap; } + public boolean isLocalView() { + return view == null || view.isLocalView(); + } + + public View getView() { + return view; + } + @Override public String tableRefToSql() { // Enclose the alias in quotes if Hive cannot parse it without quotes. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java index 1711179..6d80556 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java @@ -43,6 +43,7 @@ public abstract class Cache { protected CacheAnalyzer.CacheTable latestTable; protected CacheProxy proxy; protected HitRange hitRange; + protected String allViewExpandStmtListStr; protected Cache(TUniqueId queryId, SelectStmt selectStmt) { this.queryId = queryId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index d8a6f47..e651d00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -17,6 +17,7 @@ package org.apache.doris.qe.cache; +import org.apache.commons.lang3.StringUtils; import org.apache.doris.analysis.AggregateInfo; import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.CastExpr; @@ -25,6 +26,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.InlineViewRef; import org.apache.doris.analysis.QueryStmt; import org.apache.doris.analysis.SelectStmt; +import org.apache.doris.analysis.SetOperationStmt; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.TableRef; @@ -33,6 +35,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.RangePartitionInfo; +import org.apache.doris.catalog.View; import org.apache.doris.common.Config; import org.apache.doris.common.Status; import org.apache.doris.common.util.DebugUtil; @@ -51,8 +54,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * Analyze which caching mode a SQL is suitable for @@ -87,6 +93,7 @@ public class CacheAnalyzer { private Column partColumn; private CompoundPredicate partitionPredicate; private Cache cache; + private Set<String> allViewStmtSet; public Cache getCache() { return cache; @@ -98,6 +105,7 @@ public class CacheAnalyzer { this.parsedStmt = parsedStmt; scanNodes = planner.getScanNodes(); latestTable = new CacheTable(); + allViewStmtSet = new HashSet<>(); checkCacheConfig(); } @@ -106,6 +114,7 @@ public class CacheAnalyzer { this.context = context; this.parsedStmt = parsedStmt; this.scanNodes = scanNodes; + allViewStmtSet = new HashSet<>(); checkCacheConfig(); } @@ -209,6 +218,9 @@ public class CacheAnalyzer { latestTable = tblTimeList.get(0); latestTable.Debug(); + addAllViewStmt(selectStmt); + String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, ","); + if (now == 0) { now = nowtime(); } @@ -216,7 +228,7 @@ public class CacheAnalyzer { (now - latestTable.latestTime) >= Config.cache_last_version_interval_second * 1000) { LOG.debug("TIME:{},{},{}", now, latestTable.latestTime, Config.cache_last_version_interval_second*1000); cache = new SqlCache(this.queryId, this.selectStmt); - ((SqlCache) cache).setCacheInfo(this.latestTable); + ((SqlCache) cache).setCacheInfo(this.latestTable, allViewExpandStmtListStr); MetricRepo.COUNTER_CACHE_MODE_SQL.increase(1L); return CacheMode.Sql; } @@ -263,7 +275,7 @@ public class CacheAnalyzer { partitionPredicate = compoundPredicates.get(0); cache = new PartitionCache(this.queryId, this.selectStmt); ((PartitionCache) cache).setCacheInfo(this.latestTable, this.partitionInfo, this.partColumn, - this.partitionPredicate); + this.partitionPredicate, allViewExpandStmtListStr); MetricRepo.COUNTER_CACHE_MODE_PARTITION.increase(1L); return CacheMode.Partition; } @@ -430,6 +442,33 @@ public class CacheAnalyzer { return table; } + private void addAllViewStmt(List<TableRef> tblRefs) { + for (TableRef tblRef : tblRefs) { + if (tblRef instanceof InlineViewRef) { + InlineViewRef inlineViewRef = (InlineViewRef) tblRef; + if (inlineViewRef.isLocalView()) { + Collection<View> views = inlineViewRef.getAnalyzer().getLocalViews().values(); + for (View view : views) { + addAllViewStmt(view.getQueryStmt()); + } + } else { + addAllViewStmt(inlineViewRef.getViewStmt()); + allViewStmtSet.add(inlineViewRef.getView().getInlineViewDef()); + } + } + } + } + + private void addAllViewStmt(QueryStmt queryStmt) { + if (queryStmt instanceof SelectStmt) { + addAllViewStmt(((SelectStmt) queryStmt).getTableRefs()); + } else if (queryStmt instanceof SetOperationStmt) { + for (SetOperationStmt.SetOperand operand : ((SetOperationStmt) queryStmt).getOperands()) { + addAllViewStmt(((SelectStmt) operand.getQueryStmt()).getTableRefs()); + } + } + } + public Cache.HitRange getHitRange() { if (cacheMode == CacheMode.None) { return Cache.HitRange.None; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java index d208d77..1ff1fdb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java @@ -66,13 +66,14 @@ public class PartitionCache extends Cache { } public void setCacheInfo(CacheAnalyzer.CacheTable latestTable, RangePartitionInfo partitionInfo, Column partColumn, - CompoundPredicate partitionPredicate) { + CompoundPredicate partitionPredicate, String allViewExpandStmtListStr) { this.latestTable = latestTable; this.olapTable = latestTable.olapTable; this.partitionInfo = partitionInfo; this.partColumn = partColumn; this.partitionPredicate = partitionPredicate; this.newRangeList = Lists.newArrayList(); + this.allViewExpandStmtListStr = allViewExpandStmtListStr; } public InternalService.PFetchCacheResult getCacheData(Status status) { @@ -84,8 +85,10 @@ public class PartitionCache extends Cache { status.setStatus("analytics range error"); return null; } + + String nokeyStmtWithViewStmt = nokeyStmt.toSql() + allViewExpandStmtListStr; InternalService.PFetchCacheRequest request = InternalService.PFetchCacheRequest.newBuilder() - .setSqlKey(CacheProxy.getMd5(nokeyStmt.toSql())) + .setSqlKey(CacheProxy.getMd5(nokeyStmtWithViewStmt)) .addAllParams(range.getPartitionSingleList().stream().map( p -> InternalService.PCacheParam.newBuilder() .setPartitionKey(p.getCacheKey().realValue()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index 5d1d6d9..9485547 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -35,13 +35,15 @@ public class SqlCache extends Cache { super(queryId, selectStmt); } - public void setCacheInfo(CacheAnalyzer.CacheTable latestTable) { + public void setCacheInfo(CacheAnalyzer.CacheTable latestTable, String allViewExpandStmtListStr) { this.latestTable = latestTable; + this.allViewExpandStmtListStr = allViewExpandStmtListStr; } public InternalService.PFetchCacheResult getCacheData(Status status) { + String originStmtWithViewStmt = selectStmt.getOrigStmt().originStmt + allViewExpandStmtListStr; InternalService.PFetchCacheRequest request = InternalService.PFetchCacheRequest.newBuilder() - .setSqlKey(CacheProxy.getMd5(selectStmt.getOrigStmt().originStmt)) + .setSqlKey(CacheProxy.getMd5(originStmtWithViewStmt)) .addParams(InternalService.PCacheParam.newBuilder() .setPartitionKey(latestTable.latestPartitionId) .setLastVersion(latestTable.latestVersion) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
