This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4cc0189 [SQL Cache] Add all view stmt as the suffix of cache sqlkey
(#6832)
4cc0189 is described below
commit 4cc01892f6e32a0b3251a4bc50c58e933b928051
Author: Zeno Yang <[email protected]>
AuthorDate: Sat Oct 16 21:56:24 2021 +0800
[SQL Cache] Add all view stmt as the suffix of cache sqlkey (#6832)
Use all view stmt as the cache sqlkey suffix, so that when the view is
modified, the cache can recognize.
---
.../org/apache/doris/analysis/InlineViewRef.java | 8 ++++
.../main/java/org/apache/doris/qe/cache/Cache.java | 1 +
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 43 +++++++++++++++++++++-
.../org/apache/doris/qe/cache/PartitionCache.java | 7 +++-
.../java/org/apache/doris/qe/cache/SqlCache.java | 6 ++-
5 files changed, 59 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
index cd8b045..9eb5353 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
@@ -418,6 +418,14 @@ public class InlineViewRef extends TableRef {
return baseTblSmap;
}
+ public boolean isLocalView() {
+ return view == null || view.isLocalView();
+ }
+
+ public View getView() {
+ return view;
+ }
+
@Override
public String tableRefToSql() {
// Enclose the alias in quotes if Hive cannot parse it without quotes.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
index 1711179..6d80556 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
@@ -43,6 +43,7 @@ public abstract class Cache {
protected CacheAnalyzer.CacheTable latestTable;
protected CacheProxy proxy;
protected HitRange hitRange;
+ protected String allViewExpandStmtListStr;
protected Cache(TUniqueId queryId, SelectStmt selectStmt) {
this.queryId = queryId;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
index d8a6f47..e651d00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
@@ -17,6 +17,7 @@
package org.apache.doris.qe.cache;
+import org.apache.commons.lang3.StringUtils;
import org.apache.doris.analysis.AggregateInfo;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.CastExpr;
@@ -25,6 +26,7 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.InlineViewRef;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SelectStmt;
+import org.apache.doris.analysis.SetOperationStmt;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TableRef;
@@ -33,6 +35,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.RangePartitionInfo;
+import org.apache.doris.catalog.View;
import org.apache.doris.common.Config;
import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
@@ -51,8 +54,11 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* Analyze which caching mode a SQL is suitable for
@@ -87,6 +93,7 @@ public class CacheAnalyzer {
private Column partColumn;
private CompoundPredicate partitionPredicate;
private Cache cache;
+ private Set<String> allViewStmtSet;
public Cache getCache() {
return cache;
@@ -98,6 +105,7 @@ public class CacheAnalyzer {
this.parsedStmt = parsedStmt;
scanNodes = planner.getScanNodes();
latestTable = new CacheTable();
+ allViewStmtSet = new HashSet<>();
checkCacheConfig();
}
@@ -106,6 +114,7 @@ public class CacheAnalyzer {
this.context = context;
this.parsedStmt = parsedStmt;
this.scanNodes = scanNodes;
+ allViewStmtSet = new HashSet<>();
checkCacheConfig();
}
@@ -209,6 +218,9 @@ public class CacheAnalyzer {
latestTable = tblTimeList.get(0);
latestTable.Debug();
+ addAllViewStmt(selectStmt);
+ String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet,
",");
+
if (now == 0) {
now = nowtime();
}
@@ -216,7 +228,7 @@ public class CacheAnalyzer {
(now - latestTable.latestTime) >=
Config.cache_last_version_interval_second * 1000) {
LOG.debug("TIME:{},{},{}", now, latestTable.latestTime,
Config.cache_last_version_interval_second*1000);
cache = new SqlCache(this.queryId, this.selectStmt);
- ((SqlCache) cache).setCacheInfo(this.latestTable);
+ ((SqlCache) cache).setCacheInfo(this.latestTable,
allViewExpandStmtListStr);
MetricRepo.COUNTER_CACHE_MODE_SQL.increase(1L);
return CacheMode.Sql;
}
@@ -263,7 +275,7 @@ public class CacheAnalyzer {
partitionPredicate = compoundPredicates.get(0);
cache = new PartitionCache(this.queryId, this.selectStmt);
((PartitionCache) cache).setCacheInfo(this.latestTable,
this.partitionInfo, this.partColumn,
- this.partitionPredicate);
+ this.partitionPredicate, allViewExpandStmtListStr);
MetricRepo.COUNTER_CACHE_MODE_PARTITION.increase(1L);
return CacheMode.Partition;
}
@@ -430,6 +442,33 @@ public class CacheAnalyzer {
return table;
}
+ private void addAllViewStmt(List<TableRef> tblRefs) {
+ for (TableRef tblRef : tblRefs) {
+ if (tblRef instanceof InlineViewRef) {
+ InlineViewRef inlineViewRef = (InlineViewRef) tblRef;
+ if (inlineViewRef.isLocalView()) {
+ Collection<View> views =
inlineViewRef.getAnalyzer().getLocalViews().values();
+ for (View view : views) {
+ addAllViewStmt(view.getQueryStmt());
+ }
+ } else {
+ addAllViewStmt(inlineViewRef.getViewStmt());
+
allViewStmtSet.add(inlineViewRef.getView().getInlineViewDef());
+ }
+ }
+ }
+ }
+
+ private void addAllViewStmt(QueryStmt queryStmt) {
+ if (queryStmt instanceof SelectStmt) {
+ addAllViewStmt(((SelectStmt) queryStmt).getTableRefs());
+ } else if (queryStmt instanceof SetOperationStmt) {
+ for (SetOperationStmt.SetOperand operand : ((SetOperationStmt)
queryStmt).getOperands()) {
+ addAllViewStmt(((SelectStmt)
operand.getQueryStmt()).getTableRefs());
+ }
+ }
+ }
+
public Cache.HitRange getHitRange() {
if (cacheMode == CacheMode.None) {
return Cache.HitRange.None;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
index d208d77..1ff1fdb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
@@ -66,13 +66,14 @@ public class PartitionCache extends Cache {
}
public void setCacheInfo(CacheAnalyzer.CacheTable latestTable,
RangePartitionInfo partitionInfo, Column partColumn,
- CompoundPredicate partitionPredicate) {
+ CompoundPredicate partitionPredicate, String
allViewExpandStmtListStr) {
this.latestTable = latestTable;
this.olapTable = latestTable.olapTable;
this.partitionInfo = partitionInfo;
this.partColumn = partColumn;
this.partitionPredicate = partitionPredicate;
this.newRangeList = Lists.newArrayList();
+ this.allViewExpandStmtListStr = allViewExpandStmtListStr;
}
public InternalService.PFetchCacheResult getCacheData(Status status) {
@@ -84,8 +85,10 @@ public class PartitionCache extends Cache {
status.setStatus("analytics range error");
return null;
}
+
+ String nokeyStmtWithViewStmt = nokeyStmt.toSql() +
allViewExpandStmtListStr;
InternalService.PFetchCacheRequest request =
InternalService.PFetchCacheRequest.newBuilder()
- .setSqlKey(CacheProxy.getMd5(nokeyStmt.toSql()))
+ .setSqlKey(CacheProxy.getMd5(nokeyStmtWithViewStmt))
.addAllParams(range.getPartitionSingleList().stream().map(
p -> InternalService.PCacheParam.newBuilder()
.setPartitionKey(p.getCacheKey().realValue())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
index 5d1d6d9..9485547 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
@@ -35,13 +35,15 @@ public class SqlCache extends Cache {
super(queryId, selectStmt);
}
- public void setCacheInfo(CacheAnalyzer.CacheTable latestTable) {
+ public void setCacheInfo(CacheAnalyzer.CacheTable latestTable, String
allViewExpandStmtListStr) {
this.latestTable = latestTable;
+ this.allViewExpandStmtListStr = allViewExpandStmtListStr;
}
public InternalService.PFetchCacheResult getCacheData(Status status) {
+ String originStmtWithViewStmt = selectStmt.getOrigStmt().originStmt +
allViewExpandStmtListStr;
InternalService.PFetchCacheRequest request =
InternalService.PFetchCacheRequest.newBuilder()
-
.setSqlKey(CacheProxy.getMd5(selectStmt.getOrigStmt().originStmt))
+ .setSqlKey(CacheProxy.getMd5(originStmtWithViewStmt))
.addParams(InternalService.PCacheParam.newBuilder()
.setPartitionKey(latestTable.latestPartitionId)
.setLastVersion(latestTable.latestVersion)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
