This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch dev-1.0.1 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit f0cac16b31b5a76bf96ebc1335675bafada0c2b2 Author: ZenoYang <[email protected]> AuthorDate: Sat Apr 23 21:37:34 2022 +0800 [fix](cache) Generate md5 value using utf8 encoding for sqlkey string (#9121) --- .../java/org/apache/doris/qe/cache/CacheProxy.java | 5 ++++- .../java/org/apache/doris/qe/cache/SqlCache.java | 6 ++++++ .../org/apache/doris/qe/PartitionCacheTest.java | 25 ++++++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java index f9664d2664..54438a9db7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java @@ -25,6 +25,7 @@ import org.apache.doris.proto.Types; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; /** @@ -67,13 +68,15 @@ public abstract class CacheProxy { public static Types.PUniqueId getMd5(String str) { MessageDigest msgDigest; + final byte[] digest; try { //128 bit msgDigest = MessageDigest.getInstance("MD5"); + digest = msgDigest.digest(str.getBytes(StandardCharsets.UTF_8)); } catch (Exception e) { return null; } - final byte[] digest = msgDigest.digest(str.getBytes()); + Types.PUniqueId key = Types.PUniqueId.newBuilder() .setLo(getLongFromByte(digest, 0)) .setHi(getLongFromByte(digest, 8)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index a9ba2912a0..64b4d95f6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -22,6 +22,7 @@ import org.apache.doris.common.Status; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.metric.MetricRepo; import org.apache.doris.proto.InternalService; +import org.apache.doris.proto.Types; import org.apache.doris.qe.RowBatch; import org.apache.doris.thrift.TUniqueId; @@ -44,6 +45,11 @@ public class SqlCache extends Cache { return selectStmt.toSql() + "|" + allViewExpandStmtListStr; } + // only used for UT + public Types.PUniqueId getSqlKey() { + return CacheProxy.getMd5(getSqlWithViewStmt()); + } + public InternalService.PFetchCacheResult getCacheData(Status status) { InternalService.PFetchCacheRequest request = InternalService.PFetchCacheRequest.newBuilder() .setSqlKey(CacheProxy.getMd5(getSqlWithViewStmt())) diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java index 342c876a2f..414dc28bb1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java @@ -60,6 +60,7 @@ import org.apache.doris.planner.ScanNode; import org.apache.doris.proto.Types; import org.apache.doris.qe.cache.Cache; import org.apache.doris.qe.cache.CacheAnalyzer; +import org.apache.doris.qe.cache.CacheProxy; import org.apache.doris.qe.cache.CacheAnalyzer.CacheMode; import org.apache.doris.qe.cache.CacheCoordinator; import org.apache.doris.qe.cache.PartitionCache; @@ -393,14 +394,15 @@ public class PartitionCacheTest { } /** - * table appevent(date(pk), userid, eventid, eventtime), stream load every 5 miniutes + * table appevent(date(pk), userid, eventid, eventtime, city), stream load every 5 miniutes */ private OlapTable createEventTable() { Column column1 = new Column("eventdate", ScalarType.DATE); Column column2 = new Column("userid", ScalarType.INT); Column column3 = new Column("eventid", ScalarType.INT); Column column4 = new Column("eventtime", ScalarType.DATETIME); - List<Column> columns = Lists.newArrayList(column1, column2, column3, column4); + Column column5 = new Column("city", ScalarType.VARCHAR); + List<Column> columns = Lists.newArrayList(column1, column2, column3, column4, column5); PartitionInfo partInfo = new RangePartitionInfo(Lists.newArrayList(column1)); MaterializedIndex baseIndex = new MaterializedIndex(30001, IndexState.NORMAL); RandomDistributionInfo distInfo = new RandomDistributionInfo(10); @@ -1022,6 +1024,25 @@ public class PartitionCacheTest { ">= '2020-01-12 00:00:00' AND `eventdate` <= '2020-01-14 00:00:00' GROUP BY `eventdate`|"); } + @Test + public void testSqlCacheKeyWithChineseChar() { + Catalog.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" and city=\"北京\" GROUP BY eventdate" + ); + ArrayList<Long> selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Types.PUniqueId sqlKey2 = CacheProxy.getMd5(cacheKey.replace("北京", "上海")); + Assert.assertNotEquals(sqlCache.getSqlKey(), sqlKey2); + } + @Test public void testSqlCacheKeyWithView() { Catalog.getCurrentSystemInfo(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
