[kylin] branch master updated: KYLIN-3678 CacheStateChecker may remove a cache file that under building

[shaofengshi]

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new 8e1fd97  KYLIN-3678 CacheStateChecker may remove a cache file that 
under building
8e1fd97 is described below

commit 8e1fd97321d8c64adb328511bbd53e6d529571ab
Author: shaofengshi <shaofeng...@apache.org>
AuthorDate: Fri Nov 9 21:34:11 2018 +0800

    KYLIN-3678 CacheStateChecker may remove a cache file that under building
---
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java    | 4 ++++
 .../apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java   | 6 ++++--
 .../kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java      | 8 ++++++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 149bb49..ad51994 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -461,6 +461,10 @@ abstract public class KylinConfigBase implements 
Serializable {
         return 
Double.parseDouble(getOptional("kylin.snapshot.ext.local.cache.max-size-gb", 
"200"));
     }
 
+    public long getExtTableSnapshotLocalCacheCheckVolatileRange() {
+        return 
Long.parseLong(getOptional("kylin.snapshot.ext.local.cache.check.volatile", 
"3600000"));
+    }
+
     public boolean isShrunkenDictFromGlobalEnabled() {
         return 
Boolean.parseBoolean(this.getOptional("kylin.dictionary.shrunken-from-global-enabled",
 FALSE));
     }
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
index 559c435..c748e18 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
@@ -352,8 +352,10 @@ public class RocksDBLookupTableCache implements 
IExtLookupTableCache {
                 List<Pair<String, File>> toRemovedCachedSnapshots = 
Lists.newArrayList(FluentIterable.from(
                         allCachedSnapshots).filter(new Predicate<Pair<String, 
File>>() {
                     @Override
-                    public boolean apply(@Nullable Pair<String, File> input) {
-                        return !activeSnapshotSet.contains(input.getFirst());
+                            public boolean apply(@Nullable Pair<String, File> 
input) {
+                                long lastModified = 
input.getSecond().lastModified();
+                                return 
!activeSnapshotSet.contains(input.getFirst()) && lastModified > 0
+                                        && lastModified < 
(System.currentTimeMillis() - 
config.getExtTableSnapshotLocalCacheCheckVolatileRange());
                     }
                 }));
                 for (Pair<String, File> toRemovedCachedSnapshot : 
toRemovedCachedSnapshots) {
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
index eb66955..fa430fa 100644
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
@@ -153,7 +153,15 @@ public class RocksDBLookupTableCacheTest extends 
LocalFileMetadataTestCase {
         
ExtTableSnapshotInfoManager.getInstance(kylinConfig).removeSnapshot(snapshotInfo.getTableName(),
 snapshotInfo.getId());
         cache.checkCacheState();
         String cacheLocalPath = 
cache.getSnapshotCachePath(snapshotInfo.getTableName(), snapshotInfo.getId());
+        // won't cleanup because it is newly created in last 1 hour
+        assertTrue(new File(cacheLocalPath).exists());
+
+        // change the volatile value
+        
kylinConfig.setProperty("kylin.snapshot.ext.local.cache.check.volatile", "0");
+        cache.checkCacheState();
+        // this time it should be removed.
         assertFalse(new File(cacheLocalPath).exists());
+
         cachedLookupTable = cache.getCachedLookupTable(tableDesc, 
snapshotInfo, false);
         assertNull(cachedLookupTable);
     }