Updated Branches: refs/heads/apache-blur-0.2 103ac9785 -> 0756d357d
Fixing a bug where the last modified time of the file was not taken into accoount. Therefore if a table was destroyed and recreated the files present in the cache could have been used into of the real files in the base directory. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ddf48ff6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ddf48ff6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ddf48ff6 Branch: refs/heads/apache-blur-0.2 Commit: ddf48ff6b465a8eaf02022c8e28a35ae8e73ac61 Parents: 103ac97 Author: Aaron McCurry <[email protected]> Authored: Sat Feb 8 14:20:49 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sat Feb 8 14:20:49 2014 -0500 ---------------------------------------------------------------------- .../blur/store/blockcache_v2/BaseCache.java | 27 ++++++++++++++++++-- .../apache/blur/store/blockcache_v2/Cache.java | 13 ++++++++++ .../store/blockcache_v2/CacheIndexOutput.java | 1 + .../blur/store/hdfs_v2/JoinDirectory.java | 9 +++---- .../blockcache_v2/CacheIndexInputTest.java | 5 ++++ 5 files changed, 48 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java index bcb279c..796f59d 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java @@ -27,6 +27,7 @@ import static org.apache.blur.metrics.MetricsConstants.REMOVAL; import static org.apache.blur.metrics.MetricsConstants.SIZE; import java.io.Closeable; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashSet; import java.util.Iterator; @@ -193,9 +194,31 @@ public class BaseCache extends Cache implements Closeable { _fileNameToId.remove(cachedFileName); } + @Override + public void fileClosedForWriting(CacheDirectory directory, String fileName, long fileId) throws IOException { + if (directory.fileExists(fileName)) { + long fileModified = directory.getFileModified(fileName); + FileIdKey oldKey = new FileIdKey(directory.getDirectoryName(), fileName, -1L); + FileIdKey newKey = new FileIdKey(directory.getDirectoryName(), fileName, fileModified); + long currentFileId = _fileNameToId.get(oldKey); + if (fileId != currentFileId) { + throw new IOException("Something has gone very wrong file ids do not match [" + fileId + "] [" + currentFileId + + "] for key [" + oldKey + "]"); + } + _fileNameToId.put(newKey, fileId); + _oldFileNameIdMap.put(fileId, newKey); + _fileNameToId.remove(oldKey); + } else { + throw new FileNotFoundException("File [" + fileName + "] not found in directory [" + directory + "]"); + } + } + private FileIdKey getCacheFileName(CacheDirectory directory, String fileName) throws IOException { - long fileModified = directory.getFileModified(fileName); - return new FileIdKey(directory.getDirectoryName(), fileName, fileModified); + if (directory.fileExists(fileName)) { + long fileModified = directory.getFileModified(fileName); + return new FileIdKey(directory.getDirectoryName(), fileName, fileModified); + } + return new FileIdKey(directory.getDirectoryName(), fileName, -1L); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java index 1dddd75..402c55c 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java @@ -173,4 +173,17 @@ public abstract class Cache implements Closeable { */ public abstract boolean shouldBeQuiet(CacheDirectory directory, String fileName); + /** + * The cache internals rely on the last modified timestamp of a given file to + * know if the file is the same or not. During the writing of a given file the + * last moified date is not know, so this method tells the cache that the file + * has completed the writing phase and the last modified time should now be + * accurate. + * + * @param fileId + * the file id. + * @throws IOException + */ + public abstract void fileClosedForWriting(CacheDirectory directory, String fileName, long fileId) throws IOException; + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java index dd0dc29..caad489 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java @@ -129,6 +129,7 @@ public class CacheIndexOutput extends IndexOutput { _indexOutput.flush(); _indexOutput.close(); _store.putBuffer(_buffer); + _cache.fileClosedForWriting(_directory, _fileName, _fileId); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java index 236c7e5..3fba841 100644 --- a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java +++ b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java @@ -150,11 +150,10 @@ public class JoinDirectory extends Directory implements LastModified { @Override public long getFileModified(String name) throws IOException { - return 0; - // if (_shortTermStorage.fileExists(name)) { - // return ((LastModified) _shortTermStorage).getFileModified(name); - // } - // return ((LastModified) _longTermStorage).getFileModified(name); + if (_shortTermStorage.fileExists(name)) { + return ((LastModified) _shortTermStorage).getFileModified(name); + } + return ((LastModified) _longTermStorage).getFileModified(name); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java index ee4b1bb..36adfcc 100644 --- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java +++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java @@ -256,6 +256,11 @@ public class CacheIndexInputTest { } + @Override + public void fileClosedForWriting(CacheDirectory directory, String fileName, long fileId) throws IOException { + + } + }; return cacheFactory; }
