Mgautierfr has uploaded a new change for review.
https://gerrit.wikimedia.org/r/315237
Change subject: Do not cache uncompressed cluster by default.
......................................................................
Do not cache uncompressed cluster by default.
Uncompressed cluster can be big (if they contain videos for example).
We should not cache them by default before having a cache system who
limits itself in memory used.
Change-Id: If307af3f91a614b943fa408b2bf30e2016ebfe81
---
M zimlib/include/zim/fileimpl.h
M zimlib/src/fileimpl.cpp
2 files changed, 10 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/openzim refs/changes/37/315237/1
diff --git a/zimlib/include/zim/fileimpl.h b/zimlib/include/zim/fileimpl.h
index 1cf584d..ddf7bc1 100644
--- a/zimlib/include/zim/fileimpl.h
+++ b/zimlib/include/zim/fileimpl.h
@@ -41,6 +41,7 @@
Cache<size_type, Dirent> direntCache;
Cache<offset_type, Cluster> clusterCache;
+ bool cacheUncompressedCluster;
typedef std::map<char, size_type> NamespaceCache;
NamespaceCache namespaceBeginCache;
NamespaceCache namespaceEndCache;
diff --git a/zimlib/src/fileimpl.cpp b/zimlib/src/fileimpl.cpp
index 2aa9e1f..a1b4eee 100644
--- a/zimlib/src/fileimpl.cpp
+++ b/zimlib/src/fileimpl.cpp
@@ -41,7 +41,8 @@
FileImpl::FileImpl(const char* fname)
: zimFile(fname),
direntCache(envValue("ZIM_DIRENTCACHE", DIRENT_CACHE_SIZE)),
- clusterCache(envValue("ZIM_CLUSTERCACHE", CLUSTER_CACHE_SIZE))
+ clusterCache(envValue("ZIM_CLUSTERCACHE", CLUSTER_CACHE_SIZE)),
+ cacheUncompressedCluster(envValue("ZIM_CACHEUNCOMPRESSEDCLUSTER", false))
{
log_trace("read file \"" << fname << '"');
@@ -181,8 +182,13 @@
if (zimFile.fail())
throw ZimFileFormatError("error reading cluster data");
- log_debug("put cluster " << idx << " into cluster cache; hits " <<
clusterCache.getHits() << " misses " << clusterCache.getMisses() << " ratio "
<< clusterCache.hitRatio() * 100 << "% fillfactor " <<
clusterCache.fillfactor());
- clusterCache.put(idx, cluster);
+ if (cacheUncompressedCluster || cluster.isCompressed())
+ {
+ log_debug("put cluster " << idx << " into cluster cache; hits " <<
clusterCache.getHits() << " misses " << clusterCache.getMisses() << " ratio "
<< clusterCache.hitRatio() * 100 << "% fillfactor " <<
clusterCache.fillfactor());
+ clusterCache.put(idx, cluster);
+ }
+ else
+ log_debug("cluster " << idx << " is not compressed - do not cache");
return cluster;
}
--
To view, visit https://gerrit.wikimedia.org/r/315237
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If307af3f91a614b943fa408b2bf30e2016ebfe81
Gerrit-PatchSet: 1
Gerrit-Project: openzim
Gerrit-Branch: master
Gerrit-Owner: Mgautierfr <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
