Ori.livneh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/240316
Change subject: Improved caching for LESS file compilation
......................................................................
Improved caching for LESS file compilation
Caching the output of a LESS compiler is tricky, because a LESS file may
include additional LESS files via @imports, in which case the cache needs
to vary as the contents of those files vary (and not just the contents of
the primary LESS file).
To solve this, we first introduce a utility class, FileContentsHasher. This
class is essentially a smart version of md5_file() -- given one or more file
names, it computes a hash digest of their contents. It tries to avoid
re-reading files by caching the hash digest in APC and re-using it as long as
the files' mtimes have not changed. This is the same approach I used in
I5ceb8537c.
Next, we use this class in ResourceLoaderFileModule in the following way:
whenever we compile a LESS file, we cache the result as an associative array
with the following keys:
* `files` : the list of files whose contents influenced the compiled CSS.
* `hash` : a hash digest of the combined contents of those files.
* `css` : the CSS output of the compiler itself.
Before using a cached value, we verify that it is still current by asking
FileContentHasher for a hash of the combined contents of all referenced files,
and we compare that against the value of the `hash` key of the cached entry.
Bug: T112035
Change-Id: I1ff61153ddb95ed17e543bd4af7dd13fa3352861
---
M autoload.php
A includes/FileContentsHasher.php
M includes/resourceloader/ResourceLoaderFileModule.php
3 files changed, 125 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/16/240316/1
diff --git a/autoload.php b/autoload.php
index 4bed014..4ad921a 100644
--- a/autoload.php
+++ b/autoload.php
@@ -436,6 +436,7 @@
'FileBackendStoreShardListIterator' => __DIR__ .
'/includes/filebackend/FileBackendStore.php',
'FileBasedSiteLookup' => __DIR__ .
'/includes/site/FileBasedSiteLookup.php',
'FileCacheBase' => __DIR__ . '/includes/cache/FileCacheBase.php',
+ 'FileContentsHasher' => __DIR__ . '/includes/FileContentsHasher.php',
'FileDeleteForm' => __DIR__ . '/includes/FileDeleteForm.php',
'FileDependency' => __DIR__ . '/includes/cache/CacheDependency.php',
'FileDuplicateSearchPage' => __DIR__ .
'/includes/specials/SpecialFileDuplicateSearch.php',
diff --git a/includes/FileContentsHasher.php b/includes/FileContentsHasher.php
new file mode 100644
index 0000000..4d20059
--- /dev/null
+++ b/includes/FileContentsHasher.php
@@ -0,0 +1,90 @@
+<?php
+/**
+ * Generate hash digests of file contents to help with cache invalidation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+class FileContentsHasher {
+
+ /** @var BagOStuff */
+ protected $cache;
+
+ /**
+ * Constructor.
+ *
+ * @param BagOStuff $cache Cache object to use for storing computed
hashes.
+ */
+ public function __construct( BagOStuff $cache = null ) {
+ $this->cache = $cache ?: ObjectCache::newAccelerator(
CACHE_ANYTHING );
+ }
+
+ /**
+ * Get a hash of a file's contents, either by retrieving a previously-
+ * computed hash from the cache, or by computing a hash from the file.
+ *
+ * @param string $filePath Full path to the file.
+ * @param string $algo Name of selected hashing algorithm.
+ * @return string|bool Hash of file contents, or false if the file
could not be read.
+ */
+ public function getFileContentsHash( $filePath, $algo = 'md4' ) {
+ $mtime = filemtime( $filePath );
+ if ( $mtime === false ) {
+ return false;
+ }
+
+ $cacheKey = wfGlobalCacheKey( __CLASS__, $filePath, $algo );
+ $cachedHash = $this->cache->get( $cacheKey );
+
+ if ( isset( $cachedHash['mtime'] ) && $cachedHash['mtime'] >=
$mtime ) {
+ return $cachedHash['hash'];
+ }
+
+ $contents = file_get_contents( $filePath );
+ if ( $contents === false ) {
+ return false;
+ }
+
+ $hash = hash( $algo, $contents );
+ $cache->set( $cacheKey, array(
+ 'mtime' => $mtime,
+ 'hash' => $hash
+ ), 60 * 60 * 24 );
+
+ return $hash;
+ }
+
+ /**
+ * Get a hash of the combined contents of multiple files, either by
+ * retrieving a previously-computed hash from the cache, or by computing
+ * a hash from the files.
+ *
+ * @param array $filePaths Array of paths to files.
+ * @param string $algo Name of selected hashing algorithm.
+ * @return string|bool Hash of files' contents, or false if no file
could not be read.
+ */
+ public function getFilesContentHash( array $filePaths, $algo = 'md4' ) {
+ sort( $filePaths );
+ $that = $this;
+ $hashes = array_map( function ( $filePath ) use ( $algo, $that
) {
+ return $that->getFileContentsHash( $filePath, $algo )
?: '';
+ }, $filePaths );
+
+ $hashes = implode( '', $hashes );
+ return $hashes ? hash( $algo, $hashes ) : false;
+ }
+}
diff --git a/includes/resourceloader/ResourceLoaderFileModule.php
b/includes/resourceloader/ResourceLoaderFileModule.php
index 7fbc1cb..dbc4dad 100644
--- a/includes/resourceloader/ResourceLoaderFileModule.php
+++ b/includes/resourceloader/ResourceLoaderFileModule.php
@@ -966,12 +966,43 @@
* @return string CSS source
*/
protected function compileLessFile( $fileName, $compiler = null ) {
+ static $cache, $hasher;
+
+ if ( !$cache ) {
+ $cache = ObjectCache::newAccelerator( CACHE_ANYTHING );
+ $hasher = new FileContentsHasher( $cache );
+ }
+
+ // Construct a cache key from the LESS file name and a hash
digest
+ // of the LESS variables used for compilation.
+ $varsHash = md5( serialize( ResourceLoader::getLessVars(
$this->getConfig() ) ) );
+ $cacheKey = wfGlobalCacheKey( 'LESS', $fileName, $varsHash );
+ $cachedCompile = $cache->get( $cacheKey );
+
+ // If we got a cached value, we have to validate it by getting a
+ // checksum of all the files that were loaded by the parser and
+ // ensuring it matches the cached entry's.
+ if ( isset( $cachedCompile['hash'] ) ) {
+ $contentHash = $hasher->getFilesContentHash(
$cachedCompile['files'] );
+ if ( $contentHash === $cachedCompile['hash'] ) {
+ return $cachedCompile['css'];
+ }
+ }
+
if ( !$compiler ) {
$compiler = $this->getLessCompiler();
}
- $result = $compiler->parseFile( $fileName )->getCss();
- $this->localFileRefs += array_keys( $compiler->AllParsedFiles()
);
- return $result;
+
+ $css = $compiler->parseFile( $fileName )->getCss();
+ $files = array_keys( $compiler->AllParsedFiles() );
+ $this->localFileRefs += $files;
+ $cache->set( $cacheKey, array(
+ 'css' => $css,
+ 'files' => $files,
+ 'hash' => $hasher->getFilesContentHash( $files ),
+ ), 60 * 60 * 24 );
+
+ return $css;
}
/**
--
To view, visit https://gerrit.wikimedia.org/r/240316
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1ff61153ddb95ed17e543bd4af7dd13fa3352861
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Ori.livneh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
