https://www.mediawiki.org/wiki/Special:Code/MediaWiki/115597
Revision: 115597 Author: adamw Date: 2012-07-08 04:57:18 +0000 (Sun, 08 Jul 2012) Log Message: ----------- configurable filename munging in DumpHTML See https://bugzilla.wikimedia.org/show_bug.cgi?id=8147 Modified Paths: -------------- trunk/extensions/DumpHTML/dumpHTML.inc trunk/extensions/DumpHTML/dumpHTML.php Added Paths: ----------- trunk/extensions/DumpHTML/MungeTitle.inc Added: trunk/extensions/DumpHTML/MungeTitle.inc =================================================================== --- trunk/extensions/DumpHTML/MungeTitle.inc (rev 0) +++ trunk/extensions/DumpHTML/MungeTitle.inc 2012-07-08 04:57:18 UTC (rev 115597) @@ -0,0 +1,125 @@ +<?php + +// see https://bugzilla.wikimedia.org/show_bug.cgi?id=8147 + +class MungeTitle +{ + function __construct($type = FALSE) + { + if (!$type) + $type = 'default'; + + $fun = array('MungeTitle', 'munge' . ucfirst($type)); + if (is_callable($fun)) { + $this->method = $fun; + } + else + { + throw new Exception("no such titlemunger exists: {$type}"); + } + } + + function munge($path) + { + return call_user_func($this->method, $path); + } + + function getMethod() + { + return get_class($this) . "::" . $this->method[1]; + } + + static function mungeDefault($path) + { + $path = self::truncate_unique($path); + $path = self::getHashedDirectory($path, 3)."/".$path; + return $path; + } + + static function mungeMd5($path) + { + + $path = urldecode($path); + $match = array(); + preg_match("!(?:[^./]+)(?<ext>[.][^./]+)$!", $path, $match); + $path = md5($path) . $match['ext']; + $path = self::getHashedDirectory($path, 3)."/".substr($path, 3); + + return $path; + } + + static function mungeWindows( $name ) { + + global $wgLang; + # Replace illegal characters for Windows paths with underscores + $friendlyName = strtr( $name, '/\\*?"<>|~', '_________' ); + + # Work out lower case form. We assume we're on a system with case-insensitive + # filenames, so unless the case is of a special form, we have to disambiguate + if ( function_exists( 'mb_strtolower' ) ) { + $lowerCase = $wgLang->ucfirst( mb_strtolower( $name ) ); + } else { + $lowerCase = ucfirst( strtolower( $name ) ); + } + + # Handle colon specially by replacing it with tilde + # Thus we reduce the number of paths with hashes appended + $friendlyName = str_replace( ':', '~', $friendlyName ); + $friendlyName = self::truncate_unique($friendlyName); + $friendlyName = self::getHashedDirectory($friendlyName, 3)."/".$friendlyName; + + return $friendlyName; + } + + static function truncate_unique($filename) + { + $max_length = 255 - 5; // .html + if (strlen($filename) > $max_length) + { + # Make it mostly unique + $filename = substr($filename, 0, $max_length) . '_' . substr(md5( $filename ), 0, 4); + } + return $filename; + } + + static function getHashedDirectory( &$filename, $depth ) + { + # Find the first colon if there is one, use characters after it + $p = strpos( $filename, ':' ); + if ( $p !== false ) { + $dbk = substr( $filename, $p + 1 ); + $dbk = substr( $dbk, strspn( $dbk, '_' ) ); + } else { + $dbk = $filename; + } + + # Split into characters + $m = array(); + preg_match_all( '/./us', $dbk, $m ); + + $chars = $m[0]; + $length = count( $chars ); + $dir = ''; + + for ( $i = 0; $i < $depth; $i++ ) { + if ( $i ) { + $dir .= '/'; + } + if ( $i >= $length ) { + $dir .= '_'; + } else { + $c = $chars[$i]; + if ( ord( $c ) >= 128 || preg_match( '/[a-zA-Z0-9!#$%&()+,[\]^_`{}-]/', $c ) ) { + if ( function_exists( 'mb_strtolower' ) ) { + $dir .= mb_strtolower( $c ); + } else { + $dir .= strtolower( $c ); + } + } else { + $dir .= sprintf( "%02X", ord( $c ) ); + } + } + } + return $dir; + } +} Modified: trunk/extensions/DumpHTML/dumpHTML.inc =================================================================== --- trunk/extensions/DumpHTML/dumpHTML.inc 2012-07-07 19:34:28 UTC (rev 115596) +++ trunk/extensions/DumpHTML/dumpHTML.inc 2012-07-08 04:57:18 UTC (rev 115597) @@ -89,6 +89,7 @@ foreach ( $settings as $var => $value ) { $this->$var = $value; } + $this->mungeTitle = new MungeTitle($this->mungeTitle); $this->extdir = dirname( __FILE__ ); } @@ -245,6 +246,16 @@ print "\n"; } + function onParserAfterTidy($parser, &$text) + { + // TODO it would be much nicer to do this during page generation + # post-process links because "File:" linking doesn't expose hooks. + $munge = create_function('$m', 'return $m["pre"].'.$this->mungeTitle->getMethod().'($m["rel"]).$m["post"];'); + $text = preg_replace_callback('!(?<pre>src="[^"]*/?images/(archive|thumb|))(?<rel>[^"]+)(?<post>["])!Ui', $munge, $text); + // TODO image description page, thumbnails + return true; + } + function doImageDescriptions() { $this->doLocalImageDescriptions(); if ( !$this->noSharedDesc ) { @@ -517,19 +528,6 @@ wfProfileIn( __METHOD__ ); $filename = $this->getHashedFilename( $title ); - # Temporary hack for current dump, this should be moved to - # getFriendlyName() at the earliest opportunity. - # - # Limit filename length to 255 characters, so it works on ext3. - # Titles are in fact limited to 255 characters, but dumpHTML - # adds a suffix which may put them over the limit. - $length = strlen( $filename ); - if ( $length > 255 ) { - print "Warning: Filename too long ($length bytes). Skipping.\n"; - wfProfileOut( __METHOD__ ); - return; - } - $fullName = "{$this->dest}/$filename"; $fullDir = dirname( $fullName ); @@ -653,6 +651,7 @@ $wgHooks['GetFullURL'][] =& $this; $wgHooks['SiteNoticeBefore'][] =& $this; $wgHooks['SiteNoticeAfter'][] =& $this; + $wgHooks['ParserAfterTidy'][] =& $this; $this->oldArticlePath = wfExpandURL( $wgServer . $wgArticlePath, PROTO_CANONICAL ); $this->oldLogo = $wgLogo; $this->oldRepoGroup = RepoGroup::singleton(); @@ -975,7 +974,8 @@ if ( $params['gen'] == 'css' || $params['gen'] == 'js' ) { $file = 'gen.' . $params['gen']; } else { - $file = $this->getFriendlyName( $title->getPrefixedDBkey() ); + $file = $title->getPrefixedDBkey(); + $file = $this->mungeTitle->munge($file); // Clean up Monobook.css etc. $matches = array(); if ( preg_match( '/^(.*)\.(css|js)_[0-9a-f]{4}$/', $file, $matches ) ) { @@ -1013,86 +1013,13 @@ return 'index.html'; } - return 'articles/' . $this->getHashedDirectory( $title ) . '/' . - $this->getFriendlyName( $dbkey ) . '.html'; - } + $filename = $title->getPrefixedDBkey(); + $filename = $this->mungeTitle->munge($filename); - function getFriendlyName( $name ) { - global $wgLang; - # Replace illegal characters for Windows paths with underscores - $friendlyName = strtr( $name, '/\\*?"<>|~', '_________' ); - - # Work out lower case form. We assume we're on a system with case-insensitive - # filenames, so unless the case is of a special form, we have to disambiguate - if ( function_exists( 'mb_strtolower' ) ) { - $lowerCase = $wgLang->ucfirst( mb_strtolower( $name ) ); - } else { - $lowerCase = ucfirst( strtolower( $name ) ); - } - - # Make it mostly unique - if ( $lowerCase != $friendlyName ) { - $friendlyName .= '_' . substr(md5( $name ), 0, 4); - } - # Handle colon specially by replacing it with tilde - # Thus we reduce the number of paths with hashes appended - $friendlyName = str_replace( ':', '~', $friendlyName ); - - return $friendlyName; + return "articles/{$filename}.html"; } /** - * Get a relative directory for putting a title into - * - * @param $title Title - */ - function getHashedDirectory( &$title ) { - if ( '' != $title->getInterwiki() ) { - $pdbk = $title->getDBkey(); - } else { - $pdbk = $title->getPrefixedDBkey(); - } - - # Find the first colon if there is one, use characters after it - $p = strpos( $pdbk, ':' ); - if ( $p !== false ) { - $dbk = substr( $pdbk, $p + 1 ); - $dbk = substr( $dbk, strspn( $dbk, '_' ) ); - } else { - $dbk = $pdbk; - } - - # Split into characters - $m = array(); - preg_match_all( '/./us', $dbk, $m ); - - $chars = $m[0]; - $length = count( $chars ); - $dir = ''; - - for ( $i = 0; $i < $this->depth; $i++ ) { - if ( $i ) { - $dir .= '/'; - } - if ( $i >= $length ) { - $dir .= '_'; - } else { - $c = $chars[$i]; - if ( ord( $c ) >= 128 || preg_match( '/[a-zA-Z0-9!#$%&()+,[\]^_`{}-]/', $c ) ) { - if ( function_exists( 'mb_strtolower' ) ) { - $dir .= mb_strtolower( $c ); - } else { - $dir .= strtolower( $c ); - } - } else { - $dir .= sprintf( "%02X", ord( $c ) ); - } - } - } - return $dir; - } - - /** * Calculate the start end end of a job based on the current slice * @param integer $start * @param integer $end @@ -1257,6 +1184,8 @@ return; } + $rel = $this->dump->mungeTitle->munge($rel); + $dest = "{$this->directory}/$rel"; if ( is_callable( array( $this->backend, 'getZonePath' ) ) ) { if ( strpos( $rel, "thumb/" ) === 0 ) { // XXX @@ -1363,8 +1292,12 @@ } $source = $this->file->getPath(); - $dest = $this->repo->directory . '/' . $this->file->getRel(); + $rel = $this->file->getRel(); + $rel = $this->dump->mungeTitle->munge($rel); + + $dest = $this->repo->directory . '/' . $rel; + if ( $this->dump->pathExists( $dest ) ) { return; } Modified: trunk/extensions/DumpHTML/dumpHTML.php =================================================================== --- trunk/extensions/DumpHTML/dumpHTML.php 2012-07-07 19:34:28 UTC (rev 115596) +++ trunk/extensions/DumpHTML/dumpHTML.php 2012-07-08 04:57:18 UTC (rev 115597) @@ -30,6 +30,7 @@ --udp-profile <N> profile 1/N rendering operations using ProfilerSimpleUDP --oom-adj <N> set /proc/<pid>/oom_adj --show-titles write each article title to stdout + --munge-title <HOW> available munging algorithms: none, md5, windows --group <group> use the specified user group to read articles ENDS; @@ -61,6 +62,7 @@ } require_once( $IP."/maintenance/commandLine.inc" ); require_once( dirname(__FILE__)."/dumpHTML.inc" ); +require_once( dirname(__FILE__)."/MungeTitle.inc" ); require_once( dirname(__FILE__)."/SkinOffline.php" ); error_reporting( E_ALL & (~E_NOTICE) ); @@ -126,6 +128,7 @@ 'udpProfile' => $options['udp-profile'], 'showTitles' => $options['show-titles'], 'group' => $options['group'], + 'mungeTitle' => $options['munge-title'], )); $wgHTMLDump->setupDestDir(); _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
