Chad has uploaded a new change for review. https://gerrit.wikimedia.org/r/93622
Change subject: Fix up multiversion to not require dba_* functions ...................................................................... Fix up multiversion to not require dba_* functions - Copies the pure-PHP implementation from MW core with some minor updates to make it work outside of MW - Should allow us to run het deploy on hhvm without waiting on dba_* support upstream Bug: 56591 Change-Id: I73195536a32fc01e0fb4fa1ca376a38837868d17 --- A multiversion/Cdb.php A multiversion/Cdb_PHP.php M multiversion/MWMultiVersion.php M multiversion/getMWVersion M multiversion/refreshWikiversionsCDB 5 files changed, 696 insertions(+), 11 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config refs/changes/22/93622/1 diff --git a/multiversion/Cdb.php b/multiversion/Cdb.php new file mode 100644 index 0000000..a954a44 --- /dev/null +++ b/multiversion/Cdb.php @@ -0,0 +1,180 @@ +<?php +/** + * Native CDB file reader and writer. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Read from a CDB file. + * Native and pure PHP implementations are provided. + * http://cr.yp.to/cdb.html + */ +abstract class CdbReader { + /** + * Open a file and return a subclass instance + * + * @param $fileName string + * + * @return CdbReader + */ + public static function open( $fileName ) { + if ( self::haveExtension() ) { + return new CdbReader_DBA( $fileName ); + } else { + return new CdbReader_PHP( $fileName ); + } + } + + /** + * Returns true if the native extension is available + * + * @return bool + */ + public static function haveExtension() { + if ( !function_exists( 'dba_handlers' ) ) { + return false; + } + $handlers = dba_handlers(); + if ( !in_array( 'cdb', $handlers ) || !in_array( 'cdb_make', $handlers ) ) { + return false; + } + return true; + } + + /** + * Construct the object and open the file + */ + abstract function __construct( $fileName ); + + /** + * Close the file. Optional, you can just let the variable go out of scope. + */ + abstract function close(); + + /** + * Get a value with a given key. Only string values are supported. + * + * @param $key string + */ + abstract public function get( $key ); +} + +/** + * Write to a CDB file. + * Native and pure PHP implementations are provided. + */ +abstract class CdbWriter { + /** + * Open a writer and return a subclass instance. + * The user must have write access to the directory, for temporary file creation. + * + * @param $fileName string + * + * @return CdbWriter_DBA|CdbWriter_PHP + */ + public static function open( $fileName ) { + if ( CdbReader::haveExtension() ) { + return new CdbWriter_DBA( $fileName ); + } else { + return new CdbWriter_PHP( $fileName ); + } + } + + /** + * Create the object and open the file + * + * @param $fileName string + */ + abstract function __construct( $fileName ); + + /** + * Set a key to a given value. The value will be converted to string. + * @param $key string + * @param $value string + */ + abstract public function set( $key, $value ); + + /** + * Close the writer object. You should call this function before the object + * goes out of scope, to write out the final hashtables. + */ + abstract public function close(); +} + +/** + * Reader class which uses the DBA extension + */ +class CdbReader_DBA { + var $handle; + + function __construct( $fileName ) { + $this->handle = dba_open( $fileName, 'r-', 'cdb' ); + if ( !$this->handle ) { + throw new Exception( 'Unable to open CDB file "' . $fileName . '"' ); + } + } + + function close() { + if ( isset( $this->handle ) ) { + dba_close( $this->handle ); + } + unset( $this->handle ); + } + + function get( $key ) { + return dba_fetch( $key, $this->handle ); + } +} + +/** + * Writer class which uses the DBA extension + */ +class CdbWriter_DBA { + var $handle, $realFileName, $tmpFileName; + + function __construct( $fileName ) { + $this->realFileName = $fileName; + $this->tmpFileName = $fileName . '.tmp.' . mt_rand( 0, 0x7fffffff ); + $this->handle = dba_open( $this->tmpFileName, 'n', 'cdb_make' ); + if ( !$this->handle ) { + throw new Exception( 'Unable to open CDB file for write "' . $fileName . '"' ); + } + } + + function set( $key, $value ) { + return dba_insert( $key, $value, $this->handle ); + } + + function close() { + if ( isset( $this->handle ) ) { + dba_close( $this->handle ); + } + + if ( !rename( $this->tmpFileName, $this->realFileName ) ) { + throw new Exception( 'Unable to move the new CDB file into place.' ); + } + unset( $this->handle ); + } + + function __destruct() { + if ( isset( $this->handle ) ) { + $this->close(); + } + } +} diff --git a/multiversion/Cdb_PHP.php b/multiversion/Cdb_PHP.php new file mode 100644 index 0000000..33ce7cc --- /dev/null +++ b/multiversion/Cdb_PHP.php @@ -0,0 +1,491 @@ +<?php +/** + * This is a port of D.J. Bernstein's CDB to PHP. It's based on the copy that + * appears in PHP 5.3. Changes are: + * * Error returns replaced with exceptions + * * Exception thrown if sizes or offsets are between 2GB and 4GB + * * Some variables renamed + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Common functions for readers and writers + */ +class CdbFunctions { + /** + * Take a modulo of a signed integer as if it were an unsigned integer. + * $b must be less than 0x40000000 and greater than 0 + * + * @param $a + * @param $b + * + * @return int + */ + public static function unsignedMod( $a, $b ) { + if ( $a & 0x80000000 ) { + $m = ( $a & 0x7fffffff ) % $b + 2 * ( 0x40000000 % $b ); + return $m % $b; + } else { + return $a % $b; + } + } + + /** + * Shift a signed integer right as if it were unsigned + * @param $a + * @param $b + * @return int + */ + public static function unsignedShiftRight( $a, $b ) { + if ( $b == 0 ) { + return $a; + } + if ( $a & 0x80000000 ) { + return ( ( $a & 0x7fffffff ) >> $b ) | ( 0x40000000 >> ( $b - 1 ) ); + } else { + return $a >> $b; + } + } + + /** + * The CDB hash function. + * + * @param $s string + * + * @return + */ + public static function hash( $s ) { + $h = 5381; + for ( $i = 0; $i < strlen( $s ); $i++ ) { + $h5 = ( $h << 5 ) & 0xffffffff; + // Do a 32-bit sum + // Inlined here for speed + $sum = ( $h & 0x3fffffff ) + ( $h5 & 0x3fffffff ); + $h = + ( + ( $sum & 0x40000000 ? 1 : 0 ) + + ( $h & 0x80000000 ? 2 : 0 ) + + ( $h & 0x40000000 ? 1 : 0 ) + + ( $h5 & 0x80000000 ? 2 : 0 ) + + ( $h5 & 0x40000000 ? 1 : 0 ) + ) << 30 + | ( $sum & 0x3fffffff ); + $h ^= ord( $s[$i] ); + $h &= 0xffffffff; + } + return $h; + } +} + +/** + * CDB reader class + */ +class CdbReader_PHP extends CdbReader { + /** The filename */ + var $fileName; + + /** The file handle */ + var $handle; + + /* number of hash slots searched under this key */ + var $loop; + + /* initialized if loop is nonzero */ + var $khash; + + /* initialized if loop is nonzero */ + var $kpos; + + /* initialized if loop is nonzero */ + var $hpos; + + /* initialized if loop is nonzero */ + var $hslots; + + /* initialized if findNext() returns true */ + var $dpos; + + /* initialized if cdb_findnext() returns 1 */ + var $dlen; + + /** + * @param $fileName string + * @throws Exception + */ + function __construct( $fileName ) { + $this->fileName = $fileName; + $this->handle = fopen( $fileName, 'rb' ); + if ( !$this->handle ) { + throw new Exception( 'Unable to open CDB file "' . $this->fileName . '".' ); + } + $this->findStart(); + } + + function close() { + if ( isset( $this->handle ) ) { + fclose( $this->handle ); + } + unset( $this->handle ); + } + + /** + * @param $key + * @return bool|string + */ + public function get( $key ) { + // strval is required + if ( $this->find( strval( $key ) ) ) { + return $this->read( $this->dlen, $this->dpos ); + } else { + return false; + } + } + + /** + * @param $key + * @param $pos + * @return bool + */ + protected function match( $key, $pos ) { + $buf = $this->read( strlen( $key ), $pos ); + return $buf === $key; + } + + protected function findStart() { + $this->loop = 0; + } + + /** + * @throws Exception + * @param $length + * @param $pos + * @return string + */ + protected function read( $length, $pos ) { + if ( fseek( $this->handle, $pos ) == -1 ) { + // This can easily happen if the internal pointers are incorrect + throw new Exception( + 'Seek failed, file "' . $this->fileName . '" may be corrupted.' ); + } + + if ( $length == 0 ) { + return ''; + } + + $buf = fread( $this->handle, $length ); + if ( $buf === false || strlen( $buf ) !== $length ) { + throw new Exception( + 'Read from CDB file failed, file "' . $this->fileName . '" may be corrupted.' ); + } + return $buf; + } + + /** + * Unpack an unsigned integer and throw an exception if it needs more than 31 bits + * @param $s + * @throws Exception + * @return mixed + */ + protected function unpack31( $s ) { + $data = unpack( 'V', $s ); + if ( $data[1] > 0x7fffffff ) { + throw new Exception( + 'Error in CDB file "' . $this->fileName . '", integer too big.' ); + } + return $data[1]; + } + + /** + * Unpack a 32-bit signed integer + * @param $s + * @return int + */ + protected function unpackSigned( $s ) { + $data = unpack( 'va/vb', $s ); + return $data['a'] | ( $data['b'] << 16 ); + } + + /** + * @param $key + * @return bool + */ + protected function findNext( $key ) { + if ( !$this->loop ) { + $u = CdbFunctions::hash( $key ); + $buf = $this->read( 8, ( $u << 3 ) & 2047 ); + $this->hslots = $this->unpack31( substr( $buf, 4 ) ); + if ( !$this->hslots ) { + return false; + } + $this->hpos = $this->unpack31( substr( $buf, 0, 4 ) ); + $this->khash = $u; + $u = CdbFunctions::unsignedShiftRight( $u, 8 ); + $u = CdbFunctions::unsignedMod( $u, $this->hslots ); + $u <<= 3; + $this->kpos = $this->hpos + $u; + } + + while ( $this->loop < $this->hslots ) { + $buf = $this->read( 8, $this->kpos ); + $pos = $this->unpack31( substr( $buf, 4 ) ); + if ( !$pos ) { + return false; + } + $this->loop += 1; + $this->kpos += 8; + if ( $this->kpos == $this->hpos + ( $this->hslots << 3 ) ) { + $this->kpos = $this->hpos; + } + $u = $this->unpackSigned( substr( $buf, 0, 4 ) ); + if ( $u === $this->khash ) { + $buf = $this->read( 8, $pos ); + $keyLen = $this->unpack31( substr( $buf, 0, 4 ) ); + if ( $keyLen == strlen( $key ) && $this->match( $key, $pos + 8 ) ) { + // Found + $this->dlen = $this->unpack31( substr( $buf, 4 ) ); + $this->dpos = $pos + 8 + $keyLen; + return true; + } + } + } + return false; + } + + /** + * @param $key + * @return bool + */ + protected function find( $key ) { + $this->findStart(); + return $this->findNext( $key ); + } +} + +/** + * CDB writer class + */ +class CdbWriter_PHP extends CdbWriter { + var $handle, $realFileName, $tmpFileName; + + var $hplist; + var $numentries, $pos; + + /** + * @param $fileName string + */ + function __construct( $fileName ) { + $this->realFileName = $fileName; + $this->tmpFileName = $fileName . '.tmp.' . mt_rand( 0, 0x7fffffff ); + $this->handle = fopen( $this->tmpFileName, 'wb' ); + if ( !$this->handle ) { + $this->throwException( + 'Unable to open CDB file "' . $this->tmpFileName . '" for write.' ); + } + $this->hplist = array(); + $this->numentries = 0; + $this->pos = 2048; // leaving space for the pointer array, 256 * 8 + if ( fseek( $this->handle, $this->pos ) == -1 ) { + $this->throwException( 'fseek failed in file "' . $this->tmpFileName . '".' ); + } + } + + function __destruct() { + if ( isset( $this->handle ) ) { + $this->close(); + } + } + + /** + * @param $key + * @param $value + * @return + */ + public function set( $key, $value ) { + if ( strval( $key ) === '' ) { + // DBA cross-check hack + return; + } + $this->addbegin( strlen( $key ), strlen( $value ) ); + $this->write( $key ); + $this->write( $value ); + $this->addend( strlen( $key ), strlen( $value ), CdbFunctions::hash( $key ) ); + } + + /** + * @throws Exception + */ + public function close() { + $this->finish(); + if ( isset( $this->handle ) ) { + fclose( $this->handle ); + } + + if ( !rename( $this->tmpFileName, $this->realFileName ) ) { + $this->throwException( 'Unable to move the new CDB file into place.' ); + } + unset( $this->handle ); + } + + /** + * @throws Exception + * @param $buf + */ + protected function write( $buf ) { + $len = fwrite( $this->handle, $buf ); + if ( $len !== strlen( $buf ) ) { + $this->throwException( 'Error writing to CDB file "' . $this->tmpFileName . '".' ); + } + } + + /** + * @throws Exception + * @param $len + */ + protected function posplus( $len ) { + $newpos = $this->pos + $len; + if ( $newpos > 0x7fffffff ) { + $this->throwException( + 'A value in the CDB file "' . $this->tmpFileName . '" is too large.' ); + } + $this->pos = $newpos; + } + + /** + * @param $keylen + * @param $datalen + * @param $h + */ + protected function addend( $keylen, $datalen, $h ) { + $this->hplist[] = array( + 'h' => $h, + 'p' => $this->pos + ); + + $this->numentries++; + $this->posplus( 8 ); + $this->posplus( $keylen ); + $this->posplus( $datalen ); + } + + /** + * @throws Exception + * @param $keylen + * @param $datalen + */ + protected function addbegin( $keylen, $datalen ) { + if ( $keylen > 0x7fffffff ) { + $this->throwException( 'Key length too long in file "' . $this->tmpFileName . '".' ); + } + if ( $datalen > 0x7fffffff ) { + $this->throwException( 'Data length too long in file "' . $this->tmpFileName . '".' ); + } + $buf = pack( 'VV', $keylen, $datalen ); + $this->write( $buf ); + } + + /** + * @throws Exception + */ + protected function finish() { + // Hack for DBA cross-check + $this->hplist = array_reverse( $this->hplist ); + + // Calculate the number of items that will be in each hashtable + $counts = array_fill( 0, 256, 0 ); + foreach ( $this->hplist as $item ) { + ++ $counts[255 & $item['h']]; + } + + // Fill in $starts with the *end* indexes + $starts = array(); + $pos = 0; + for ( $i = 0; $i < 256; ++$i ) { + $pos += $counts[$i]; + $starts[$i] = $pos; + } + + // Excessively clever and indulgent code to simultaneously fill $packedTables + // with the packed hashtables, and adjust the elements of $starts + // to actually point to the starts instead of the ends. + $packedTables = array_fill( 0, $this->numentries, false ); + foreach ( $this->hplist as $item ) { + $packedTables[--$starts[255 & $item['h']]] = $item; + } + + $final = ''; + for ( $i = 0; $i < 256; ++$i ) { + $count = $counts[$i]; + + // The size of the hashtable will be double the item count. + // The rest of the slots will be empty. + $len = $count + $count; + $final .= pack( 'VV', $this->pos, $len ); + + $hashtable = array(); + for ( $u = 0; $u < $len; ++$u ) { + $hashtable[$u] = array( 'h' => 0, 'p' => 0 ); + } + + // Fill the hashtable, using the next empty slot if the hashed slot + // is taken. + for ( $u = 0; $u < $count; ++$u ) { + $hp = $packedTables[$starts[$i] + $u]; + $where = CdbFunctions::unsignedMod( + CdbFunctions::unsignedShiftRight( $hp['h'], 8 ), $len ); + while ( $hashtable[$where]['p'] ) { + if ( ++$where == $len ) { + $where = 0; + } + } + $hashtable[$where] = $hp; + } + + // Write the hashtable + for ( $u = 0; $u < $len; ++$u ) { + $buf = pack( 'vvV', + $hashtable[$u]['h'] & 0xffff, + CdbFunctions::unsignedShiftRight( $hashtable[$u]['h'], 16 ), + $hashtable[$u]['p'] ); + $this->write( $buf ); + $this->posplus( 8 ); + } + } + + // Write the pointer array at the start of the file + rewind( $this->handle ); + if ( ftell( $this->handle ) != 0 ) { + $this->throwException( 'Error rewinding to start of file "' . $this->tmpFileName . '".' ); + } + $this->write( $final ); + } + + /** + * Clean up the temp file and throw an exception + * + * @param $msg string + * @throws Exception + */ + protected function throwException( $msg ) { + if ( $this->handle ) { + fclose( $this->handle ); + unlink( $this->tmpFileName ); + } + throw new Exception( $msg ); + } +} diff --git a/multiversion/MWMultiVersion.php b/multiversion/MWMultiVersion.php index 3dff9ab..c3267af 100644 --- a/multiversion/MWMultiVersion.php +++ b/multiversion/MWMultiVersion.php @@ -1,6 +1,8 @@ <?php require_once( dirname( __FILE__ ) . '/defines.php' ); require_once( dirname( __FILE__ ) . '/MWRealm.php' ); +require_once( dirname( __FILE__ ) . '/Cdb.php' ); +require_once( dirname( __FILE__ ) . '/Cdb_PHP.php' ); /** * Class to handle basic information related to what @@ -268,21 +270,24 @@ MULTIVER_CDB_DIR_APACHE . '/wikiversions.cdb' ); - $db = dba_open( $cdbFilename, 'r', 'cdb' ); + try { + $db = CdbReader::open( $cdbFilename ); + } catch( Exception $e ) {} + if ( $db ) { - $version = dba_fetch( "ver:{$this->db}", $db ); + $version = $db->get( "ver:{$this->db}" ); if ( $version === false ) { $extraVersion = false; } else { if ( strpos( $version, 'php-' ) !== 0 ) { self::error( "$cdbFilename version entry does not start with `php-` (got `$version`).\n" ); } - $extraVersion = dba_fetch( "ext:{$this->db}", $db ); + $extraVersion = $db->get( "ext:{$this->db}" ); if ( $extraVersion === false ) { self::error( "$cdbFilename has no extra version entry for `$db`.\n" ); } } - dba_close( $db ); + $db->close(); } else { self::error( "Unable to open $cdbFilename.\n" ); } diff --git a/multiversion/getMWVersion b/multiversion/getMWVersion index ae1024d..3525b21 100755 --- a/multiversion/getMWVersion +++ b/multiversion/getMWVersion @@ -4,6 +4,8 @@ require_once( dirname( __FILE__ ) . '/defines.php' ); require_once( dirname( __FILE__ ) . '/MWRealm.php' ); +require_once( dirname( __FILE__ ) . '/Cdb.php' ); +require_once( dirname( __FILE__ ) . '/Cdb_PHP.php' ); /** * This script prints the MW version associated with a specified wikidb. @@ -21,10 +23,14 @@ $cdbFilename = getRealmSpecificFilename( MULTIVER_CDB_DIR_APACHE . '/wikiversions.cdb' ); - $db = dba_open( $cdbFilename, 'r', 'cdb' ); + try { + $db = CdbReader::open( $cdbFilename ); + } catch( Exception $e ) {} + + if ( $db ) { - $version = dba_fetch( "ver:$dbName", $db ); - dba_close( $db ); + $version = $db->get( "ver:$dbName" ); + $db->close(); if ( $version !== false ) { return $version; // found version entry } diff --git a/multiversion/refreshWikiversionsCDB b/multiversion/refreshWikiversionsCDB index 504ced3..6a7dad7 100755 --- a/multiversion/refreshWikiversionsCDB +++ b/multiversion/refreshWikiversionsCDB @@ -46,7 +46,10 @@ # Build the new database at the temp location... @unlink( $tmpDBPath ); // clear any old temp file for sanity - $db = dba_open( $tmpDBPath, "n", "cdb_make" ); + try { + $db = CdbWriter::open( $tmpDBPath ); + } catch ( Exception $e ) {} + if ( !$db ) { die( "Unable to create {$tmpDBPath}.\n" ); } @@ -55,10 +58,10 @@ if ( $extVersion === '*' ) { $extVersion = ''; // "*" means "none" } - dba_insert( "ver:$dbName", $version, $db ); - dba_insert( "ext:$dbName", $extVersion, $db ); + $db->set( "ver:$dbName", $version ); + $db->set( "ext:$dbName", $extVersion ); } - dba_close( $db ); + $db->close(); # Sanity check the temp file... if ( !is_file( $tmpDBPath ) ) { -- To view, visit https://gerrit.wikimedia.org/r/93622 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I73195536a32fc01e0fb4fa1ca376a38837868d17 Gerrit-PatchSet: 1 Gerrit-Project: operations/mediawiki-config Gerrit-Branch: master Gerrit-Owner: Chad <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
