Aaron Schulz has submitted this change and it was merged.
Change subject: Committed script to compare backend zone sizes for sanity.
......................................................................
Committed script to compare backend zone sizes for sanity.
Change-Id: Ib5c4a0f0d2279eab5bd8ca6440018d003ffbad6c
---
A filebackend/measureZoneSizes.php
1 file changed, 96 insertions(+), 0 deletions(-)
Approvals:
Aaron Schulz: Verified; Looks good to me, approved
jenkins-bot: Checked
diff --git a/filebackend/measureZoneSizes.php b/filebackend/measureZoneSizes.php
new file mode 100644
index 0000000..2f06a61
--- /dev/null
+++ b/filebackend/measureZoneSizes.php
@@ -0,0 +1,96 @@
+<?php
+require_once( dirname( __FILE__ ) . '/Maintenance.php' );
+
+class MeasureZoneSizes extends Maintenance {
+ public function __construct() {
+ parent::__construct();
+ $this->addOption( 'backend1', true, true );
+ $this->addOption( 'backend2', true, true );
+ $this->addOption( 'fast', 'Estimate size from a sample of
shards.' );
+ $this->addOption( 'outfile', 'File to report backend size
discrepencies.', false, true );
+ }
+
+ public function execute() {
+ global $wgDBname;
+
+ $b1Name = $this->getOption( 'backend1' );
+ $b1 = FileBackendGroup::singleton()->get( $b1Name );
+ $b2Name = $this->getOption( 'backend2' );
+ $b2 = FileBackendGroup::singleton()->get( $b2Name );
+
+ $file = $this->getOption( 'outfile' );
+ $fast = $this->hasOption( 'fast' );
+
+ if ( $fast ) {
+ $psuffix = dechex( mt_rand( 0, 15 ) );
+ $psuffix .= '/' . $psuffix . dechex( mt_rand( 0, 15 ) );
+ $dsuffix = dechex( mt_rand( 0, 15 ) ) . '/' . dechex(
mt_rand( 0, 15 ) );
+ $pfactor = 256;
+ $dfactor = 1296;
+ } else {
+ $psuffix = dechex( mt_rand( 0, 15 ) );
+ $dsuffix = dechex( mt_rand( 0, 15 ) );
+ $pfactor = 16;
+ $dfactor = 36;
+ }
+
+ $output = "Sampling from public shard '$psuffix'.\n";
+ $output .= "Store\t\tZone\t\tObjects\t\tTotal bytes\t\n";
+ // Public zone files (Backend 1)...
+ $dirC = $b1->getRootStoragePath() . "/local-public/$psuffix";
// current
+ $dirA = $b1->getRootStoragePath() .
"/local-public/archive/$psuffix"; // archived
+ list( $countC, $bytesC ) = $this->getSizeOfDirectory( $b1,
$dirC );
+ list( $countA, $bytesA ) = $this->getSizeOfDirectory( $b1,
$dirA );
+ $b1_count = ( $countC + $countA ) * $pfactor;
+ $b1_bytes = ( $bytesC + $bytesA ) * $pfactor;
+ $output .= "B1\t\tpublic\t\t$b1_count\t\t$b1_bytes\t\n";
+ // Public zone files (Backend 2)...
+ $dirC = $b2->getRootStoragePath() . "/local-public/$psuffix";
// current
+ $dirA = $b2->getRootStoragePath() .
"/local-public/archive/$psuffix"; // archived
+ list( $countC, $bytesC ) = $this->getSizeOfDirectory( $b2,
$dirC );
+ list( $countA, $bytesA ) = $this->getSizeOfDirectory( $b2,
$dirA );
+ $b2_count = ( $countC + $countA ) * $pfactor;
+ $b2_bytes = ( $bytesC + $bytesA ) * $pfactor;
+ $output .= "B2\t\tpublic\t\t$b2_count\t\t$b2_bytes\t\n";
+
+ if ( $file && ( $b2_count <= .97 * $b1_count || $b2_bytes < .97
* $b1_bytes ) ) {
+ file_put_contents( $file, "$wgDBname:\n$output",
LOCK_EX | FILE_APPEND );
+ }
+ $this->output( $output );
+
+ $output = "Sampling from deleted shard '$dsuffix'.\n";
+ $output .= "Store\t\tZone\t\tObjects\t\tTotal bytes\t\n";
+ // Deleted zone files (Backend 1)...
+ $dir = $b1->getRootStoragePath() . "/local-deleted/$dsuffix";
+ list( $count, $bytes ) = $this->getSizeOfDirectory( $b1, $dir );
+ $b1_count = $count * $dfactor;
+ $b1_bytes = $bytes * $dfactor;
+ $output .= "B1\t\tdeleted\t\t$b1_count\t\t$b1_bytes\t\n";
+ // Deleted zone files (Backend 2)...
+ $dir = $b2->getRootStoragePath() . "/local-deleted/$dsuffix";
+ list( $count, $bytes ) = $this->getSizeOfDirectory( $b2, $dir );
+ $b2_count = $count * $dfactor;
+ $b2_bytes = $bytes * $dfactor;
+ $output .= "B2\t\tdeleted\t\t$b2_count\t\t$b2_bytes\t\n";
+
+ if ( $file && ( $b2_count <= .97 * $b1_count || $b2_bytes < .97
* $b1_bytes ) ) {
+ file_put_contents( $file, "$wgDBname:\n$output",
LOCK_EX | FILE_APPEND );
+ }
+ $this->output( $output );
+
+ }
+
+ protected function getSizeOfDirectory( FileBackend $backend, $dir ) {
+ $bytes = 0;
+ $count = 0;
+ $list = $backend->getFileList( array( 'dir' => $dir ) );
+ foreach ( $list as $relPath ) {
+ $bytes += (int)$backend->getFileSize( array( 'src' =>
"{$dir}/{$relPath}" ) );
+ $count++;
+ }
+ return array( $count, $bytes );
+ }
+}
+$maintClass = 'MeasureZoneSizes';
+require_once( RUN_MAINTENANCE_IF_MAIN );
+
--
To view, visit https://gerrit.wikimedia.org/r/61421
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib5c4a0f0d2279eab5bd8ca6440018d003ffbad6c
Gerrit-PatchSet: 3
Gerrit-Project: mediawiki/extensions/WikimediaMaintenance
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
Gerrit-Reviewer: Aaron Schulz <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
