EBernhardson has uploaded a new change for review.
https://gerrit.wikimedia.org/r/247478
Change subject: Revert "Revert "Enable config for all three search clusters,
but only write to eqiad""
......................................................................
Revert "Revert "Enable config for all three search clusters, but only write to
eqiad""
This reverts commit baced1cd63c221a449f0ab399ed6ae83274ce88e.
Change-Id: I86440998474afe7d5673ed4af2f30406268c2c4e
---
M tests/cirrusTest.php
M wmf-config/CirrusSearch-common.php
M wmf-config/CirrusSearch-labs.php
M wmf-config/CirrusSearch-production.php
M wmf-config/CommonSettings.php
M wmf-config/InitialiseSettings-labs.php
M wmf-config/InitialiseSettings.php
7 files changed, 83 insertions(+), 53 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/78/247478/1
diff --git a/tests/cirrusTest.php b/tests/cirrusTest.php
index 131b68a..c07c497 100644
--- a/tests/cirrusTest.php
+++ b/tests/cirrusTest.php
@@ -10,21 +10,50 @@
$this->assertArrayHasKey( 'wgCirrusSearchClusters', $config );
$this->assertArrayHasKey( 'wgCirrusSearchDefaultCluster',
$config );
$this->assertEquals( 'eqiad',
$config['wgCirrusSearchDefaultCluster'] );
+ $this->assertCount( 3, $config['wgCirrusSearchClusters'] );
+
+ // testwiki writes to eqiad and the lab replica
+ $this->assertCount( 2, $config['wgCirrusSearchWriteClusters'] );
+
+ $this->assertArrayHasKey(
+ $config['wgCirrusSearchDefaultCluster'],
+ $config['wgCirrusSearchClusters']
+ );
+
+ foreach ( $config['wgCirrusSearchWriteClusters'] as
$writeCluster ) {
+ $this->assertArrayHasKey(
+ $writeCluster,
+ $config['wgCirrusSearchClusters']
+ );
+ }
}
public function testClusterConfigurationForProdEnwiki() {
$config = $this->loadCirrusConfig( 'production', 'enwiki',
'wiki', 'en', 'wikipedia' );
- $this->assertArrayHasKey( 'wgCirrusSearchServers', $config );
+ $this->assertArrayNotHasKey( 'wgCirrusSearchServers', $config );
$this->assertArrayHasKey( 'wgCirrusSearchClusters', $config );
- $this->assertCount( 1, $config['wgCirrusSearchClusters'] );
- $this->assertEquals(
- $config['wgCirrusSearchServers'],
- reset( $config['wgCirrusSearchClusters'] )
- );
- $clusters = array_keys( $config['wgCirrusSearchClusters'] );
- $this->assertEquals(
+ $this->assertArrayHasKey( 'wgCirrusSearchDefaultCluster',
$config );
+ $this->assertCount( 3, $config['wgCirrusSearchClusters'] );
+ $this->assertCount( 3, $config['wgCirrusSearchShardCount'] );
+ $this->assertCount( 3, $config['wgCirrusSearchReplicas'] );
+ $this->assertCount( 3,
$config['wgCirrusSearchClientSideConnectTimeout'] );
+
+ foreach ( array_keys ( $config['wgCirrusSearchClusters'] ) as
$cluster ) {
+ $this->assertArrayHasKey( $cluster,
$config['wgCirrusSearchShardCount'] );
+ $this->assertArrayHasKey( $cluster,
$config['wgCirrusSearchReplicas'] );
+ $this->assertArrayHasKey( $cluster,
$config['wgCirrusSearchClientSideConnectTimeout'] );
+ }
+
+ // Only eqiad for now
+ $this->assertCount( 1, $config['wgCirrusSearchWriteClusters'] );
+ $this->assertArrayHasKey(
$config['wgCirrusSearchDefaultCluster'],
- reset( $clusters )
+ $config['wgCirrusSearchClusters']
+ );
+
+ $this->assertArrayHasKey(
+ reset( $config['wgCirrusSearchWriteClusters'] ),
+ $config['wgCirrusSearchClusters']
);
}
diff --git a/wmf-config/CirrusSearch-common.php
b/wmf-config/CirrusSearch-common.php
index 5e419f5..4dbbdfd 100644
--- a/wmf-config/CirrusSearch-common.php
+++ b/wmf-config/CirrusSearch-common.php
@@ -21,11 +21,10 @@
$wgJobTypeConf['default'];
}
-if ( isset( $wmgCirrusSearchServers ) ) {
- $wgCirrusSearchServers = $wmgCirrusSearchServers;
-}
-$wgCirrusSearchClusters = $wmgCirrusSearchClusters;
+# Set up the the default cluster to send queries to,
+# and the list of clusters to write to.
$wgCirrusSearchDefaultCluster = $wmgCirrusSearchDefaultCluster;
+$wgCirrusSearchWriteClusters = $wmgCirrusSearchWriteClusters;
# Enable user testing
$wgCirrusSearchUserTesting = $wmgCirrusSearchUserTesting;
diff --git a/wmf-config/CirrusSearch-labs.php b/wmf-config/CirrusSearch-labs.php
index f5edb48..d0e97d3 100644
--- a/wmf-config/CirrusSearch-labs.php
+++ b/wmf-config/CirrusSearch-labs.php
@@ -5,6 +5,15 @@
# to the 'labs' realm which in most of the cases means the beta cluster.
# It should be loaded AFTER CirrusSearch-common.php
+$wgCirrusSearchClusters = array(
+ 'eqiad' => array(
+ 'deployment-elastic05',
+ 'deployment-elastic06',
+ 'deployment-elastic07',
+ 'deployment-elastic08',
+ ),
+);
+
if ( $wgDBname == 'enwiki' ) {
$wgCirrusSearchInterwikiSources = array(
'wiktionary' => 'enwiktionary',
@@ -27,6 +36,9 @@
)
);
+# write to all configured clusters, there should only be one in labs
+$wgCirrusSearchWriteClusters = null;
+
$wgCirrusSearchEnableSearchLogging = true;
$wgCirrusSearchLanguageToWikiMap = array(
diff --git a/wmf-config/CirrusSearch-production.php
b/wmf-config/CirrusSearch-production.php
index 1bb20f7..487ac1d 100644
--- a/wmf-config/CirrusSearch-production.php
+++ b/wmf-config/CirrusSearch-production.php
@@ -6,6 +6,12 @@
# It should be loaded AFTER CirrusSearch-common.php
+$wgCirrusSearchClusters = array(
+ 'eqiad' => array( '10.2.2.30' ), // search.svc.eqiad.wmnet
+ 'codfw' => array( '10.2.1.30' ), // search.svc.codfw.wmnet
+ 'labsearch' => array( '10.64.37.14' ), // nobelium.eqiad.wmnet
+);
+
$wgCirrusSearchConnectionAttempts = 3;
$wgCirrusSearchBackup['backups'] = array(
@@ -42,3 +48,23 @@
}
$wgCirrusSearchEnableSearchLogging = true;
+
+// The default configuration is a single-cluster configuration, expand
+// that here into the necessary multi-cluster config
+$wgCirrusSearchShardCount = array(
+ 'eqiad' => $wgCirrusSearchShardCount,
+ 'codfw' => array_map( function($x) { return min( 7, $x ); },
$wgCirrusSearchShardCount ),
+ 'labsearch' => array_map( function() { return 1; },
$wgCirrusSearchShardCount ),
+);
+
+$wgCirrusSearchReplicas = array(
+ 'eqiad' => $wgCirrusSearchReplicas,
+ 'codfw' => $wgCirrusSearchReplicas,
+ 'labsearch' => array_map( function() { return 'false'; },
$wgCirrusSearchReplicas ),
+);
+
+$wgCirrusSearchClientSideConnectTimeout = array(
+ 'eqiad' => $wmfDatacenter === 'eqiad' ? 5 : 10,
+ 'codfw' => $wmfDatacenter === 'codfw' ? 5 : 10,
+ 'labsearch' => 2,
+);
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index c7037ec..5a6a673 100755
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -2179,7 +2179,7 @@
if ( $wmgUseTranslationMemory ) {
$servers = array_map(
function ( $v ) { return array( 'host' => $v ); },
- isset( $wgCirrusSearchServers ) ?
$wgCirrusSearchServers : $wgCirrusSearchClusters[$wgCirrusSearchDefaultCluster]
+ $wgCirrusSearchClusters[$wgCirrusSearchDefaultCluster]
);
// Read only until renamed to 'TTMServer'
$wgTranslateTranslationServices['TTMServer'] = array(
diff --git a/wmf-config/InitialiseSettings-labs.php
b/wmf-config/InitialiseSettings-labs.php
index 9240aa7..8bc71be 100644
--- a/wmf-config/InitialiseSettings-labs.php
+++ b/wmf-config/InitialiseSettings-labs.php
@@ -947,32 +947,6 @@
'wmgUseCirrus' => array(
'default' => true,
- 'commonswiki' => true,
- 'dewiki' => true,
- 'enwiki' => true,
- 'eswiki' => true,
- 'frwiki' => true,
- 'jawiki' => true,
- 'nlwiki' => true,
- 'plwiki' => true,
- 'ruwiki' => true,
- 'svwiki' => true,
- 'zhwiki' => true,
- ),
-
- '-wmgCirrusSearchServers' => array(
- 'default' => null,
- ),
-
- '-wmgCirrusSearchClusters' => array(
- 'default' => array(
- 'eqiad' => array(
- 'deployment-elastic05',
- 'deployment-elastic06',
- 'deployment-elastic07',
- 'deployment-elastic08',
- ),
- ),
),
'wmgUseFlow' => array(
diff --git a/wmf-config/InitialiseSettings.php
b/wmf-config/InitialiseSettings.php
index c6d074f..eb5ee6a 100644
--- a/wmf-config/InitialiseSettings.php
+++ b/wmf-config/InitialiseSettings.php
@@ -15334,23 +15334,13 @@
'default' => true,
),
-'wmgCirrusSearchServers' => array(
- 'default' => array( '10.2.2.30' ), // search.svc.eqiad.wmnet
- 'testwiki' => null,
-),
-
'wmgCirrusSearchDefaultCluster' => array(
'default' => 'eqiad',
),
-'wmgCirrusSearchClusters' => array(
- 'default' => array(
- 'eqiad' => array( '10.2.2.30' ), // search.svc.eqiad.wmnet
- ),
- 'testwiki' => array(
- 'eqiad' => array( '10.2.2.30' ), // search.svc.eqiad.wmnet
- 'labsearch' => array( 'nobelium.eqiad.wmnet' ),
- ),
+'wmgCirrusSearchWriteClusters' => array(
+ 'default' => array( 'eqiad' ),
+ 'testwiki' => array( 'eqiad', 'labsearch' )
),
'wmgCirrusPerUserPoolCounter' => array(
--
To view, visit https://gerrit.wikimedia.org/r/247478
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I86440998474afe7d5673ed4af2f30406268c2c4e
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
