Smalyshev has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/351738 )
Change subject: Add version information to mw_cirrus_metastore
......................................................................
Add version information to mw_cirrus_metastore
Add to the version store:
- Mediawiki version
- Core git commit
- CirrusSearch git commit
This patch also does some refactoring so we have all data in one place.
Bug: T163851
Change-Id: I972cb8b5ee5b3712c44b3db337bc81f5451065c3
---
M includes/CirrusSearch.php
M includes/Maintenance/MetaStoreIndex.php
M maintenance/metastore.php
M maintenance/updateSuggesterIndex.php
4 files changed, 139 insertions(+), 103 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/38/351738/1
diff --git a/includes/CirrusSearch.php b/includes/CirrusSearch.php
index f136167..af23eb7 100644
--- a/includes/CirrusSearch.php
+++ b/includes/CirrusSearch.php
@@ -79,7 +79,7 @@
* Current request.
* @var WebRequest
*/
- private $request;
+ protected $request;
/**
* CirrusSearchIndexFieldFactory
diff --git a/includes/Maintenance/MetaStoreIndex.php
b/includes/Maintenance/MetaStoreIndex.php
index 689c7e8..134583a 100644
--- a/includes/Maintenance/MetaStoreIndex.php
+++ b/includes/Maintenance/MetaStoreIndex.php
@@ -3,6 +3,7 @@
namespace CirrusSearch\Maintenance;
use CirrusSearch\Connection;
+use GitInfo;
/**
* This program is free software; you can redistribute it and/or modify
@@ -37,7 +38,7 @@
* @const int minor version increment only when adding a new field to
* an existing mapping or a new mapping
*/
- const METASTORE_MINOR_VERSION = 2;
+ const METASTORE_MINOR_VERSION = 3;
/**
* @const string the doc id used to store version information related
@@ -87,7 +88,7 @@
/**
* @var Maintenance|null initiator maintenance script
- */
+ */
private $out;
/**
@@ -105,7 +106,8 @@
* @param Maintenance $out
* @param string $masterTimeout
*/
- public function __construct( Connection $connection, Maintenance $out,
$masterTimeout = '10000s' ) {
+ public function __construct( Connection $connection, Maintenance $out,
+ $masterTimeout = '10000s' ) {
$this->connection = $connection;
$this->client = $connection->getClient();
$this->configUtils = new ConfigUtils( $this->client, $out );
@@ -124,11 +126,8 @@
*/
public function versionIsAtLeast( array $expected ) {
// $expected >= $version
- return (bool)version_compare(
- implode( '.', $expected ),
- implode( '.', $this->metastoreVersion() ),
- '>='
- );
+ return (bool)version_compare( implode( '.', $expected ),
+ implode( '.', $this->metastoreVersion() ), '>=' );
}
public function createOrUpgradeIfNecessary() {
@@ -144,11 +143,18 @@
if ( $major < self::METASTORE_MAJOR_VERSION ) {
$this->log( self::INDEX_NAME . " major version
mismatch upgrading.\n" );
$this->majorUpgrade();
- } elseif( $major == self::METASTORE_MAJOR_VERSION &&
$minor < self::METASTORE_MINOR_VERSION ) {
- $this->log( self::INDEX_NAME . " minor version
mismatch trying to upgrade mapping.\n" );
+ } elseif ( $major == self::METASTORE_MAJOR_VERSION &&
+ $minor < self::METASTORE_MINOR_VERSION
+ ) {
+ $this->log( self::INDEX_NAME .
+ " minor version mismatch trying to
upgrade mapping.\n" );
$this->minorUpgrade();
- } elseif ( $major > self::METASTORE_MAJOR_VERSION ||
$minor > self::METASTORE_MINOR_VERSION ) {
- throw new \Exception( "Metastore version
$major.$minor found, cannot upgrade to a lower version: " .
self::METASTORE_MAJOR_VERSION . "." . self::METASTORE_MINOR_VERSION );
+ } elseif ( $major > self::METASTORE_MAJOR_VERSION ||
+ $minor > self::METASTORE_MINOR_VERSION
+ ) {
+ throw new \Exception( "Metastore version
$major.$minor found, cannot upgrade to a lower version: " .
+
self::METASTORE_MAJOR_VERSION . "." .
+
self::METASTORE_MINOR_VERSION );
}
}
}
@@ -174,12 +180,8 @@
// @todo utilize $this->getIndex()->create(...) once it
supports setting
// the master_timeout parameter.
$index = $this->client->getIndex( $name );
- $index->request(
- '',
- \Elastica\Request::PUT,
- $args,
- [ 'master_timeout' => $this->masterTimeout ]
- );
+ $index->request( '', \Elastica\Request::PUT, $args,
+ [ 'master_timeout' => $this->masterTimeout ] );
$this->log( " ok\n" );
$this->configUtils->waitForGreen( $index->getName(), 3600 );
$this->storeMetastoreVersion( $index );
@@ -201,6 +203,9 @@
'mapping_maj' => [ 'type' => 'long',
'include_in_all' => false ],
'mapping_min' => [ 'type' => 'long',
'include_in_all' => false ],
'shard_count' => [ 'type' => 'long',
'include_in_all' => false ],
+ 'mediawiki_version' => ['type' =>
'keyword', 'include_in_all' => false ],
+ 'mediawiki_commit' => ['type' =>
'keyword', 'include_in_all' => false ],
+ 'cirrus_commit' => ['type' =>
'keyword', 'include_in_all' => false ],
],
],
self::FROZEN_TYPE => [
@@ -243,15 +248,10 @@
private function minorUpgrade() {
$index = $this->connection->getIndex( self::INDEX_NAME );
- foreach( $this->buildMapping() as $type => $mapping ) {
- $index->getType( $type )->request(
- '_mapping',
- \Elastica\Request::PUT,
- $mapping,
- [
+ foreach ( $this->buildMapping() as $type => $mapping ) {
+ $index->getType( $type )->request( '_mapping',
\Elastica\Request::PUT, $mapping, [
'master_timeout' =>
$this->masterTimeout,
- ]
- );
+ ] );
}
$this->storeMetastoreVersion( $index );
}
@@ -274,21 +274,23 @@
}
// Create the alias
$path = '_aliases';
- $data = [ 'actions' => [
- [
- 'add' => [
- 'index' => $name,
- 'alias' => self::INDEX_NAME,
- ]
- ],
- ] ];
- if ( $oldIndexName !== null ) {
- $data['actions'][] = [
- 'remove' => [
- 'index' => $oldIndexName,
+ $data = [
+ 'actions' => [
+ [
+ 'add' => [
+ 'index' => $name,
'alias' => self::INDEX_NAME,
]
- ];
+ ],
+ ]
+ ];
+ if ( $oldIndexName !== null ) {
+ $data['actions'][] = [
+ 'remove' => [
+ 'index' => $oldIndexName,
+ 'alias' => self::INDEX_NAME,
+ ]
+ ];
}
$this->client->request( $path, \Elastica\Request::POST, $data,
[ 'master_timeout' => $this->masterTimeout ] );
@@ -312,10 +314,11 @@
}
$resp = $this->client->request( '_alias/' . self::INDEX_NAME,
\Elastica\Request::GET, [] );
$indexName = null;
- foreach( $resp->getData() as $index => $aliases ) {
+ foreach ( $resp->getData() as $index => $aliases ) {
if ( isset( $aliases['aliases'][self::INDEX_NAME] ) ) {
if ( $indexName !== null ) {
- throw new \Exception( "Multiple indices
are aliased with " . self::INDEX_NAME . ", please fix manually." );
+ throw new \Exception( "Multiple indices
are aliased with " . self::INDEX_NAME .
+ ", please fix
manually." );
}
$indexName = $index;
}
@@ -329,7 +332,7 @@
if ( !array_search( 'reindex', $plugins ) ) {
throw new \Exception( "The reindex module is mandatory
to upgrade the metastore" );
}
- $index = $this->createNewIndex( (string) time() );
+ $index = $this->createNewIndex( (string)time() );
// Reindex everything except the internal type, it's not clear
// yet if we just need to filter the metastore version info or
// the whole internal type. Currently we only use the internal
@@ -350,11 +353,9 @@
// reindex is extremely fast so we can wait for it
// we might consider using the task manager if this process
// becomes longer and/or prone to curl timeouts
- $resp = $this->client->request( '_reindex',
- \Elastica\Request::POST,
- $reindex,
- [ 'wait_for_completion' => true ]
- );
+ $resp =
+ $this->client->request( '_reindex',
\Elastica\Request::POST, $reindex,
+ [ 'wait_for_completion' => true ] );
$index->refresh();
$this->switchAliasTo( $index );
}
@@ -399,22 +400,18 @@
* @param \Elastica\Index $index new index
*/
private function storeMetastoreVersion( $index ) {
- $index->getType( self::INTERNAL_TYPE )->addDocument(
- new \Elastica\Document(
- self::METASTORE_VERSION_DOCID,
- [
- 'metastore_major_version' =>
self::METASTORE_MAJOR_VERSION,
- 'metastore_minor_version' =>
self::METASTORE_MINOR_VERSION,
- ]
- )
- );
+ $index->getType( self::INTERNAL_TYPE )
+ ->addDocument( new \Elastica\Document(
self::METASTORE_VERSION_DOCID, [
+ 'metastore_major_version' =>
self::METASTORE_MAJOR_VERSION,
+ 'metastore_minor_version' =>
self::METASTORE_MINOR_VERSION,
+ ] ) );
}
/**
* @param string $msg log message
*/
private function log( $msg ) {
- if ($this->out ) {
+ if ( $this->out ) {
$this->out->output( $msg );
}
}
@@ -441,6 +438,14 @@
*/
public function sanitizeType() {
return self::getSanitizeType( $this->connection );
+ }
+
+ /**
+ * Update versions for all types on the index.
+ * @param $baseName
+ */
+ public function updateAllVersions( $baseName ) {
+ self::updateAllMetastoreVersions( $this->connection, $baseName
);
}
/**
@@ -494,7 +499,7 @@
*/
public static function cirrusReady( Connection $connection ) {
return $connection->getIndex( self::INDEX_NAME )->exists() ||
- $connection->getIndex( self::OLD_INDEX_NAME )->exists();
+ $connection->getIndex( self::OLD_INDEX_NAME )->exists();
}
/**
@@ -504,24 +509,86 @@
*/
public static function getMetastoreVersion( Connection $connection ) {
try {
- $doc = self::getInternalType( $connection
)->getDocument( self::METASTORE_VERSION_DOCID );
- } catch ( \Elastica\Exception\NotFoundException $e ) {
+ $doc =
+ self::getInternalType( $connection
)->getDocument( self::METASTORE_VERSION_DOCID );
+ }
+ catch ( \Elastica\Exception\NotFoundException $e ) {
return [ 0, 0 ];
- } catch( \Elastica\Exception\ResponseException $e ) {
+ }
+ catch ( \Elastica\Exception\ResponseException $e ) {
// BC code in case the metastore alias does not exist
yet
$fullError = $e->getResponse()->getFullError();
- if ( isset( $fullError['type'] )
- && $fullError['type'] ===
'index_not_found_exception'
- && isset( $fullError['index'] )
- && $fullError['index'] === self::INDEX_NAME
+ if ( isset( $fullError['type'] ) &&
+ $fullError['type'] === 'index_not_found_exception'
&&
+ isset( $fullError['index'] ) &&
$fullError['index'] === self::INDEX_NAME
) {
return [ 0, 0 ];
}
throw $e;
}
return [
- (int) $doc->get('metastore_major_version'),
- (int) $doc->get('metastore_minor_version')
+ (int)$doc->get( 'metastore_major_version' ),
+ (int)$doc->get( 'metastore_minor_version' )
];
}
+
+ /**
+ * Create version data for index type.
+ * @param Connection $connection
+ * @param $indexBaseName
+ * @param $indexTypeName
+ * @return \Elastica\Document
+ */
+ public static function versionData( Connection $connection,
$indexBaseName,
+ $indexTypeName ) {
+ global $IP, $wgVersion;
+ list( $aMaj, $aMin ) = explode( '.',
AnalysisConfigBuilder::VERSION );
+ list( $mMaj, $mMin ) = explode( '.',
MappingConfigBuilder::VERSION );
+ $mwInfo = new GitInfo( $IP );
+ $cirrusInfo = new GitInfo( __DIR__ . '/../..');
+ $data = [
+ 'analysis_maj' => $aMaj,
+ 'analysis_min' => $aMin,
+ 'mapping_maj' => $mMaj,
+ 'mapping_min' => $mMin,
+ 'shard_count' =>
$connection->getSettings()->getShardCount( $indexTypeName ),
+ 'mediawiki_version' => $wgVersion,
+ 'mediawiki_commit' => $mwInfo->getHeadSHA1(),
+ 'cirrus_commit' => $cirrusInfo->getHeadSHA1(),
+ ];
+
+ return new \Elastica\Document( $connection->getIndexName(
$indexBaseName, $indexTypeName ),
+ $data );
+ }
+
+ /**
+ * Update version metastore for certain index.
+ * @param Connection $connection
+ * @param $indexBaseName
+ * @param $indexTypeName
+ * @throws \Exception
+ */
+ public static function updateMetastoreVersions( Connection $connection,
$indexBaseName,
+ $indexTypeName ) {
+ $index = self::getVersionType( $connection );
+ if ( !$index->exists() ) {
+ throw new \Exception( "meta store does not exist, you
must index your data first" );
+ }
+ $index->addDocument( self::versionData( $connection,
$indexBaseName, $indexTypeName ) );
+ }
+
+ /**
+ * Update metastore versions for all types of certain index.
+ * @param Connection $connection
+ * @param string $baseName Base name of the index.
+ */
+ public static function updateAllMetastoreVersions( Connection
$connection, $baseName ) {
+ $versionType = self::getVersionType( $connection );
+ $docs = [];
+ foreach( $connection->getAllIndexTypes() as $type ) {
+ $docs[] = self::versionData( $connection, $baseName,
$type );
+ }
+ $versionType->addDocuments( $docs );
+
+ }
}
diff --git a/maintenance/metastore.php b/maintenance/metastore.php
index 5fbd084..9a65d2c 100644
--- a/maintenance/metastore.php
+++ b/maintenance/metastore.php
@@ -119,6 +119,7 @@
$this->outputIndented( "index name: " . $r->getId() .
"\n" );
$this->outputIndented( " analysis version:
{$data['analysis_maj']}.{$data['analysis_min']}\n" );
$this->outputIndented( " mapping version:
{$data['mapping_maj']}.{$data['mapping_min']}\n" );
+ $this->outputIndented( " code version:
{$data['mediawiki_version']} ({$data['mediawiki_commit']}, Cirrus:
{$data['cirrus_commit']})\n" );
$this->outputIndented( " shards:
{$data['shard_count']}\n" );
}
}
@@ -127,25 +128,8 @@
* @param string $baseName
*/
private function updateIndexVersion( $baseName ) {
- $versionType = $this->metaStore->versionType();
$this->outputIndented( "Updating tracking indexes..." );
- $docs = [];
- list( $aMaj, $aMin ) = explode( '.',
\CirrusSearch\Maintenance\AnalysisConfigBuilder::VERSION );
- list( $mMaj, $mMin ) = explode( '.',
\CirrusSearch\Maintenance\MappingConfigBuilder::VERSION );
- $connSettings = $this->getConnection()->getSettings();
- foreach( $this->getConnection()->getAllIndexTypes() as $type ) {
- $docs[] = new \Elastica\Document(
- $this->getConnection()->getIndexName(
$baseName, $type ),
- [
- 'analysis_maj' => $aMaj,
- 'analysis_min' => $aMin,
- 'mapping_maj' => $mMaj,
- 'mapping_min' => $mMin,
- 'shard_count' =>
$connSettings->getShardCount( $type ),
- ]
- );
- }
- $versionType->addDocuments( $docs );
+ $this->metaStore->updateAllVersions( $baseName );
$this->output( "done\n" );
}
diff --git a/maintenance/updateSuggesterIndex.php
b/maintenance/updateSuggesterIndex.php
index 1cd2848..2307dbc 100644
--- a/maintenance/updateSuggesterIndex.php
+++ b/maintenance/updateSuggesterIndex.php
@@ -740,23 +740,8 @@
private function updateVersions() {
$this->log( "Updating tracking indexes..." );
- $index = MetaStoreIndex::getVersionType( $this->getConnection()
);
- if ( !$index->exists() ) {
- throw new \Exception("meta store does not exist, you
must index your data first");
- }
- list( $aMaj, $aMin ) = explode( '.',
\CirrusSearch\Maintenance\SuggesterAnalysisConfigBuilder::VERSION );
- list( $mMaj, $mMin ) = explode( '.',
\CirrusSearch\Maintenance\SuggesterMappingConfigBuilder::VERSION );
- $doc = new \Elastica\Document(
- $this->getIndexTypeName(),
- [
- 'analysis_maj' => $aMaj,
- 'analysis_min' => $aMin,
- 'mapping_maj' => $mMaj,
- 'mapping_min' => $mMin,
- 'shard_count' => $this->getShardCount(),
- ]
- );
- $index->addDocument( $doc );
+ MetaStoreIndex::updateMetastoreVersions(
$this->getConnection(), $this->indexBaseName,
+ $this->indexTypeName );
$this->output("ok.\n");
}
--
To view, visit https://gerrit.wikimedia.org/r/351738
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I972cb8b5ee5b3712c44b3db337bc81f5451065c3
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
