[MediaWiki-commits] [Gerrit] operations/puppet[production]: labsdb: maintain-views laundry list of changes
Rush has submitted this change and it was merged.
Change subject: labsdb: maintain-views laundry list of changes
..
labsdb: maintain-views laundry list of changes
* Convert config file handling to yaml
* Add packages needed for maintain-views to puppet
* Change behavior from default to modifying all dbs
to default to requiring an explicit flag
* move view definer to attribute
* separate dry-run and debug options
* add user_exists method to verify definer
* add doc strings to methods for class SchemaOperations
* check_customview_source method return a consistent type
* verify source database exists before creating _p variant
* make natively single tenant rather than expecting slices
* move do_dbhost function logic into main (as we are expecting
single tenant)
* separate metadata assignment and sensitive DB exclusion
* qualify password module lookup for toplevel
Change-Id: I95275471471ce2ba8b63ed0b309a126c5a8ab051
---
M modules/role/files/labsdb/maintain-views.py
M modules/role/manifests/labsdb/views.pp
D modules/role/templates/labsdb/maintain-views.json
A modules/role/templates/labsdb/maintain-views.yaml
4 files changed, 526 insertions(+), 292 deletions(-)
Approvals:
Rush: Looks good to me, approved
Madhuvishy: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/modules/role/files/labsdb/maintain-views.py
b/modules/role/files/labsdb/maintain-views.py
index eefaa6f..b1cb9d0 100644
--- a/modules/role/files/labsdb/maintain-views.py
+++ b/modules/role/files/labsdb/maintain-views.py
@@ -18,21 +18,18 @@
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
-#
# This script maintains the databases containing sanitized views to
# the replicated databases (in the form _p for every )
-#
-# By default, it processes every database but it accepts a list of
-# databases to process
#
# Information on available and operational databases is sourced from
# a checkout of mediawiki-config.
#
import argparse
-import json
import logging
import re
+import sys
+import yaml
import pymysql
@@ -45,17 +42,33 @@
self.db_p = db + '_p'
self.db_size = db_size
self.cursor = cursor
+self.definer = 'viewmaster'
def write_execute(self, query):
-if self.dry_run:
-logging.info("DRY RUN: Would execute: {}".format(query))
-else:
+""" Do operation or simulate
+:param query: str
+"""
+logging.debug("SQL: {}".format(query))
+if not self.dry_run:
self.cursor.execute(query)
-def table_exists(self, table, database):
+def user_exists(self, name):
+""" Check if a user exists
+:param name: str
"""
-Determine whether a table of the given name exists in the database of
+self.cursor.execute("""
+SELECT 1
+FROM `mysql`.`user`
+WHERE `user`=%s;
+""", args=(name))
+return bool(self.cursor.rowcount)
+
+def table_exists(self, table, database):
+""" Determine whether a table of the given name exists in the database
of
the given name.
+:param table: str
+:param database: str
+:returns: bool
"""
self.cursor.execute("""
SELECT `table_name`
@@ -64,33 +77,21 @@
""", args=(table, database))
return bool(self.cursor.rowcount)
-def execute(self, fullviews, customviews):
+def database_exists(self, database):
+""" Verify if a DB exists
+:param database: str
+:return: bool
"""
-Begin checking/creating views for this schema.
-"""
-
self.cursor.execute("""
SELECT `schema_name`
FROM `information_schema`.`schemata`
WHERE `schema_name`=%s
-""", args=(self.db_p,))
-if not self.cursor.rowcount:
-# Can't use pymysql to build this
-self.write_execute(
-"CREATE DATABASE `{}`;".format(self.db_p)
-)
-
-logging.info("Full views for {}:".format(self.db))
-for view in fullviews:
-self.do_fullview(view)
-
-logging.info("Custom views for {}:".format(self.db))
-for view_name, view_details in customviews.items():
-self.do_customview(view_name, view_details)
+""", args=(database,))
+return bool(self.cursor.rowcount)
def do_fullview(self, view):
-"""
-Check whether the source table exists, and if so, create the view.
+""" Check whether the source table exists, and if so, create the view.
+:param view: str
"""
if self.table_exists(view, self.db):
# If it does, create or replace the view for it.
@@ -101,10 +102,10 @@
# Can't us
[MediaWiki-commits] [Gerrit] operations/puppet[production]: labsdb: maintain-views laundry list of changes
Rush has uploaded a new change for review.
https://gerrit.wikimedia.org/r/315534
Change subject: labsdb: maintain-views laundry list of changes
..
labsdb: maintain-views laundry list of changes
* Convert config file handling to yaml
* Add packages needed for maintain-views to puppet
* Change behavior from default to modifying all dbs
to default to requiring an explicit flag
* move view definer to attribute
* separate dry-run and debug options
* add user_exists method to verify definer
* add doc strings to methods for class SchemaOperations
* check_customview_source method return a consistent type
* make natively single tenant rather than expecting slices
* move do_dbhost function logic into main (as we are expecting
single tenant)
* separate metadata assignment and sensitive DB exclusion
* qualify password module lookup for toplevel
Change-Id: I95275471471ce2ba8b63ed0b309a126c5a8ab051
---
M modules/role/files/labsdb/maintain-views.py
M modules/role/manifests/labsdb/views.pp
D modules/role/templates/labsdb/maintain-views.json
A modules/role/templates/labsdb/maintain-views.yaml
4 files changed, 531 insertions(+), 296 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/34/315534/1
diff --git a/modules/role/files/labsdb/maintain-views.py
b/modules/role/files/labsdb/maintain-views.py
index eefaa6f..cde0b84 100644
--- a/modules/role/files/labsdb/maintain-views.py
+++ b/modules/role/files/labsdb/maintain-views.py
@@ -18,12 +18,8 @@
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
-#
# This script maintains the databases containing sanitized views to
# the replicated databases (in the form _p for every )
-#
-# By default, it processes every database but it accepts a list of
-# databases to process
#
# Information on available and operational databases is sourced from
# a checkout of mediawiki-config.
@@ -33,6 +29,8 @@
import json
import logging
import re
+import sys
+import yaml
import pymysql
@@ -45,17 +43,33 @@
self.db_p = db + '_p'
self.db_size = db_size
self.cursor = cursor
+self.definer = 'viewmaster'
def write_execute(self, query):
-if self.dry_run:
-logging.info("DRY RUN: Would execute: {}".format(query))
-else:
+""" Do operation or simulate
+:param query: str
+"""
+logging.debug("SQL: {}".format(query))
+if not self.dry_run:
self.cursor.execute(query)
-def table_exists(self, table, database):
+def user_exists(self, name):
+""" Check if a user exists
+:param name: str
"""
-Determine whether a table of the given name exists in the database of
+self.cursor.execute("""
+SELECT 1
+FROM `mysql`.`user`
+WHERE `user`=%s;
+""", args=(name))
+return bool(self.cursor.rowcount)
+
+def table_exists(self, table, database):
+""" Determine whether a table of the given name exists in the database
of
the given name.
+:param table: str
+:param database: str
+:returns: bool
"""
self.cursor.execute("""
SELECT `table_name`
@@ -64,33 +78,21 @@
""", args=(table, database))
return bool(self.cursor.rowcount)
-def execute(self, fullviews, customviews):
+def database_exists(self, database):
+""" Verify if a DB exists
+:param database: str
+:return: bool
"""
-Begin checking/creating views for this schema.
-"""
-
self.cursor.execute("""
SELECT `schema_name`
FROM `information_schema`.`schemata`
WHERE `schema_name`=%s
-""", args=(self.db_p,))
-if not self.cursor.rowcount:
-# Can't use pymysql to build this
-self.write_execute(
-"CREATE DATABASE `{}`;".format(self.db_p)
-)
-
-logging.info("Full views for {}:".format(self.db))
-for view in fullviews:
-self.do_fullview(view)
-
-logging.info("Custom views for {}:".format(self.db))
-for view_name, view_details in customviews.items():
-self.do_customview(view_name, view_details)
+""", args=(database,))
+return bool(self.cursor.rowcount)
def do_fullview(self, view):
-"""
-Check whether the source table exists, and if so, create the view.
+""" Check whether the source table exists, and if so, create the view.
+:param view: str
"""
if self.table_exists(view, self.db):
# If it does, create or replace the view for it.
@@ -101,10 +103,10 @@
# Can't use pymysql to build this
self.write_execute("""
