Yuvipanda has submitted this change and it was merged.
Change subject: beta: Add script from Jenkins beta-update-databases
......................................................................
beta: Add script from Jenkins beta-update-databases
Bug: T96199
Bug: T98342
Change-Id: Ie063022b7b4bfc7fa4e1dac2fd15f32bf3a02d3d
---
A modules/beta/files/wmf-beta-update-databases.py
M modules/beta/manifests/autoupdater.pp
2 files changed, 105 insertions(+), 0 deletions(-)
Approvals:
Yuvipanda: Verified; Looks good to me, approved
diff --git a/modules/beta/files/wmf-beta-update-databases.py
b/modules/beta/files/wmf-beta-update-databases.py
new file mode 100755
index 0000000..a4b08e9
--- /dev/null
+++ b/modules/beta/files/wmf-beta-update-databases.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+#######################################################################
+# WARNING: this file is managed by Puppet
+# puppet:///modules/beta/wmf-beta-update-databases.py
+#######################################################################
+
+"""
+Run update.php for all dbs listed in a dblist
+"""
+import os
+import sys
+import errno
+import multiprocessing
+import subprocess
+import argparse
+
+
+def get_staging_dir():
+ return os.getenv("MEDIAWIKI_STAGING_DIR", "/srv/mediawiki-staging")
+
+
+def get_default_dblist():
+ return os.path.join(get_staging_dir(), 'all-labs.dblist')
+
+
+def do_wait(procs):
+ """
+ wait for command in procs array to execute, dump their output
+ """
+ for p, f, cmd in procs:
+ if p.wait() > 0:
+ raise Exception("command: ", cmd, "output: ", f.read())
+ f.seek(0)
+ print f.read().strip()
+ f.close()
+
+
+def run_updates(staging, cores):
+ """
+ run update.php on each wiki found in dblist
+ """
+ procs = []
+ with open(staging, "r") as dblist:
+ for db in dblist:
+ db = db.strip()
+ f = os.tmpfile()
+ cmd = "/usr/local/bin/mwscript update.php --wiki=%s --quick" % db
+ p = subprocess.Popen(cmd, stdout=f, stderr=f, shell=True)
+ procs.append((p, f, cmd))
+ if (len(procs) >= cores):
+ do_wait(procs)
+ procs = []
+
+ # catches odd cases where dblist file is smaller than batch size
+ if (len(procs) > 0):
+ do_wait(procs)
+
+
+def parse_args():
+ """
+ parse arguments
+ """
+ ap = argparse.ArgumentParser(
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+ ap.add_argument("-b", "--batch", required=False, type=int,
+ default=get_cores(),
+ help="Number of databases to update in parallel")
+ ap.add_argument("-d", "--dblist", required=False,
+ default=get_default_dblist(), help="Path to dblist file")
+
+ return ap.parse_args()
+
+
+def get_cores():
+ """
+ max of 1/2 cpu_count or 1
+ """
+ return max(multiprocessing.cpu_count() / 2, 1)
+
+
+def check_dblist(dblist):
+ """
+ check and return path to dblist
+ """
+ if not os.path.exists(dblist):
+ raise IOError(errno.ENOENT, "Labs dblist not found", dblist)
+
+ return dblist
+
+
+def main():
+ args = parse_args()
+ dblist = check_dblist(args.dblist)
+ run_updates(dblist, args.batch)
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/modules/beta/manifests/autoupdater.pp
b/modules/beta/manifests/autoupdater.pp
index 3ca9821..9990b7c 100644
--- a/modules/beta/manifests/autoupdater.pp
+++ b/modules/beta/manifests/autoupdater.pp
@@ -30,6 +30,13 @@
content => template('beta/wmf-beta-mwconfig-update.erb'),
}
+ file { '/usr/local/bin/wmf-beta-update-databases.py':
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ source => 'puppet:///modules/beta/wmf-beta-update-databases.py',
+ }
+
git::clone { 'mediawiki/core':
directory => "${stage_dir}/php-master",
branch => 'master',
--
To view, visit https://gerrit.wikimedia.org/r/210618
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie063022b7b4bfc7fa4e1dac2fd15f32bf3a02d3d
Gerrit-PatchSet: 9
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Thcipriani <[email protected]>
Gerrit-Reviewer: Filippo Giunchedi <[email protected]>
Gerrit-Reviewer: Hashar <[email protected]>
Gerrit-Reviewer: Yuvipanda <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
