Ori.livneh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/179027
Change subject: mediawiki: tidy `cleanup_cache` script
......................................................................
mediawiki: tidy `cleanup_cache` script
* Place in /usr/local/sbin rather than /usr/local/bin.
* Rename to more descriptive (IMO) name 'hhbc-clean' ('cache' is ambiguous).
* Simplify logging by always logging to both stderr and syslog.
* PEP8 fixes.
Change-Id: Ib2a0401a7aa5632fb79a5b17c0d0cef8955cf990
---
D modules/mediawiki/files/hhvm/cleanup_cache
A modules/mediawiki/files/hhvm/hhbc-clean
M modules/mediawiki/manifests/hhvm.pp
3 files changed, 37 insertions(+), 87 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/27/179027/1
diff --git a/modules/mediawiki/files/hhvm/cleanup_cache
b/modules/mediawiki/files/hhvm/cleanup_cache
deleted file mode 100755
index e439c27..0000000
--- a/modules/mediawiki/files/hhvm/cleanup_cache
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/python
-import urllib2
-import sqlite3, sys
-import logging
-from logging.handlers import SysLogHandler
-import argparse
-def get_repo_schema():
- """
- Gets the repository schema version from the hhvm admin interface
- """
- response = urllib2.urlopen('http://localhost:9002/repo-schema')
- return response.read()
-
-def delete_and_vacuum(dbh, tables):
- """
- Drops stale tables and vacuums the database
- """
- log = logging.getLogger('cleanup_hhvm_cache')
- cur = dbh.cursor()
- log.info("Deleting tables")
- for table in tables:
- log.debug("Deleting table %s", table)
- cur.execute("DROP TABLE {}".format(table))
- log.info("Vacuuming the db")
- cur.execute("VACUUM")
- log.info("Done")
-
-def setup_logging(debug=False):
- """
- Setting up logging
- """
- log_format = '%(name)s: %(levelname)s - %(message)s'
- log = logging.getLogger('cleanup_hhvm_cache')
- if not debug:
- #if normal mode, log to syslog
- log.setLevel(logging.INFO)
- log.propagate = False
- handler = SysLogHandler(
- address='/dev/log',
- facility=SysLogHandler.LOG_LOCAL3)
- formatter = logging.Formatter(fmt=log_format)
- handler.setFormatter(formatter)
- log.addHandler(handler)
- else:
- #if debug mode, print to stderr
- logging.basicConfig(level=logging.DEBUG, format=log_format)
- return log
-
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--debug', action='store_true', default=False)
- parser.add_argument('--noop', action='store_true', default=False)
- parser.add_argument('filename')
- args = parser.parse_args()
- log = setup_logging(args.debug)
- try:
- repo_schema = get_repo_schema()
- if not repo_schema:
- log.error("Got an empty schema, cannot continue")
- sys.exit(1)
- else:
- log.info("Current schema version is %s", repo_schema)
- hhvm_db = args.filename
- with sqlite3.connect(hhvm_db) as dbh:
- cursor = dbh.cursor()
- cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
- tables_to_clean = [table for (table,) in cursor.fetchall() \
- if not table.endswith(repo_schema)]
- printable_tables = ", ".join(tables_to_clean)
- if args.noop:
- log.info("Tables to remove: %s", printable_tables)
- log.info("NOT deleting tables (noop)")
- else:
- log.debug("Tables to remove: %s", printable_tables)
- delete_and_vacuum(dbh, tables_to_clean)
- except Exception as e:
- log.error("Execution failed with error %s", e)
- sys.exit(1)
-
-if __name__ == '__main__':
- main()
diff --git a/modules/mediawiki/files/hhvm/hhbc-clean
b/modules/mediawiki/files/hhvm/hhbc-clean
new file mode 100644
index 0000000..9e1172c
--- /dev/null
+++ b/modules/mediawiki/files/hhvm/hhbc-clean
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+ Prune old tables from an HHVM HHBC file
+
+"""
+import logging
+import logging.handlers
+import sqlite3
+import subprocess
+
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+log.addHandler(logging.StreamHandler())
+log.addHandler(logging.handlers.SysLogHandler(
+ '/dev/log', logging.handlers.SysLogHandler.LOG_LOCAL3))
+
+schema = subprocess.check_output(('/usr/bin/hhvm', '--repo-schema')).rstrip()
+hhbc = '/run/hhvm/cache/fcgi.hhbc.sq3'
+
+try:
+ with sqlite3.connect(hhbc, timeout=10) as db:
+ cursor = db.cursor()
+ cursor.execute('select name from sqlite_master where type="table" '
+ ' and name not like "%%%s"' % schema)
+ for table, in cursor.fetchall():
+ log.info('dropping table "%s" from %s', table, hhbc)
+ db.execute('drop table %s' % table)
+ db.execute('vacuum')
+except:
+ log.exception('failed to prune %s:', hhbc)
diff --git a/modules/mediawiki/manifests/hhvm.pp
b/modules/mediawiki/manifests/hhvm.pp
index 7c96b7a..19b8009 100644
--- a/modules/mediawiki/manifests/hhvm.pp
+++ b/modules/mediawiki/manifests/hhvm.pp
@@ -49,11 +49,12 @@
mode => '0555',
}
- # This command is useful prune the hhvm bytecode cache from old tables that
- # are just left around
+ # Prune unused tables from HHVM's HHBC (sqlite bytecode repo).
+ # Tables are deemed unused if they reference a repo schema
+ # other than the one reported by 'hhvm --repo-schema'.
- file { '/usr/local/bin/hhvm_cleanup_cache':
- source => 'puppet:///modules/mediawiki/hhvm/cleanup_cache',
+ file { '/usr/local/sbin/hhbc-clean':
+ source => 'puppet:///modules/mediawiki/hhvm/hhbc-clean',
owner => 'root',
group => 'root',
mode => '0555',
--
To view, visit https://gerrit.wikimedia.org/r/179027
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib2a0401a7aa5632fb79a5b17c0d0cef8955cf990
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ori.livneh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
