Giuseppe Lavagetto has uploaded a new change for review.
https://gerrit.wikimedia.org/r/179102
Change subject: mediawiki: enhancements to hhvm_cleanup_cache
......................................................................
mediawiki: enhancements to hhvm_cleanup_cache
* Import some of the additions in
Ib2a0401a7aa5632fb79a5b17c0d0cef8955cf990
* Move to /usr/local/sbin
* Move to mediawiki::hhvm::housekeeping, which is probably the best
place for this script
Change-Id: Ibcad98a95413044fd6c5e9bd3c0a6fb486bd5fe9
---
M modules/mediawiki/files/hhvm/cleanup_cache
M modules/mediawiki/manifests/hhvm.pp
M modules/mediawiki/manifests/hhvm/housekeeping.pp
3 files changed, 56 insertions(+), 24 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/02/179102/1
diff --git a/modules/mediawiki/files/hhvm/cleanup_cache
b/modules/mediawiki/files/hhvm/cleanup_cache
index e439c27..97e5cdb 100755
--- a/modules/mediawiki/files/hhvm/cleanup_cache
+++ b/modules/mediawiki/files/hhvm/cleanup_cache
@@ -1,15 +1,30 @@
-#!/usr/bin/python
-import urllib2
-import sqlite3, sys
+#!/usr/bin/env python
+"""
+ hhvm_cleanup_cache
+
+ Prune stale tables from the HHVM bytecode cache.
+ Tables are deemed unused if they reference a repo schema other than the
+ current one.
+"""
+import sys
import logging
from logging.handlers import SysLogHandler
+import os.path
+import subprocess
+import sqlite3
import argparse
+
+TABLES_QUERY = """
+SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%{}'
+"""
+
+
def get_repo_schema():
"""
Gets the repository schema version from the hhvm admin interface
"""
- response = urllib2.urlopen('http://localhost:9002/repo-schema')
- return response.read()
+ return subprocess.check_output(['/usr/bin/hhvm', '--repo-schema']).rstrip()
+
def delete_and_vacuum(dbh, tables):
"""
@@ -25,6 +40,7 @@
cur.execute("VACUUM")
log.info("Done")
+
def setup_logging(debug=False):
"""
Setting up logging
@@ -32,7 +48,7 @@
log_format = '%(name)s: %(levelname)s - %(message)s'
log = logging.getLogger('cleanup_hhvm_cache')
if not debug:
- #if normal mode, log to syslog
+ # if normal mode, log to syslog
log.setLevel(logging.INFO)
log.propagate = False
handler = SysLogHandler(
@@ -42,19 +58,26 @@
handler.setFormatter(formatter)
log.addHandler(handler)
else:
- #if debug mode, print to stderr
+ # if debug mode, print to stderr
logging.basicConfig(level=logging.DEBUG, format=log_format)
return log
-
def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--debug', action='store_true', default=False)
- parser.add_argument('--noop', action='store_true', default=False)
- parser.add_argument('filename')
+ parser = argparse.ArgumentParser(
+ prog="hhvm_cleanup_cache",
+ description="Prune unused entries from a HHVM bytecode cache database"
+ )
+ parser.add_argument('--debug', action='store_true',
+ default=False, help="print debug information to
stdout")
+ parser.add_argument('--noop', action='store_true', default=False,
+ help="show what would be done, but take no action")
+ parser.add_argument('filename',
+ help="the path of the bytecode cache database")
+
args = parser.parse_args()
log = setup_logging(args.debug)
+ repo_size_before = os.path.getsize(args.filename)
try:
repo_schema = get_repo_schema()
if not repo_schema:
@@ -65,9 +88,10 @@
hhvm_db = args.filename
with sqlite3.connect(hhvm_db) as dbh:
cursor = dbh.cursor()
- cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
- tables_to_clean = [table for (table,) in cursor.fetchall() \
- if not table.endswith(repo_schema)]
+ tables_to_clean = cursor.execute(
+ TABLES_QUERY.format(repo_schema)
+ ).fetchall()
+ # To remove the annoying unicode marker
printable_tables = ", ".join(tables_to_clean)
if args.noop:
log.info("Tables to remove: %s", printable_tables)
@@ -75,6 +99,9 @@
else:
log.debug("Tables to remove: %s", printable_tables)
delete_and_vacuum(dbh, tables_to_clean)
+ repo_size_after = os.path.getsize(args.repo)
+ kb_change = (repo_size_before - repo_size_after) / 1024.0
+ log.info('Pruned %.2f kB.', kb_change)
except Exception as e:
log.error("Execution failed with error %s", e)
sys.exit(1)
diff --git a/modules/mediawiki/manifests/hhvm.pp
b/modules/mediawiki/manifests/hhvm.pp
index 7c96b7a..fdeb83f 100644
--- a/modules/mediawiki/manifests/hhvm.pp
+++ b/modules/mediawiki/manifests/hhvm.pp
@@ -49,15 +49,6 @@
mode => '0555',
}
- # This command is useful prune the hhvm bytecode cache from old tables that
- # are just left around
-
- file { '/usr/local/bin/hhvm_cleanup_cache':
- source => 'puppet:///modules/mediawiki/hhvm/cleanup_cache',
- owner => 'root',
- group => 'root',
- mode => '0555',
- }
# Provision an Upstart task (a short-running process) that runs
# when HHVM is started and that warms up the JIT by repeatedly
diff --git a/modules/mediawiki/manifests/hhvm/housekeeping.pp
b/modules/mediawiki/manifests/hhvm/housekeeping.pp
index 2d87599..676ec72 100644
--- a/modules/mediawiki/manifests/hhvm/housekeeping.pp
+++ b/modules/mediawiki/manifests/hhvm/housekeeping.pp
@@ -14,4 +14,18 @@
require => [Service['hhvm'],Service['apache2']],
}
+ # This command is useful prune the hhvm bytecode cache from old tables that
+ # are just left around
+
+ file { '/usr/local/sbin/hhvm_cleanup_cache':
+ source => 'puppet:///modules/mediawiki/hhvm/cleanup_cache',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ }
+
+ # Temporary - remove the old script location
+ file { '/usr/local/bin/hhvm_cleanup_cache':
+ ensure => absent,
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/179102
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibcad98a95413044fd6c5e9bd3c0a6fb486bd5fe9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
