All,
As of 4am last night, the Drizzle Automation team (read: me and monty)
were able to fully automate the running of code coverage analysis,
source code documentation with Doxygen, and production of historical
coverage data for each revision of the Drizzle server. (Yeah! \o/)
You can now always find the most recent code coverage reports at
http://drizzle.org/lcov and the most recent source code documentation at
http://drizzle.org/doxygen.
What's left to do? Well, let's discuss! I've attached the automation
script which is producing the coverage results in the hope that I can
get input from everyone on the following:
a) ways to improve the script
b) other pieces of data to collect and store historically
c) other stuff we should automate
Please feel free to comment and thanks in advance.
Cheers,
Jay
#! /usr/bin/python
# Script to automate lcov reporting
#
# We do the following in this script:
#
# 1) Pull a specific revno from the Drizzle development trunk
#
# 2) Build the server with coverage enabled
#
# 3) Zero out any coverage counters and run the Drizzle test suite
#
# 4) Run lcov --coverage on specific directories to produce coverage
# reports
#
# 5) Process the coverage reports to remove bad files since genhtml
# craps out on them
#
# 6) Combine reports into a single file and run genhtml on it,
# producing HTML reports of code coverage
#
# 7) Scrape the index.html report for percentages and log the numbers
# for all top-level directories in a database for historical capture
#
# 8) rsync the content up to drizzle.org
import sys
import os
import os.path
import logging
import commands
logging.basicConfig(stream= sys.stdout, level= logging.DEBUG, format='%(asctime)s %(levelname)s: %(message)s')
basedir= os.getcwd()
# The branch we'll run coverage reports on
coverage_branch= "lp:drizzle"
# Set of directories we're interested in running coverage reports on
coverage_dirs= [
'drizzled'
, 'client'
, 'libdrizzleclient'
, 'mysys'
, 'mystrings'
, 'storage/heap'
, 'storage/myisam'
, 'storage/innobase'
, 'storage/archive'
]
# Step #1a: Pull the latest changes for the branch
(retcode, output)= commands.getstatusoutput("bzr pull")
if not retcode == 0:
logging.error("""Failed to pull latest BZR.
This is likely because you have uncommitted changes in this branch.
Here is the output we got from bzr pull:
%s
Exiting.""" % output)
sys.exit(1)
# Step #1b: Grab the bzr info for the latest rev
(retcode, latest_revno)= commands.getstatusoutput("bzr revno")
if not retcode == 0:
logging.error("Failed to grab latest BZR revision number. Exiting.")
sys.exit(1)
logging.info("Processing code coverage in %s for %s at revision %s." % (basedir, coverage_branch, latest_revno))
# Step #1c: Clear any previous lcov stuff
logging.info("Clearing any previous run data.")
(retcode, ignored)= commands.getstatusoutput("make clean")
# Ignore the output. If it can't clean, it's likely b/c it's a fresh branch....
# Step #2a: Build Drizzle server with coverage enabled
logging.info("Configuring build for coverage and profiling.")
(retcode, ignored)= commands.getstatusoutput("./config/autorun.sh && ./configure --with-debug --enable-coverage --enable-profiling")
if not retcode == 0:
logging.error("Failed to configure with coverage and profiling enabled. Exiting.")
sys.exit(1)
# Step #2b: Reset all coverage counters
logging.info("Resetting all LCOV coverage counters.")
for dir in coverage_dirs:
cmd= "lcov --directory %s/%s --zerocounters" % (basedir, dir)
(retcode, output)= commands.getstatusoutput(cmd)
if not retcode == 0:
logging.error("Failed to zero-out coverage counters for %s/%s. Exiting." % (basedir, dir))
sys.exit(1)
# Step #2c: Build the server
logging.info("Building Drizzle server.")
(retcode, ignored)= commands.getstatusoutput("make -j2")
if not retcode == 0:
logging.error("Failed to build server.")
sys.exit(1)
# Step #2d: Run the test suite
logging.info("Running Drizzle test suite.")
(retcode, ignored)= commands.getstatusoutput("make test")
if not retcode == 0:
logging.error("Test suite failed to pass all tests. Please check test suite logs. Exiting")
sys.exit(1)
# Step #5: Produce coverage info files
logging.info("Producing GCOV info files for all directories.")
for dir in coverage_dirs:
cmd= "cd %s/%s; lcov --capture --directory . --base-directory . --output-file %s.out" % (basedir, dir, dir.replace(os.path.sep, '.'))
(retcode, output)= commands.getstatusoutput(cmd)
if not retcode == 0:
logging.error("Failed to zero-out coverage counters for %s/%s. Exiting." % (basedir, dir))
sys.exit(1)
# Step #6: Combine info files into single info file
logging.info("Aggregating GCOV info files.")
cmds= ["rm -rf lcov; mkdir -p lcov; cd %s; lcov" % basedir]
cmds= cmds + [
"-a %s/%s/%s.out" % (basedir, dir, dir.replace(os.path.sep, "."))
for dir in coverage_dirs
]
cmds= cmds + ["-o lcov/all.out"]
cmd= " ".join(cmds)
(retcode, output)= commands.getstatusoutput(cmd)
if not retcode == 0:
logging.error("Failed to aggregate GCOV files. Got error: %s" % output)
sys.exit(1)
# Step #7: Produce HTML coverage reports
logging.info("Producing LCOV HTML coverage reports.")
(retcode, output)= commands.getstatusoutput("genhtml -o lcov lcov/all.out")
if not retcode == 0:
logging.error("Failed to produce LCOV HTML coverage reports. Got error: %s" % output)
sys.exit(1)
# Step #8: rsync the reports to drizzle.org
logging.info("Syncing LCOV results with drizzle.org.")
(retcode, output)= commands.getstatusoutput("rsync -avz lcov/ drizzle.org:web/lcov/")
if not retcode == 0:
logging.error("Failed to rsync the reports to drizzle.org. Got error: %s" % output)
sys.exit(1)
# Step #9: Scrape the top-level percentages from lcov/index.html and store them in DB
logging.info("Storing results of this run in local DB.")
import MySQLdb
mysql_user= "XXXX"
mysql_pass= "XXXX"
mysql_db_name= "drizzle_lcov"
mysql_host= "localhost"
init_sql= """
CREATE DATABASE %s;
GRANT ALL ON drizzle_lcov TO %...@%s;
""" % (mysql_db_name, mysql_user, mysql_host)
create_sql= """
CREATE TABLE directories (
dir_name VARCHAR(255) NOT NULL PRIMARY KEY
, max_coverage_percent DECIMAL(5,2)
, min_coverage_percent DECIMAL(5,2)
) ENGINE=InnoDB;
CREATE TABLE directory_revision_history (
dir_name VARCHAR(255) NOT NULL
, revno INT NOT NULL
, run_date DATETIME NOT NULL
, coverage_percent DECIMAL(5,2) NOT NULL
, PRIMARY KEY (dir_name, revno)
) ENGINE=InnoDB;
"""
# If DB doesn't exist, create it.
try:
db= MySQLdb.connect(user= mysql_user, host= mysql_host, passwd= mysql_pass, db= mysql_db_name)
except:
logging.info("Initializing database.")
init_db_filename= "tmpinitdbfile.sql"
f= open(init_db_filename, 'w')
f.write(init_sql)
f.close()
(retcode, output)= commands.getstatusoutput("mysql --user=%s --password=%s --host=%s mysql < %s" % (mysql_user, mysql_pass, mysql_host, init_db_filename))
if not retcode == 0:
os.unlink(init_db_filename)
logging.error("Failed to init the database. Got error: %s" % output)
sys.exit(1)
else:
os.unlink(init_db_filename)
logging.info("Creating database.")
create_db_filename= "tmpcreatedbfile.sql"
f= open(create_db_filename, 'w')
f.write(create_sql)
f.close()
(retcode, output)= commands.getstatusoutput("mysql --user=%s --password=%s --host=%s %s < %s" % (mysql_user, mysql_pass, mysql_host, mysql_db_name, create_db_filename))
if not retcode == 0:
os.unlink(create_db_filename)
logging.error("Failed to create the database. Got error: %s" % output)
sys.exit(1)
else:
os.unlink(create_db_filename)
db= MySQLdb.connect(user= mysql_user, host= mysql_host, passwd= mysql_pass, db= mysql_db_name)
# Grab our index file for the lcov reports
index_file_contents= open("lcov/index.html").read()
# Scrape the percentages...
percentages= {}
for dir in coverage_dirs:
pattern= "href=\"%s/index.html\"" % dir
pos= index_file_contents.find(pattern)
if pos:
float_pattern= "alt=\""
new_pos= index_file_contents.find(float_pattern, pos)
if new_pos:
new_pos= new_pos + 5 # lenght of alt="
end_pos= new_pos + 1
while index_file_contents[end_pos].isdigit() or index_file_contents[end_pos] == '.':
end_pos= end_pos + 1
dir_percent= float(index_file_contents[new_pos:end_pos])
percentages[dir]= dir_percent
insert_db_filename= "tmpinsertdbfile.sql"
f= open(insert_db_filename, 'w')
# Insert into the DB...
for dir in coverage_dirs:
insert_revision_sql= """
INSERT INTO directory_revision_history (dir_name, revno, run_date, coverage_percent) VALUES ("%s",%d,NOW(),%0.2f) ON DUPLICATE KEY UPDATE run_date= NOW();
""" % (dir, latest_revno, percentages[dir])
f.write(insert_revision_sql)
# Update min/maxes for directories (this also accomplishes inserting initially...
update_minmax_sql= """
INSERT INTO directories (dir_name, max_coverage_percent, min_coverage_percent)
VALUES ("%s", %0.2f, %0.2f)
ON DUPLICATE KEY UPDATE
max_coverage_percent= IF(max_coverage_percent > %0.2f, max_coverage_percent, %0.2f)
, min_coverage_percent= IF(min_coverage_percent < %0.2f, min_coverage_percent, %0.2f);
""" % (dir, percentages[dir], percentages[dir], percentages[dir], percentages[dir], percentages[dir], percentages[dir])
f.write(update_minmax_sql)
f.close()
(retcode, output)= commands.getstatusoutput("mysql --user=%s --password=%s --host=%s %s < %s" % (mysql_user, mysql_pass, mysql_host, mysql_db_name, insert_db_filename))
if not retcode == 0:
os.unlink(insert_db_filename)
logging.error("Failed to insert into the database. Got error: %s" % output)
sys.exit(1)
else:
os.unlink(insert_db_filename)
logging.info("Inserted new data into database.")
_______________________________________________
Mailing list: https://launchpad.net/~drizzle-discuss
Post to : [email protected]
Unsubscribe : https://launchpad.net/~drizzle-discuss
More help : https://help.launchpad.net/ListHelp
